From 2c63a62eb2f1ed9e89da5fa709a2f24fa40ae2a5 Mon Sep 17 00:00:00 2001 From: Nathan Denny Date: Sat, 23 Nov 2024 20:35:19 -0500 Subject: [PATCH] Working on packer and improving human readable report output. --- lib/Bastion/CARP.py | 37 +++++++++- lib/Bastion/Common.py | 19 +++++ lib/Bastion/Humane.py | 30 ++++++++ lib/Bastion/Model.py | 13 ++++ lib/Bastion/Movers/SFTP.py | 34 +-------- lib/Bastion/Packers/CARP.py | 13 +--- lib/Bastion/Packers/TARs.py | 140 +++++++++++++++++++++++++++++++++--- 7 files changed, 232 insertions(+), 54 deletions(-) create mode 100644 lib/Bastion/Humane.py diff --git a/lib/Bastion/CARP.py b/lib/Bastion/CARP.py index 6fda2ee..6febfde 100644 --- a/lib/Bastion/CARP.py +++ b/lib/Bastion/CARP.py @@ -4,10 +4,10 @@ Common Action-Result/Report Protocol """ import uuid +import traceback from Bastion.Common import toJDN - class ReplyStatus(tuple): DEFAULT_CATEGORY_GLOSS = { '1': 'FYI', @@ -166,6 +166,10 @@ def __init__(self, request, status, obj, **kwargs): if context is not None: self.context = context.copy() + @property + def elapsed(self): + return (self.when - self.request.when) + def toJDN(self, **kwargs): #-- Build result stanza. result = { @@ -174,6 +178,7 @@ def toJDN(self, **kwargs): 'message': self.status.gloss, 'lede': self.lede, 'answered': self.when.isoformat( ), + 'elapsed': self.elapsed.total_seconds(), 'context': { }, 'record': toJDN(self.record) } @@ -205,7 +210,7 @@ class Receipt(isReceipt): class Report(isReceipt): - def __init__(self, request, status, *args, **kwargs): + def __init__(self, request, status, obj = None, *args, **kwargs): isReceipt.__init__(self, request, status, *args, **kwargs) self._body = kwargs.get('report', None) @@ -215,7 +220,7 @@ def changed(self, *args): def toJDN(self, **kwargs): jdn = isReceipt.toJDN(self, **kwargs) - jdn['request']['result']['report'] = str(self.report) + jdn['request']['result']['report'] = str(self) return jdn @property @@ -225,6 +230,32 @@ def body(self): return self._body +class ReportException(Report): + def __init__(self, request, *args, **kwargs): + #-- Can be called as... + #-- ReportException(request, errobj), or... + #-- ReportException(request, status, errobj) + if isinstance(args[0], ReplyStatus): + status = args[0] + errobj = args[1] + else: + status = ReplyStatus.failed + errobj = args[0] + + assert status.indicates_failure, "ReportException must be created with a reply status indicating failure" + assert isinstance(errobj, Exception), "error object must be an instance of Exception" + + Report.__init__(self, request, status, errobj) + + self.xobj = x + self.xname = type(xobj).__name__ + self.xmsg = str(xobj) + + def __str__(self): + return ''.join( traceback.format_exception(self.xobj) ) + + + class Request(isRequest): #-- Subclasses can set their own REPORT. diff --git a/lib/Bastion/Common.py b/lib/Bastion/Common.py index 653d229..346c9d3 100644 --- a/lib/Bastion/Common.py +++ b/lib/Bastion/Common.py @@ -351,6 +351,25 @@ def Boggle(*args): raise ValueError +def prefer(options, **kwargs): + """ + prefer(options) - chooses the first item in sequence (options) that is not None + Optional keyword args... + * choose - a function that returns True for options that should be in the selection. + * n - an integer, if larger than 1 answers a sequence of up to n items, if 1 (default) answers the first item (scalar) in the selection. + """ + choose = kwargs.get("choose", lambda x: (x is not None)) + n = kwargs.get("n", 1) + + choices = [option for option in options if choose(option)] + + if n == 1: + return choices[0] + + return choiecs[:n] + + + def asLogLevel(x): """ Accessor/wrapper for answering the ordinal log level (integer) that is assocationed with one of the common symbolic (string) log level names. diff --git a/lib/Bastion/Humane.py b/lib/Bastion/Humane.py new file mode 100644 index 0000000..0ec444b --- /dev/null +++ b/lib/Bastion/Humane.py @@ -0,0 +1,30 @@ +""" +Bastion.Humane + +Utilities mostly for creating "humane" (i.e. human scale) messages. +""" +import datetime + +from Bastion.Common import Alchemist + +humanize = Alchemist('Humane', "humanize") + +def _humanize_timedelta(elapsed, **kwargs): + dt = elapsed.total_seconds( ) + + if dt > (86400*2): + days = round(dt/86400.0, 1) + return "{:.1f} days".format(days) + + if dt > 7200.0: + hrs = round(dt/3600.0, 1) + return "{:.1f} hours".format(hrs) + + if dt > 600.0: + mins = round(dt/60.0, 1) + return "{:.1f} minutes".format(dt / mins) + + return "{:.1f} seconds".format(dt) + + +humanize.spells.append( (datetime.timedelta, _humanize_timedelta) ) diff --git a/lib/Bastion/Model.py b/lib/Bastion/Model.py index 747548e..07ea99f 100644 --- a/lib/Bastion/Model.py +++ b/lib/Bastion/Model.py @@ -320,6 +320,10 @@ class isClerk: """ abstract class for metadata and file management specific to the capabilities of a given vault type. """ + def __init__(self, vault): + assert isinstance(vault, isVault), "vault must be an instance of Bastion.Model.isVault" + self.vault = vault + @property def ARKs(self): """ @@ -359,6 +363,11 @@ def manifest(self, ark): class isPacker: + def __init__(self, vault): + assert isinstance(vault, isVault), "vault must be an instance of Bastion.Model.isVault" + + self.vault = vault + def pack(self, asset, basis = None, **kwargs): """ Given a local asset, I package (.tar, .zip, etc) the asset into my scratch (spool) space. @@ -386,6 +395,10 @@ class isMover: """ abstract class for file movement in to and out of a specific vault type. """ + def __init__(self, vault): + assert isinstance(vault, isVault), "vault must be an instance of Bastion.Model.isVault" + self.vault = vault + def provision(self, *args): """ provision(ark) - ensures that the site, zone, and asset folders exist. diff --git a/lib/Bastion/Movers/SFTP.py b/lib/Bastion/Movers/SFTP.py index 960f03a..55b7ced 100644 --- a/lib/Bastion/Movers/SFTP.py +++ b/lib/Bastion/Movers/SFTP.py @@ -31,22 +31,9 @@ def asPurePath(x): class Mover(Bastion.Model.isMover): def __init__(self, vault, **kwargs): - isClerk.__init__(self) - self.vault = vault + Bastion.Model.isMover.__init__(self, vault) self.scurler = SCURLer(self.vault.sfURL, keyfile = self.vault.keypath) - @property - def mover(self): - if getattr(self, '_mover', None) is None: - self._mover = Mover(self, self.host, self.login, self.keypath) - return self._mover - - @property - def clerk(self): - if getattr(self, '_clerk', None) is None: - self._clerk = Clerk(self, target, self.host, self.login, self.keypath) - return self._clerk - @property def bank(self): client = SCURLer(self.login, self.host, self.root, keyfile = self.key) @@ -59,14 +46,7 @@ def provision(self, *args): provision(ark) - ensures that the site, zone, and asset folders exist. provision(site, zone, asset_name) - an alias for provision(ark) """ - ark = None - if len(args) == 1: - ark = args[0] - elif len(args) == 3: - ark = ARK(args[0], args[1], args[2]) - else: - raise ValueError - + ark = ARK(*args) repo = self.scurler / ark.site / ark.zone / ark.asset return repo.mkdir(parents = True, exist_ok = True) @@ -116,7 +96,6 @@ def pack(self, asset, basis = None, **kwargs): #-- use the built-in python tar archiver, using "pax" (POSIX.1-2001) extensions. pax((self.scratch / tag), asset.halo, **opts) - #-- Answer the receipt. return receipt @@ -165,12 +144,3 @@ def get(self, tag, halo, **kwargs): #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ #-- END Bastion.Model.Vault PROTOCOL | #------------------------------------- - def _provision_ark(self, ark): - repo = self.bank / ark.site / ark.zone / ark.asset - return repo.mkdir(parents = True, exist_ok = True) - - def _provision_site_zone_asset(self, site, zone, asset_name): - return self._provision_ark( ARK(site, zone, asset_name) ) - - -Vault.register() diff --git a/lib/Bastion/Packers/CARP.py b/lib/Bastion/Packers/CARP.py index eeeedb1..2bff559 100644 --- a/lib/Bastion/Packers/CARP.py +++ b/lib/Bastion/Packers/CARP.py @@ -9,16 +9,7 @@ logger = logging.getLogger(__name__) -class PackReportSuccess(Report): - def __init__(self, request, status, *args, **kwargs): - Report.__init__(self, request, status, *args, **kwargs) - - -class PackRequest(Request): - REPORTS = { - RequestStatus.Ok.code: PackReportSuccess - } - +class PackRequest(Bastion.CARP.Request): def __init__(self, asset, basis = None, **kwargs): #-- Which asset will be packed? #-- If differential backup, what basis is used for determining changes? @@ -29,7 +20,7 @@ def __init__(self, asset, basis = None, **kwargs): if basis: self.context['detail'] = 'D' - self.context['basis'] = str(kwargs['basis']) + self.context['basis'] = str(basis) self.context['whence'] = kwargs['whence'].isoformat() if 'genus' in kwargs: self.context['genus'] = kwargs['genus'] diff --git a/lib/Bastion/Packers/TARs.py b/lib/Bastion/Packers/TARs.py index 8609469..7ded4ce 100644 --- a/lib/Bastion/Packers/TARs.py +++ b/lib/Bastion/Packers/TARs.py @@ -6,9 +6,11 @@ import datetime import pathlib import tarfile +import tempfile import logging from Bastion.Common import * +from Bastion.CARP import ReportException logger = logging.getLogger(__name__) @@ -45,7 +47,7 @@ def __call__(self, tarinfo): return None -def pax(tarp, asset, **kwargs): +def pax(tarp, folder, **kwargs): """ pax uses the python tarfile module to create a tar file using the extended POSIX.1-2001 (aka "PAX") format. @@ -54,18 +56,140 @@ def pax(tarp, asset, **kwargs): {asset} can be a file path or an instance of Bastion.Model.isAsset (e.g. Bastion.Site.Asset) differential backups can be done by supplying the "since" keyword set the datetime which is the earliest allowed modification time for files to be admitted into the tar. """ - if entity(asset).isAsset: - src = asset.halo - else: - src = pathlib.Path(asset) - + #-- If tar runs into any problems, it should throw an exception. with tarfile.open(tarp, "w", format = tarfile.PAX_FORMAT) as tar: if 'since' in kwargs: when = kwargs['since'] - tar.add(src, filter = ifFileChanged(when)) + tar.add(folder, filter = ifFileChanged(when)) else: - tar.add(src) + tar.add(folder) #-- if no exceptions were generated during the tar construction, #-- then we get here and we can return a happy True! return True + + + +class PaxReportSuccess(Report): + TEMPLATE = """ +# REQUEST {UUID} +Request ({request}) opened at {opened}. +# RESULT {UUID} {code} {gloss} +{cls} answered request at {answered} ({elapsed}). +Archive created for {asset} at host area path {halo}. +Archive is {size} and contains {toc} files. +## CATALOG +The following items are included in the archive... +{items} +""" + TEMPLATE_CATALOG_ITEM = "* {size} {modified} {path}" + + def __init__(self, request, catalog): + Report.__init__(self, request, Resultstatus.Ok, catalog) + + def __str__(self): + """ + Answers a textual representation of self which is the "print" form of self. + """ + #-- ask the request to generate a markdown section for itself. + h1request = self.request.toMD() + + #-- build the "madlibs" structure to substitute into the result section. + h1result = TEMPLATE_RESULT.format(**madlibs) + + #-- build the catalog (sub)section + #-- first, we build the items stanza... + txitems = '\n'.join( [TEMPLATE_CATALOG_ITEM.format(**entry) for entry in self.catalog] ) + h2catalog = TEMPLATE_H2CATALOG.format(items = txitems) + + + + +class Packer(Bastion.Model.isPacker): + def __init__(self, vault, **kwargs): + Bastion.Model.isPacker.__init__(self, vault) + + #-- find a scratch folder in order of lowest (default) to highest priority. + #-- 1. /tmp + #-- 2. vault's scratch folder (if any) + #-- 3. "scratch" kwarg override + self.scratch = asPath( prefer([kwargs.get("scratch", None), getattr(self.vault, "scratch", None), "/tmp"]) ) + + def pack(self, asset, basis = None, **kwargs): + """ + Given a local asset, I package (.tar, .zip, etc) the asset into my scratch (spool) space. + Without a given basis, I package everything (i.e. a full backup). + When a basis is given as either a datetime or an anchor (BLONDE), I do a differential backup. + I answer a tuple like (blonde, tag, repo, packaged) + Where... + * blonde is the BLONDE of the newly created archive + * tag is the relative path to the archive file + * spool is the absolute path to the folder holding the new archive file. + * package is the name of the archive file in the spool folder + """ + detail = 'F' + whence = None + basis = None + genus = None + blonded = None + tarp = None + ark = asset.ARK + opts = { } + + if basis: + detail = 'D' + if isinstance(basis, BLONDE): + #-- I was given a BLONDE (a reference to a full backup) + anchor = basis + whence = anchor.when.earliest + genus = anchor.genus + blonded = BLONDE.forDiffBackup(anchor) + + if isinstance(basis, datetime.datetime): + whence = basis + genus = "___" + blonded = BLONDE(asset.ARK, detail, genus) + + opts['since'] = whence + + else: + blonded = BLONDE.forFullBackup(asset.ARK) + + package = "{}.tar".format(str(blonded)) + tag = pathlib.PurePosixPath(ark.site) / ark.zone / ark.asset / package + spool = (self.scratch / tag).parent + + #-- assure that the scratch path exists and all of the subpaths that create the repo folder for this asset. + spool.mkdir(parents = True, exist_ok = True) + + #-- generate a formal request object. + context = { } + if whence: + context['whence'] = whence + if genus: + context['genus'] = genus + request = PackRequest(asset, basis, **context) + + #-- use the built-in python tar archiver, using "pax" (POSIX.1-2001) extensions. + try: + pax((self.scratch / tag), asset.halo, **opts) + except Exception as err: + report = ReportException(request, err) + else: + report = PaxReportSuccess(request) + + #↑↑↑↑↑↑ Need to do a clean version of what exactly the catalog is and how its made and also how an exception is described in the failure report. + + #-- answer the BLONDE of the newly created package. + return (blonded, tag, spool, package) + + + def unpack(self, halo, root, **kwargs): + """ + Given a local packed archive object (e.g. .tar, .zip, etc.) at halo, + I unpack the archive into the given (local) root path. + """ + raise NotImplementedError + + +