diff --git a/bin/bastion.py b/bin/bastion.py index 3fdec2d..404123c 100755 --- a/bin/bastion.py +++ b/bin/bastion.py @@ -11,7 +11,7 @@ from ruamel.yaml.scalarstring import PreservedScalarString logger = logging.getLogger() -logging.basicConfig(level = logging.WARN) +logging.basicConfig(level = logging.DEBUG) BIN_PATH = pathlib.Path(sys.argv[0]).absolute().parent @@ -26,8 +26,8 @@ from Bastion.Chronology import Quantim from Bastion.Site import Site from Bastion.Condo import Condex -from Bastion.Model import ARK -from Bastion.CARP import Request, isResult, isReport +from Bastion.Model import ARK, isAsset +from Bastion.CARP import Request, isResult, isReport, BagReceipt import Bastion.Vaults.HPSS import Bastion.Vaults.BFD import Bastion.Vaults.SFTP @@ -109,7 +109,7 @@ def logroot(self): def site(self, s): if isinstance(s, ARK): - return Site(s.name).configure(self.conf) + return Site(s.site).configured(self.conf) elif isinstance(s, str): #-- Assume that s is the name of a site. return Site(s).configured(self.conf) @@ -119,18 +119,31 @@ def site(self, s): def asset(self, ark): return self.site(ark.site).asset(ark) - def vault(self, name, site = None): + def vault(self, which, site = None): """ I answer an instance of a storage vault given a name and optional site. + .vault(name:str) + .vault(asset:isAsset) + .vault(ark:ARK) """ - if ((name[0] == '{') and (name[-1] == '}')): - name = name[1:-1] - if name in self.conf['vaults']: - protocol = self.conf['vaults'][name]['protocol'] - cls = Bastion.Model.Vault.handling(protocol) - return cls(name).configured(self.conf) - else: - return None + if isinstance(which, str): + name = which + if ((name[0] == '{') and (name[-1] == '}')): + name = name[1:-1] + if name in self.conf['vaults']: + protocol = self.conf['vaults'][name]['protocol'] + cls = Bastion.Model.Vault.handling(protocol) + return cls(name).configured(self.conf) + + if isinstance(which, isAsset): + asset = which + return self.vault(asset.policy.vault) + + if isinstance(which, ARK): + asset = self.asset(which) + return self.vault(asset.policy.vault) + + return None @property def sites(self): @@ -287,6 +300,7 @@ def emit_JSON(self, result, ostream): def emit_PROSE(self, result, ostream): request = result.request + ostream.write("ID: {}\n".format(request.ID)) ostream.write("request: {}\n".format(request.lede)) ostream.write("reply: {reply.code} {reply.gloss}\n".format(reply=result.status)) ostream.write("# {reply.lede}\n".format(reply=result)) @@ -385,7 +399,7 @@ def do_bank_zone(self, request): site = self.site(ark) zone = site.zone(ark) - arks = [asset.ark for asset in zone] + arks = [asset.ARK for asset in zone] request['log.scope'] = "site/{}".format(site.name) return self._bank_assets(request, arks) @@ -401,18 +415,17 @@ def _bank_assets(self, request, arks): """ helper method to push (bank) many assets given a single (batched) request. """ - results = [ ] + results = BagReceipt() for ark in arks: asset = self.asset(ark) - vault = self.vault(asset.policy.vault) + vault = self.vault(asset) result = vault.push(asset, client = self.hostname) results.append(result) - all_succeeded = all([result.indicates_success for result in results]) - all_failed = all([result.indicates_failure for result in results]) - - pushed = [result['blonde'] for result in results if result.indicates_success] + all_succeeded = all([result.succeeded for result in results]) + all_failed = all([result.failed for result in results]) + pushed = [result.blonde for result in results if result.succeeded] if all_succeeded: return request.succeeded(results, report = "all {} assets successfully pushed".format(len(results))) elif all_failed: @@ -482,7 +495,7 @@ def do_amend_zone(self, request): site = self.site(ark) zone = site.zone(ark) - arks = [asset.ark for asset in zone] + arks = [asset.ARK for asset in zone] request['log.scope'] = "site/{}".format(site.name) return self._amend_assets(request, arks) @@ -490,17 +503,17 @@ def _amend_assets(self, request, arks): """ helper method to amend (differential backup) many assets given a single (batched) request. """ - results = [ ] + results = BagReceipt() for ark in arks: asset = self.asset(ark) vault = self.vault(asset.policy.vault) result = vault.amend(asset, client = self.hostname) results.append(result) - all_succeeded = all([result.indicates_success for result in results]) - all_failed = all([result.indicates_failure for result in results]) - - amended = [result['blonde'] for result in results if result.indicates_success] + all_succeeded = all([result.succeeded for result in results]) + all_failed = all([result.failed for result in results]) + + amended = [result.blonde for result in results if result.succeeded] if all_succeeded: return request.succeeded(result, report = "all {} assets successfully amended".format(len(results))) @@ -686,9 +699,7 @@ def do_export_zones_provisioned(self, request): export zones provisioned {vault} {site} * lists all zones provisioned in the given vault for the named site """ - print(request.args[0]) vault = self.vault( request.args[0] ) - print(vault.name) site_name = request.args[1] zone_names = vault.zones(site_name) @@ -812,7 +823,7 @@ def do_refresh_keytab(self, request): rusina = app.site('rusina') soundscapes = rusina.zone('soundscapes') asset = soundscapes['HackathonData'] - vault = app.vault(asset.policy.vault) + vault = app.vault(asset) else: app.run( ) diff --git a/lib/Bastion/CARP.py b/lib/Bastion/CARP.py index c3b1280..5728a90 100644 --- a/lib/Bastion/CARP.py +++ b/lib/Bastion/CARP.py @@ -354,6 +354,36 @@ def toJDN(self, **kwargs): raise NotImplementedError(".toJDN is subclass responsibility") +class isBagReceipt: + """ + I am a thin wrapper around a sequence of results. + """ + def __init__(self): + self.results = [ ] + + def append(self, result, *args): + if isinstance(result, isResult): + self.results.append(result) + else: + raise ValueError("isBagReceipt: cannot append a value of type {}".format(type(result))) + + def __iter__(self): + return iter(self.results) + + + def __getitem__(self, i): + return self.results[i] + + def __len__(self): + return len(self.results) + + def toJDN(self, **kwargs): + return [toJDN(result) for result in self] + + +BagReceipt = isBagReceipt + + class isWorkflowReceipt(isReceipt, isTagged): """ I am a thin wrapper around a sequence of results. diff --git a/lib/Bastion/Clerks/SFTP.py b/lib/Bastion/Clerks/SFTP.py index 9a6edd2..42c62f8 100644 --- a/lib/Bastion/Clerks/SFTP.py +++ b/lib/Bastion/Clerks/SFTP.py @@ -11,7 +11,7 @@ class Clerk(isClerk): def __init__(self, vault, **kwargs): - isClerk.__init__(self) + isClerk.__init__(self, vault) self.vault = vault self.scurler = SCURLer(self.vault.sfURL, keyfile = self.vault.keypath) @@ -73,10 +73,12 @@ def manifest(self, *args): else: raise ValueError + blondes = [ ] cell = self.scurler / ark.site / ark.zone / ark.asset - if cell.exists(): - blondes = [ ] + print("CELL:", cell) + if cell.exists: for alien in cell.lsall(): + print(alien) if not alien.is_dir(): blondes.append( BLONDE.decode(alien.stem) ) manifest = Manifest(ark, blondes) diff --git a/lib/Bastion/Curator.py b/lib/Bastion/Curator.py index 99d3323..b3c26e3 100644 --- a/lib/Bastion/Curator.py +++ b/lib/Bastion/Curator.py @@ -86,7 +86,7 @@ def forFullBackup(cls, ark, **kwargs): @classmethod def forDiffBackup(cls, anchor, **kwargs): """ - Given a full backup reference (BLONDE), I generate a new BLONDE for a differential backup. + Given a full backup reference (anchor:BLONDE), I answer a new BLONDE for a differential backup. """ when = kwargs.get('when', datetime.datetime.now()) return cls(anchor.badge, 'D', anchor.genus, when) diff --git a/lib/Bastion/Model.py b/lib/Bastion/Model.py index 5272702..2b30d85 100644 --- a/lib/Bastion/Model.py +++ b/lib/Bastion/Model.py @@ -232,9 +232,9 @@ def amend(self, asset, **kwargs): Given an asset, I push a differential backup based on the most recent genus. """ #-- Get the current genus (aka full backup or anchor) - manifest = self.clerk.manifest(asset.ark) + manifest = self.clerk.manifest(asset.ARK) #-- Call .push with the genus as the basis for the differential backup. - return self.push(asset, manifest.head.genus, **kwargs) + return self.push(asset, manifest.head.anchor, **kwargs) def push(self, asset, basis = None, **kwargs): """ diff --git a/lib/Bastion/Movers/SFTP.py b/lib/Bastion/Movers/SFTP.py index 7a288f2..459579a 100644 --- a/lib/Bastion/Movers/SFTP.py +++ b/lib/Bastion/Movers/SFTP.py @@ -26,7 +26,7 @@ def provision(self, *args): provision(ark) - ensures that the site, zone, and asset folders exist. provision(site, zone, asset_name) - an alias for provision(ark) """ - raise NotImplementedError + pass def put(self, halo, tag, **kwargs): here = halo diff --git a/lib/Bastion/Packers/CARP.py b/lib/Bastion/Packers/CARP.py index 9133879..a6a991a 100644 --- a/lib/Bastion/Packers/CARP.py +++ b/lib/Bastion/Packers/CARP.py @@ -4,7 +4,7 @@ from Bastion.Model import isAsset from Bastion.CARP import isRequest, isReceipt -from Bastion.Curator import BLONDE +from Bastion.Curator import BLONDE, Snap logger = logging.getLogger(__name__) @@ -63,10 +63,14 @@ def __init__(self, subject, *args, **kwargs): whence = basis.when.earliest genus = basis.genus blonded = BLONDE.forDiffBackup(basis) - if isinstance(basis, datetime.datetime): + elif isinstance(basis, datetime.datetime): whence = basis genus = "___" blonded = BLONDE(asset.ARK, detail, genus) + elif isinstance(basis, Snap): + whence = basis.earliest + genus = basis.genus + blonded = BLONDE.forDiffBackup(basis.anchor) else: detail = 'F' blonded = BLONDE.forFullBackup(asset.ARK) diff --git a/lib/Bastion/Packers/TARs.py b/lib/Bastion/Packers/TARs.py index e9782cb..625bb58 100644 --- a/lib/Bastion/Packers/TARs.py +++ b/lib/Bastion/Packers/TARs.py @@ -10,7 +10,7 @@ from Bastion.Chronology import quantim from Bastion.Common import asPath, prefer -import Bastion.Model +from Bastion.Model import isAsset, isPacker from Bastion.CARP import isRequest, Report from Bastion.Packers.CARP import PackRequest, PackingReceipt @@ -61,11 +61,17 @@ def pax(tarp, folder, **kwargs): """ #-- If tar runs into any problems, it should throw an exception. with tarfile.open(tarp, "w", format = tarfile.PAX_FORMAT) as tar: + opts = { } if 'since' in kwargs: when = kwargs['since'] - tar.add(folder, filter = ifFileChanged(when)) - else: - tar.add(folder) + if isinstance(when, datetime.datetime): + opts['filter'] = ifFileChanged( kwargs['since'] ) + else: + raise ValueError("pax: since must be an instance of datetime") + if 'rebase' in kwargs: + opts['arcname'] = kwargs['rebase'] + + tar.add(folder, **opts) #-- if no exceptions were generated during the tar construction, #-- then we get here and we can return a happy True! @@ -73,9 +79,9 @@ def pax(tarp, folder, **kwargs): -class Packer(Bastion.Model.isPacker): +class Packer(isPacker): def __init__(self, vault, **kwargs): - Bastion.Model.isPacker.__init__(self, vault) + isPacker.__init__(self, vault) #-- find a scratch folder in order of lowest (default) to highest priority. #-- 1. /tmp #-- 2. vault's scratch folder (if any) @@ -88,10 +94,8 @@ def pack(self, subject, *args, **kwargs): Can be invoked as... .pack(request) .pack(asset) - .pack(asset, basis) - .pack(asset, since) - Without a given basis, I package everything (i.e. a full backup). - When a basis is given as either a datetime or an anchor (BLONDE), I do a differential backup. + .pack(asset, since:datetime) + Without a given "since", I package everything (i.e. a full backup). On success, I answer a report that wraps an instance of Packers.CARP.Packed On failure, I answer a ReportException Where... @@ -107,11 +111,13 @@ def pack(self, subject, *args, **kwargs): elif isinstance(subject, isRequest): #-- .pack(request:isRequest) request = PackRequest(subject) - else: + elif isinstance(subject, isAsset): #-- .pack(asset) #-- .pack(asset, basis) #-- .pack(asset, since) request = PackRequest(subject, *args, **kwargs) + else: + raise ValueError ark = request.asset.ARK package = "{}.tar".format(str(request.blonde)) @@ -124,7 +130,12 @@ def pack(self, subject, *args, **kwargs): try: #-- use the built-in python tar archiver, using "pax" (POSIX.1-2001) extensions. - pax(tarp, request.asset.halo, **kwargs) + opts = { + 'rebase': pathlib.PurePosixPath(ark.zone) / ark.asset + } + if request.basis is not None: + opts['since'] = request.since + pax(tarp, request.asset.halo, **opts) except Exception as err: report = request.failed(err) else: @@ -156,5 +167,5 @@ def catalog(tarp): for info in tark.getmembers(): sz = info.size mq = quantim( datetime.datetime.fromtimestamp(info.mtime) ) - cats.append( (sz, mq, info.name) ) + cats.append( [sz, mq, info.name] ) return cats diff --git a/lib/Bastion/Site.py b/lib/Bastion/Site.py index 4f2b807..37bc8c4 100644 --- a/lib/Bastion/Site.py +++ b/lib/Bastion/Site.py @@ -59,8 +59,10 @@ def __init__(self, name): self._catalogs = { } self._configured = False - def assets(self, zone): + def assets(self, zone, catalog = None): k = RDN(zone) + if catalog is not None: + self._catalogs[k] = catalog if k not in self._catalogs: self._catalogs[k] = AssetCatalog(self, k) return self._catalogs[k] @@ -112,26 +114,23 @@ def configured(self, conf): zname = str(zkey) #-- cast from Condo.CxKey to the zone's simple name. logger.info("reading assets for resource zone {} in site {}".format(zname, self.name)) zone = self.zone(zname) - for aspec in aspecs: - if entity(aspec).isString: - #-- Short form declaration of an asset is just the subfolder's name. - zone.assets.add( aspec ) - logger.debug("added asset {} to zone {} by short form description".format(aspec, zname)) + if isinstance(aspecs, str): + if aspecs[0] == '/': + zone.assets.monitor(aspecs) else: - #-- Long form declaration of an asset is a dictionary with additional definition key-value pairs. - zone.assets.add( Asset(zone, aspec) ) - logger.debug("added asset {} to zone {} by long form description".format(aspec, zname)) - + logger.warning("glob pattern should start with '/' for lazy loaded zone {}".format(zname)) + else: + for aspec in aspecs: + if entity(aspec).isString: + #-- Short form declaration of an asset is just the subfolder's name. + zone.assets.add( aspec ) + logger.debug("added asset {} to zone {} by short form description".format(aspec, zname)) + else: + #-- Long form declaration of an asset is a dictionary with additional definition key-value pairs. + zone.assets.add( Asset(zone, aspec) ) + logger.debug("added asset {} to zone {} by long form description".format(aspec, zname)) return self - def resources(self, zone): - zname = zone.name if isinstance(zone, Zone) else zone - - if zname not in self._catalogs: - self._catalogs[zname] = AssetCatalog(self, zname) - - return self._catalogs[zname] - def asset(self, ark): """ Will search through all zones to locate the asset identified by the given ARK. @@ -304,77 +303,131 @@ def modified(self): class AssetCatalog: def __init__(self, context, *args): + self._xRDNs = { } #-- map of RDN to index in my assets + self._xnames = { } #-- map of name to index in my assets + self._assets = tuple() #-- my assets, sorted by name. + self.globq = None #-- a glob pattern if assets are lazy loaded from zone's root folder. + self.fresh = True #-- False if we need to rescan the zone's folder + + largs = list(args) + if isinstance(context, Zone): self.zone = context self.site = context.site elif isinstance(context, Site): self.site = context - self.zone = self.site.zone(args[0]) + self.zone = self.site.zone(largs.pop()) elif isinstance(context, str): self.site = Site(context) - self.zone = self.site.zone(args[0]) + self.zone = self.site.zone(largs.pop()) else: raise Exception("AssetCatalog.__init__ - I don't know how to construct the requested AssetCatalog instance") - self.xRDNs = { } - self.xnames = { } - self._sorted = None + if largs: + #-- If we have an additional argument, this is the glob pattern for lazy loading. + self.monitor( largs.pop() ) @property - def any(self): - k = random.choice( list(self.xRDNs.keys()) ) - return self.xRDNs[k] + def assets(self): + if not self.fresh: + self.gather() + return self._assets - def add(self, obj): - if isinstance(obj, str): - asset = Asset(self.zone, obj) - elif isinstance(obj, pathlib.PurePath): - asset = Asset(self.zone, str(obj)) - elif isinstance(obj, Asset): - asset = obj - else: - raise ValueError("AssetCatalog.add - I don't know how to add an object of type {}".format(type(obj))) + @property + def xRDNs(self): + if not self.fresh: + self.gather() + return self._xRDNs - if asset.RDN not in self.xRDNs: - self.xRDNs[asset.RDN] = asset - self.xnames[asset.name] = asset - self._sorted = None - else: - raise Exception("duplicate asset RDN added!") + @property + def xnames(self): + if not self.fresh: + self.gather() + return self._xnames + + @property + def any(self): + return random.choice( self.assets ) - def __getitem__(self, x): - if isinstance(x, int): - if self._sorted is None: - self._sorted = sorted(list(self), key = lambda asset: asset.name) - return self._sorted[x] + def add(self, obj): + return self.extend( [obj] ) + + def extend(self, objs): + self._assets = list(self._assets) + for obj in objs: + if isinstance(obj, str): + asset = Asset(self.zone, obj) + elif isinstance(obj, pathlib.PurePath): + asset = Asset(self.zone, str(obj)) + elif isinstance(obj, Asset): + asset = obj + else: + raise ValueError("AssetCatalog.add - I don't know how to add an object of type {}".format(type(obj))) + self._assets.append(asset) + self.changed() + + return len(objs) + + def changed(self): + #-- invalidates any cached info + self._assets = tuple(sorted(self._assets, key = lambda asset: asset.name)) + self._xRDNs = { } + self._xnames = { } + for i, asset in enumerate(self._assets): + if asset.RDN not in self._xRDNs: + self._xRDNs[asset.RDN] = i + self._xnames[asset.name] = i + else: + raise Exception("duplicate asset RDN added!") + + def monitor(self, globQ): + self.globQ = pathlib.Path(globQ) + self.fresh = False + self.reset() + + def reset(self): + self._assets = tuple() + self._xRDNs = { } + self._xnames = { } + + def gather(self): + self._assets = tuple() + gathered = self.extend( [p.name for p in self.zone.root.glob(self.globQ.name)] ) + self.fresh = True + return gathered + + def __getitem__(self, k): + if isinstance(k, int): + return self.assets[x] else: - if x in self.xRDNs: - return self.xRDNs[x] + if k in self.xRDNs: + i = self.xRDNs[k] + return self.assets[i] + elif k in self.xnames: + i = self.xnames[k] + return self.assets[i] else: return None def __contains__(self, slug): return (slug in self.xRDNs) - def update(self, asset): - self.xRDNs[asset.RDN] = asset - self.xnames[asset.name] = asset - def named(self, name): - if name in self.xnames: - return self.xnames[name] + k = self.xnames.get(name, None) + if k is not None: + return self.assets[k] else: return None @property def names(self): - return tuple(sorted(self.xnames.keys())) + return tuple([asset.name for asset in self.assets]) def __len__(self): - return len(self.xRDNs) + return len(self.assets) def __iter__(self): - return iter(sorted(self.xRDNs.values(), key = lambda x: str(x.halo))) + return iter(self.assets) class Zone(isZone): @@ -412,7 +465,7 @@ def __div__(self, name): return Asset(self, name) def __getitem__(self, name): - return self.site.assets(self.name).named(name) + return self.site.assets(self).named(name) def __iter__(self): for asset in self.site.assets(self.name): diff --git a/lib/Bastion/Vaults/CARP.py b/lib/Bastion/Vaults/CARP.py index dd4ab35..744f699 100644 --- a/lib/Bastion/Vaults/CARP.py +++ b/lib/Bastion/Vaults/CARP.py @@ -2,7 +2,7 @@ import datetime from Bastion.CARP import isRequest, isWorkflowReceipt, Report -from Bastion.Curator import BLONDE +from Bastion.Curator import BLONDE, Snap logger = logging.getLogger(__name__) @@ -53,6 +53,7 @@ class PushRequest(isRequest): PushRequest(asset) - implies a full backup of the asset PushRequest(asset, since) - implies a differential backup of the asset relative to the given datetime (since) PushRequest(asset, basis) - implies a differential backup of the asset relative to the given basis (BLONDE) + PushRequest(asset, snap) - implies a differential backup of the asset relative to the anchor of the given snap. """ def __init__(self, subject, *args, **kwargs): if isinstance(subject, isRequest): @@ -71,10 +72,17 @@ def __init__(self, subject, *args, **kwargs): whence = basis.when.earliest genus = basis.genus blonded = BLONDE.forDiffBackup(basis) - if isinstance(basis, datetime.datetime): + elif isinstance(basis, datetime.datetime): whence = basis genus = "___" blonded = BLONDE(asset.ARK, detail, genus) + elif isinstance(basis, Snap): + snap = basis + whence = snap.earliest + genus = snap.genus + blonded = BLONDE.forDiffBackup(snap.anchor) + else: + raise ValueError("PushRequest.__init__ basis should be BLONDE or datetime not {}".format(type(basis))) else: detail = 'F' blonded = BLONDE.forFullBackup(asset.ARK) diff --git a/lib/Bastion/Vaults/Common.py b/lib/Bastion/Vaults/Common.py index 07dac8e..1900f44 100644 --- a/lib/Bastion/Vaults/Common.py +++ b/lib/Bastion/Vaults/Common.py @@ -45,6 +45,9 @@ def push(self, asset, basis = None, **kwargs): if packing.failed: return request.failed(receipt) + #-- Provision the receiving end, if needed. + self.provision(asset.ARK) + movement = self.put(packing.record.tarp, packing.record.tag) receipt.append(movement, 'moved')