From 5328fedd272a0a211108b74c98df27b1209a269b Mon Sep 17 00:00:00 2001 From: Nathan Denny Date: Thu, 3 Oct 2024 03:14:53 -0400 Subject: [PATCH] whirlwind of updates; rapidly evolving and debugging --- bin/bastion.py | 22 +++-- etc/{conf-idifhub.yaml => example-conf.yaml} | 0 lib/Bastion/Common.py | 13 ++- lib/Bastion/Curator.py | 92 +++++++++++--------- lib/Bastion/Model.py | 48 ++++++++-- lib/Bastion/Site.py | 10 ++- lib/Bastion/Vaults/BFD.py | 31 +++++-- 7 files changed, 148 insertions(+), 68 deletions(-) rename etc/{conf-idifhub.yaml => example-conf.yaml} (100%) diff --git a/bin/bastion.py b/bin/bastion.py index 6d4c4c6..dcbedf7 100755 --- a/bin/bastion.py +++ b/bin/bastion.py @@ -77,6 +77,7 @@ class App: CONF_SEARCH_ORDER = [ pathlib.Path('/etc/bastion'), APP_PATH / 'etc', + pathlib.Path('/CONF'), #-- mostly used for dockerized instances. pathlib.Path('~/.bastion').expanduser() ] @@ -154,6 +155,12 @@ def vault(self, name, site = None): else: return None + @property + def sites(self): + #-- I am an iterator over Site instances that I know. + for nick in self.conf['sites'].keys(): + yield self.site(nick) + def run(self): #-- scan the command line for options of the form "-{opt}:{value}" #-- options are removed from the command sequence @@ -355,6 +362,7 @@ def do_bank_zone(self, comargs, comdex, opts): * zone can be given as two arguments (site, zone) * zone can be given as a single argument in ARK format """ + raise NotImplementedError def do_bank_asset(self, comargs, comdex, opts): """ @@ -438,8 +446,8 @@ def do_update_asset(self, comargs, comdex, opts): vault = self.vault(asset.policy.vault) banked = vault.manifest(ark) if banked.anchors: - basis = banked.anchors[-1] - blonde = vault.push(asset, basis) + anchor = banked.anchors[-1] + blonde = vault.push(asset, anchor) else: blonde = vault.push(asset) @@ -576,7 +584,6 @@ def do_refresh_keytab(self, comargs, comdex, opts): extras['log.scope'] = "vault" return FAILED("vault {} is not declared".format(subject), context = extras) - if not callable( getattr(vault, 'refresh_keytab', None) ): #-- FAIL immediately since this vault doesn't declare #-- a method to refresh keytabs ... probably because the @@ -596,10 +603,11 @@ def do_refresh_keytab(self, comargs, comdex, opts): if sys.argv[1:] != ['shell']: app.run( ) else: - rusina = app.site('rusina') - soundscapes = rusina.zone('soundscapes') - asset = soundscapes['HackathonData'] - vault = app.vault(asset.policy.vault) + if os.environ.get('BASTION_SITE', None) == 'rusina': + rusina = app.site('rusina') + soundscapes = rusina.zone('soundscapes') + asset = soundscapes['HackathonData'] + vault = app.vault(asset.policy.vault) #-- does a full, level 0 backup #bastion backup site {site} diff --git a/etc/conf-idifhub.yaml b/etc/example-conf.yaml similarity index 100% rename from etc/conf-idifhub.yaml rename to etc/example-conf.yaml diff --git a/lib/Bastion/Common.py b/lib/Bastion/Common.py index b72e034..a59c7ac 100644 --- a/lib/Bastion/Common.py +++ b/lib/Bastion/Common.py @@ -35,7 +35,7 @@ def __init__(self, **kwargs): def RDN(x): """ Answers the relatively distinguishing name (RDN) of object, x. - If x is a string, it is assumed that x is the name. + If x is a string, it is assumed that x is already a name. If x is an object with an RDN attribute, answers x.RDN """ if isinstance(x, str): @@ -69,6 +69,10 @@ def isDuration(self): def isString(self): return isinstance(self.subject, str) + @property + def hasRDN(self): + return hasattr(self.subject, 'RDN') + class UnknownType(object): @@ -297,7 +301,11 @@ def Boggle(*args): #-- This is an error. raise ValueError + def asLogLevel(x): + """ + Accessor/wrapper for answering the ordinal log level (integer) that is assocationed with one of the common symbolic (string) log level names. + """ if isinstance(x, int): return x elif isinstance(x, str): @@ -308,4 +316,7 @@ def asLogLevel(x): def asPath(x): + """ + Accessor/wrapper for answering x already cast as a pathlib.Path instance + """ return pathlib.Path(x) diff --git a/lib/Bastion/Curator.py b/lib/Bastion/Curator.py index 4ffd4e6..672f33b 100644 --- a/lib/Bastion/Curator.py +++ b/lib/Bastion/Curator.py @@ -15,7 +15,7 @@ class BLONDE(canTextify): BLOb Name and Description Encoding I am a structured name describing a point in time for a single ARK. """ - def __init__(self, asset, detail, basis = None, when = None): + def __init__(self, asset, detail, genus = None, when = None): if isinstance(asset, ARK): self.badge = asset.badge elif isinstance(asset, isAsset): @@ -29,34 +29,40 @@ def __init__(self, asset, detail, basis = None, when = None): self.when = Quantim(when) self.detail = detail - self.basis = basis + self.genus = genus self.RDN = str(self) - #-- Automatically create a basis reference for full backups. - if (self.detail == 'F') and (self.basis is None): - self.basis = Boggle(3) + #-- Automatically create a new genus as lineage reference for full backups. + if (self.detail == 'F') and (self.genus is None): + self.genus = Boggle(3) - @property - def anchor(self): - """ - I am an alternate attribute name for the anchor reference ID. - """ - return self.basis + def revise(self, whence = None): + if self.isRevision: + raise Exception("Can only create revisions of anchor backups") + + if whence is not None: + return BLONDE.forDiffBackup(self) + else: + return BLONDE.forDiffBackup(self, when = whence) + + def age(self, whence = None): + whence = whence if (whence is not None) else datetime.datetime.now() + return (whence - self.when.earliest) @property def isAnchor(self): return (self.detail == 'F') @property - def isDifferential(self): + def isRevision(self): return (self.detail == 'D') @staticmethod - def encode(badge, when, detail, basis): - return "{}{}{}{}".format(badge, str(Quantim(when)), detail, basis) + def encode(badge, when, detail, genus): + return "{}{}{}{}".format(badge, str(Quantim(when)), detail, genus) def __str__(self): - return self.encode(self.badge, self.when, self.detail, self.basis) + return self.encode(self.badge, self.when, self.detail, self.genus) @classmethod def decode(cls, blonde): @@ -64,25 +70,23 @@ def decode(cls, blonde): 'badge': blonde[0:8], 'when': Quantim(blonde[8:16]), 'detail': blonde[16], - 'basis': blonde[17:20], + 'genus': blonde[17:20], }) - return cls(ds.badge, ds.detail, ds.basis, ds.when) + return cls(ds.badge, ds.detail, ds.genus, ds.when) @classmethod def forFullBackup(cls, ark, **kwargs): - when = kwargs.get('when', datetime.datetime.utcnow()) - basis = kwargs.get('basis', Boggle(3)) - return cls(ark, 'F', basis, when) + when = kwargs.get('when', datetime.datetime.now()) + genus = kwargs.get('genus', Boggle(3)) + return cls(ark, 'F', genus, when) @classmethod def forDiffBackup(cls, anchor, **kwargs): """ Given a full backup reference (BLONDE), I generate a new BLONDE for a differential backup. """ - when = kwargs.get('when', datetime.datetime.utcnow()) - return cls(anchor.badge, 'D', anchor.basis, when) - - + when = kwargs.get('when', datetime.datetime.now()) + return cls(anchor.badge, 'D', anchor.genus, when) class Thread(tuple): @@ -96,8 +100,12 @@ def anchor(self): return self[0] @property - def basis(self): - return self.anchor.basis + def revisions(self): + return tuple(self[1:]) + + @property + def genus(self): + return self.anchor.genus @property def earliest(self): @@ -109,11 +117,11 @@ def latest(self): @property def begins(self): - return self.anchor.when.datetime() + return self.earliest.when.datetime() @property def ends(self): - return self.head.when.datetime() + return self.latest.when.datetime() @property def drift(self): @@ -125,10 +133,10 @@ def drift(self): #-- Each snap is a 2-tuple of (anchor, differential) #-- An "anchor" is a full backup of the dataset. #-- Each blob in the archive is recorded as ... -#-- {slug}{quantim}[A|D]{anchor} +#-- {slug}{quantim}[F|D]{anchor} #-- Where {slug} is the Slug40 encoding (a 8 character, base32 word) of the dataset name (relative to the Rz), #-- {quantim} is the 8 character encoding of timestamp using the Quantim method -#-- A if the blob is an anchor (full backup) and D if the blob is a differential. +#-- F if the blob is an anchor (full backup) and D if the blob is a differential. #-- {anchor} - is a 3 character random string that cannot conflict with any other anchors currently in the archive. class Manifest(canTextify): """ @@ -144,7 +152,7 @@ def __init__(self, asset, items): blondes = list(items) self._items = tuple(sorted([blonde for blonde in blondes if (blonde.badge == self.badge)], key = lambda b: b.RDN)) - self._anchors = dict([(item.basis, item) for item in self._items if item.isAnchor]) + self._anchors = dict([(item.genus, item) for item in self._items if item.isAnchor]) def __iter__(self): return iter(self._items) @@ -179,10 +187,10 @@ def thread(self, ankle): """ if isinstance(ankle, str): anchor = ankle - elif isinstance(ankle, snap): - anchor = ankle.anchor.basis + elif isinstance(ankle, Snap): + anchor = ankle.anchor.genus elif isinstance(ankle, BLONDE): - anchor = ankle.basis + anchor = ankle.genus return Thread([snap for snap in self.snaps if snap.basis == anchor]) def anchor(self, item): @@ -227,21 +235,21 @@ def __init__(self, head, anchor): self.anchor = anchor @property - def basis(self): - return self.anchor.basis + def genus(self): + return self.anchor.genus def age(self, whence = None): """ I answer a datetime timedelta (elapsed time) between whence and the encoded datetime of this snap. - If no "whence" is explicitly given, I assume the current UTC time. + If no "whence" is explicitly given, I assume the current local time. """ - whence = whence if whence is not None else datetime.datetime.utcnow() - return (whence - self.head.when.datetime()) + whence = whence if whence is not None else datetime.datetime.now() + return (whence - self.head.when.datetime) @property def drift(self): #-- elapsed time between head and its anchor. - return (self.head.when.datetime() - self.anchor.when.datetime()) + return (self.head.when.datetime - self.anchor.when.datetime) @property def isAnchor(self): @@ -250,3 +258,7 @@ def isAnchor(self): @property def isDifferential(self): return self.head.isDifferential + + @property + def isRevision(self): + return self.head.isRevision diff --git a/lib/Bastion/Model.py b/lib/Bastion/Model.py index 032811d..68215a8 100644 --- a/lib/Bastion/Model.py +++ b/lib/Bastion/Model.py @@ -3,7 +3,7 @@ """ import pathlib -from .Common import RDN, CURIE, Slug40 +from .Common import RDN, CURIE, Slug40, entity class ARK(tuple): @@ -76,8 +76,14 @@ def badge(self): return Slug40(str(self.CURIE)) +class isVault: + """ + abstract class for all vaults. + """ + pass -class Vault: + +class Vault(isVault): """ I am the base class for all storage vaults. """ @@ -190,13 +196,6 @@ def RDN(self): self._RDN = self.badge return self._RDN -# @property -# def path(self): -# """ -# I answer the local (host) file system path to this asset. -# """ -# return self.zone.root / pathlib.Path(self.name) - @property def CURIE(self): """ @@ -225,6 +224,9 @@ def halo(self): @property def zolo(self): + """ + zolo (zone location) is the path of this object relative to it's zone. + """ return self.ARK.zolo @@ -248,3 +250,31 @@ def RDN(self): def __div__(self, name): return self.ASSET_CLS(self, name) + +class isSite: + """ + asbtract site class; forward declaration of Site class(es). + """ + pass + + +#-- Monkey patch the "entity" class for syntax sugar. +def entity_isSite(self): + return isinstance(self.subject, isSite) + +def entity_isAsset(self): + return isinstance(self.subject, isAsset) + +def entity_isZone(self): + return isinstance(self.subject, isZone) + +def entity_isARK(self): + return isinstance(self.subject, isARK) + +def entity_isVault(self): + return isinstance(self.subject, isVault) + +entity.isAsset = property(entity_isAsset) +entity.isZone = property(entity_isZone) +entity.isSite = property(entity_isSite) +entity.isVault = property(entity_isVault) diff --git a/lib/Bastion/Site.py b/lib/Bastion/Site.py index 9917b42..a5e0045 100644 --- a/lib/Bastion/Site.py +++ b/lib/Bastion/Site.py @@ -7,7 +7,7 @@ from .Common import * from .Condo import CxNode -from .Model import ARK, isAsset, isZone +from .Model import ARK, isAsset, isZone, isSite #from .Curator import Asset logger = logging.getLogger(__name__) @@ -19,7 +19,7 @@ DEFAULT_ASSET_HISTORY = 60 * DAYS DEFAULT_ANCHOR_DRIFT = 30 * DAYS DEFAULT_ARCHIVE_VAULT = "fortress" -DEFAULT_LOGGING_PATH = pathlib.Path("/var/log/bastion") +DEFAULT_LOGGING_PATH = pathlib.Path("~/.bastion/log").expanduser() class canConfigure: @@ -35,7 +35,7 @@ def configured(condex): -class Site(canConfigure, canTextify): +class Site(isSite, canConfigure, canTextify): SITES = { } @staticmethod @@ -420,4 +420,8 @@ def __div__(self, name): def __getitem__(self, name): return self.site.assets(self.name).named(name) + def __iter__(self): + for asset in self.site.assets(self.name): + yield asset + diff --git a/lib/Bastion/Vaults/BFD.py b/lib/Bastion/Vaults/BFD.py index a8f53f6..54b8609 100644 --- a/lib/Bastion/Vaults/BFD.py +++ b/lib/Bastion/Vaults/BFD.py @@ -52,6 +52,9 @@ def configured(self, conf): #↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ @property def ARKs(self): + """ + I answer a sorted collection of all assets (ARKs) held in this vault. + """ arks = [ ] for site in self.sites: for zone in self.zones(site): @@ -61,6 +64,9 @@ def ARKs(self): @property def sites(self): + """ + I answer a sorted collection of the names of all sites known to this vault. + """ #-- sites are top level elements relative to the root of the vault. sites = [ ] for entry in self.root.iterdir(): @@ -70,10 +76,13 @@ def sites(self): return tuple(sorted(sites)) def zones(self, site): + """ + Given a site name, I answer a sorted collection of zone names that are known to this vault. + """ #-- a zone will be a subdirectory (subfolder) of the given site. #-- look for all of the subfolders of root / site zones = [ ] - sroot = self.root / site + sroot = self.root / RDN(site) if sroot.exists( ): for entry in sroot.iterdir( ): if entry.is_dir(): @@ -96,7 +105,12 @@ def manifest(self, *args): manifest(site, zone, asset) """ if len(args) == 1: - return self._manifest_ark( args[0] ) + arg = args[0] + if isinstance(arg, Bastion.Model.isAsset): + ark = arg.ARK + else: + ark = arg + return self._manifest_ark( ark ) elif len(args) == 3: return self._manifest_site_zone_asset( args[0], args[1], args[2] ) else: @@ -124,22 +138,23 @@ def push(self, asset, basis = None, **kwargs): """ detail = 'F' whence = None - branch = None + genus = None blonded = None tarp = None ark = asset.ARK opts = { } - if basis: + if anchor: detail = 'D' if isinstance(basis, BLONDE): whence = basis.when.earliest - branch = basis.anchor + genus = basis.genus if isinstance(basis, datetime.datetime): whence = basis - branch = "___" - blonded = BLONDE(asset.ARK, detail, branch) - tarp = "{}.tar".format(str(blonded)) + genus = "___" + blonded = BLONDE(asset.ARK, detail, genus) + + tarp = "{}.tar".format(str(blonded)) opts['since'] = whence else: