From decc7cc8a8493e92c5384469575d7890d5637591 Mon Sep 17 00:00:00 2001 From: Nathan Denny Date: Mon, 6 Jan 2025 06:26:09 -0500 Subject: [PATCH] refactoring and debugging ... still a work in progress --- bin/bastion.py | 13 ++++++++++--- lib/Bastion/CARP.py | 2 +- lib/Bastion/Clerks/BFD.py | 11 +++++------ lib/Bastion/Common.py | 37 ++++++++++++++++++++---------------- lib/Bastion/Condo.py | 2 +- lib/Bastion/Curator.py | 6 +++--- lib/Bastion/Model.py | 7 ++++--- lib/Bastion/Movers/BFD.py | 20 +++++++++---------- lib/Bastion/Movers/CARP.py | 16 ++++++++-------- lib/Bastion/Packers/CARP.py | 16 +++++++++------- lib/Bastion/Site.py | 18 +++--------------- lib/Bastion/Vaults/BFD.py | 3 ++- lib/Bastion/Vaults/CARP.py | 2 +- lib/Bastion/Vaults/Common.py | 8 ++++---- 14 files changed, 81 insertions(+), 80 deletions(-) diff --git a/bin/bastion.py b/bin/bastion.py index b505e32..8ffb46d 100755 --- a/bin/bastion.py +++ b/bin/bastion.py @@ -29,8 +29,8 @@ from Bastion.Condo import * from Bastion.Actions import * from Bastion.Model import ARK -from Bastion.CARP import Request, isReceipt -import Bastion.Vaults.HPSS +from Bastion.CARP import isRequest, isReceipt +#import Bastion.Vaults.HPSS import Bastion.Vaults.BFD """ @@ -603,11 +603,18 @@ def do_refresh_keytab(self, comargs, comdex, opts): if sys.argv[1:] != ['shell']: app.run( ) else: - if os.environ.get('BASTION_SITE', None) == 'rusina': + vehicle = os.environ.get('BASTION_SITE', None) + if vehicle == 'rusina': rusina = app.site('rusina') soundscapes = rusina.zone('soundscapes') asset = soundscapes['HackathonData'] vault = app.vault(asset.policy.vault) + elif vehicle == 'scout': + scout = app.site('scout') + ADS = scout.zone('ADS') + asset = ADS['FSIL'] + vault = app.vault(asset.policy.vault) + #-- does a full, level 0 backup #bastion backup site {site} diff --git a/lib/Bastion/CARP.py b/lib/Bastion/CARP.py index 189cf21..7e5796e 100644 --- a/lib/Bastion/CARP.py +++ b/lib/Bastion/CARP.py @@ -281,7 +281,7 @@ def __str__(self): class Report(isReport): @classmethod def Success(cls, request, record = None, *args, **kwargs): - return cls(request. ResultStatus.Ok, record, *args, **kwargs) + return cls(request, ResultStatus.Ok, record, *args, **kwargs) @classmethod def Failed(cls, request, record = None, *args, **kwargs): diff --git a/lib/Bastion/Clerks/BFD.py b/lib/Bastion/Clerks/BFD.py index 11a1de4..0a0deec 100644 --- a/lib/Bastion/Clerks/BFD.py +++ b/lib/Bastion/Clerks/BFD.py @@ -9,14 +9,13 @@ logger = logging.getLogger(__name__) class Clerk(isClerk): - def __init__(self, vault, root, **kwargs): - isClerk.__init__(self) + def __init__(self, vault, root = None, **kwargs): + isClerk.__init__(self, vault) - assert isinstance(vault, isVault), "vault must be an instance of Bastion.Model.isVault" - assert isinstance(root, pathlib.PosixPath), "root must be an instance of PosixPath" + if root: + assert isinstance(root, pathlib.PosixPath), "root must be an instance of PosixPath" - self.vault = vault - self.root = root + self.root = root if root is not None else vault.bank #----------------------------------------- #-- BEGIN Bastion.Model.isClerk PROTOCOL | diff --git a/lib/Bastion/Common.py b/lib/Bastion/Common.py index caabbed..295e64d 100644 --- a/lib/Bastion/Common.py +++ b/lib/Bastion/Common.py @@ -86,7 +86,7 @@ def __call__(self, subject, **kwargs): #-- If we get to here, then the subject doesn't know how to map itself, #-- we'll go through our internal list of spells and see if anything matches. - for indicator, transmute in self.spells: + for indicator, transmute in self.magic: if isinstance(indicator, type): recognize = lambda x: isinstance(x, indicator) else: @@ -281,17 +281,32 @@ def Slug40(text): return base64.b32encode(bs).decode('utf-8') -class canTextify: +class canConStruct: """ - I am a trait for serialization using JSON and YAML. + I am a trait for constructing instances of self from a given JSON normalized dictionary (JDN) """ - def toJDN(self, **kwargs): - raise NotImplementedError(".toJDN is subclass responsibility") - @classmethod def fromJDN(cls, jdn, **kwargs): raise NotImplementedError(".fromJDN is subclass responsibility") + @classmethod + def fromJSON(cls, js, **kwargs): + jdn = json.loads(js) + return cls.fromJDN(jdn, **kwargs) + + @classmethod + def fromYAML(cls, ydoc, **kwargs): + jdn = yaml.safe_load(ydoc) + return cls.fromJDN(jdn, **kwargs) + + +class canStruct: + """ + I am a trait for creating a data structure of self expressed in JSON dictionary normal (JDN) form. + """ + def toJDN(self, **kwargs): + raise NotImplementedError(".toJDN is subclass responsibility") + #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ #-- host class protocol. | #-- host classes must implement the methods, above. | @@ -305,16 +320,6 @@ def toYAML(self, **kwargs): jdn = self.toJDN(**kwargs) return yaml.dump(jdn, default_flow_style = False, indent = 3) - @classmethod - def fromJSON(cls, js, **kwargs): - jdn = json.loads(js) - return cls.fromJDN(jdn, **kwargs) - - @classmethod - def fromYAML(cls, ydoc, **kwargs): - jdn = yaml.safe_load(ydoc) - return cls.fromJDN(jdn, **kwargs) - #-- Used as pre-defined symbol sets for the Boggle function, below. BOGGLES = { diff --git a/lib/Bastion/Condo.py b/lib/Bastion/Condo.py index 9ac6fa0..ae4f526 100644 --- a/lib/Bastion/Condo.py +++ b/lib/Bastion/Condo.py @@ -208,7 +208,7 @@ def copy(self): Answers a new nested dictionary by shallow (objects are referenced, not copied) copy. """ clone = CxNode() - for k, v in self.flattened: + for k, v in self.flattened.items(): clone[k] = v return clone diff --git a/lib/Bastion/Curator.py b/lib/Bastion/Curator.py index 507aef7..92b84e1 100644 --- a/lib/Bastion/Curator.py +++ b/lib/Bastion/Curator.py @@ -5,12 +5,12 @@ """ import datetime -from Bastion.Common import Thing, Boggle, canTextify +from Bastion.Common import Thing, Boggle, canStruct from Bastion.Model import ARK, isAsset from Bastion.Chronology import Quantim -class BLONDE(canTextify): +class BLONDE: """ BLOb Name and Description Encoding I am a structured name describing a point in time for a single ARK. @@ -138,7 +138,7 @@ def drift(self): #-- {quantim} is the 8 character encoding of timestamp using the Quantim method #-- F if the blob is an anchor (full backup) and D if the blob is a differential. #-- {anchor} - is a 3 character random string that cannot conflict with any other anchors currently in the archive. -class Manifest(canTextify): +class Manifest(canStruct): """ I represent the chronicled archive of some dataset. I hold a series of "snaps". diff --git a/lib/Bastion/Model.py b/lib/Bastion/Model.py index fc08c50..e2f301c 100644 --- a/lib/Bastion/Model.py +++ b/lib/Bastion/Model.py @@ -98,7 +98,7 @@ def perform(self, request): raise NotImplementedError -class isVault: +class isVault(canConfigure): """ I am the base class for all storage vaults. Users of Vaults primarily interact with the vault through the .push and .pull methods. @@ -211,7 +211,7 @@ def pack(self, asset, basis = None, **kwargs): On success, I answer a report that wraps an instance of Packers.CARP.Packed On failure, I answer a Report.Failed """ - self.packer.pack(asset, basis, **kwargs) + return self.packer.pack(asset, basis, **kwargs) def unpack(self, halo, root, **kwargs): """ @@ -266,7 +266,8 @@ def for_protocol(protocol): def handling(protocol): return isVault.PROTOCOLS[protocol] - +#-- nickname +Vault = isVault class isAsset: """ diff --git a/lib/Bastion/Movers/BFD.py b/lib/Bastion/Movers/BFD.py index a3bdafe..d92bc2a 100644 --- a/lib/Bastion/Movers/BFD.py +++ b/lib/Bastion/Movers/BFD.py @@ -10,13 +10,11 @@ class Mover(isMover): - def __init__(self, vault, **kwargs): - isMover.__init__(self) - self.vault = vault - self.root = kwargs.get('root', vault.root) - - assert isinstance(self.vault, isVault), "vault must be an instance of Bastion.Model.isVault" - assert isinstance(self.root, pathlib.Path), "root must be an instance of Path" + def __init__(self, vault, bank = None, **kwargs): + isMover.__init__(self, vault) + if bank: + assert isinstance(bank, pathlib.Path), "bank must be an instance of Path" + self.bank = bank if bank is not None else vault.bank #----------------------------------------- #-- BEGIN Bastion.Model.isMover PROTOCOL | @@ -34,16 +32,16 @@ def provision(self, *args): else: raise ValueError - repo = self.root / ark.site / ark.zone / ark.asset + repo = self.bank / ark.site / ark.zone / ark.asset return repo.mkdir(parents = True, exist_ok = True) def put(self, halo, tag, **kwargs): here = halo - there = self.root / tag + there = self.bank / tag logger.debug( "put source {} to {}".format(here.as_uri(), there.as_uri()) ) #-- Create the put request. - request = PutRequest(halo, self.root.as_uri(), tag) + request = PutRequest(halo, self.bank.as_uri(), tag) try: shutil.copy(here, there) @@ -51,7 +49,7 @@ def put(self, halo, tag, **kwargs): #-- shutil should throw an error if there was a problem copying. report = request.failed(err) else: - report = request.succeeded( PutReceipt(halo, self.root.as_uri(), tag) ) + report = request.succeeded( PutReceipt(halo, self.bank.as_uri(), tag) ) return report diff --git a/lib/Bastion/Movers/CARP.py b/lib/Bastion/Movers/CARP.py index af0e0ea..63c30c7 100644 --- a/lib/Bastion/Movers/CARP.py +++ b/lib/Bastion/Movers/CARP.py @@ -3,13 +3,13 @@ Common Action-Result Protocol """ - +import pathlib import logging -from Bastion.Common import Thing, Unknown -from Bastion.Model import isRequest, isAsset, isResult +from Bastion.Common import Thing, Unknown, toJDN +from Bastion.Model import isAsset from Bastion.Curator import Manifest, BLONDE, Snap -from Bastion.CARP import isReceipt, isRequest +from Bastion.CARP import isRequest, isReceipt logger = logging.getLogger(__name__) @@ -72,8 +72,8 @@ def __init__(self, halo, rendpoint, tag): self.bank = Thing(endpoint = rendpoint, tag = tag) xtras = { - "local.path": str(self.local) - "bank.endpoint": toJDN(self.bank.endpoint) + "local.path": str(self.local), + "bank.endpoint": toJDN(self.bank.endpoint), "bank.tag": toJDN(self.bank.tag) } @@ -97,8 +97,8 @@ def __init__(self, rendpoint, tag, halo): self.bank = Thing(endpoint = rendpoint, tag = tag) xtras = { - "local.path": str(self.local) - "bank.endpoint": toJDN(self.bank.endpoint) + "local.path": str(self.local), + "bank.endpoint": toJDN(self.bank.endpoint), "bank.tag": toJDN(self.bank.tag) } diff --git a/lib/Bastion/Packers/CARP.py b/lib/Bastion/Packers/CARP.py index 8ce1d62..1f8c180 100644 --- a/lib/Bastion/Packers/CARP.py +++ b/lib/Bastion/Packers/CARP.py @@ -20,11 +20,11 @@ def __init__(self, asset, tag, packaged, tarp): assert isinstance(tag, pathlib.PurePosixPath), "tag must be pathlib.PurePosixPath" assert isinstance(tarp, pathlib.PosixPath), "tarp must be pathlib.PosixPath" - self.asset = asset #-- the asset that was packed - self.tag = tag #-- the full name of the packed object - self.packaged = packaged #-- an inventory of what was packed - self.tarp = tarp #-- the path to the local package (TARfile Path → "tarp") - self.blonde = BLONDE.decode(tag.name.stem) #-- the BLONDE for the packed object + self.asset = asset #-- the asset that was packed + self.tag = tag #-- the full name of the packed object + self.packaged = packaged #-- an inventory of what was packed + self.tarp = tarp #-- the path to the local package (TARfile Path → "tarp") + self.blonde = BLONDE.decode(tag.stem) #-- the BLONDE for the packed object def toJDN(self): jdn = { @@ -50,7 +50,7 @@ def __init__(self, subject, *args, **kwargs): if isinstance(subject, isRequest): isRequest.__init__("PackRequest", ID = subject.ID, opened = subject.when, context = subject.context) - else: + elif isinstance(subject, isAsset): asset = subject basis = args[0] if args else None whence = None @@ -81,7 +81,9 @@ def __init__(self, subject, *args, **kwargs): 'halo': str(asset.halo) } - isRequest.__init__("PackRequest", asset, **xtras) + isRequest.__init__(self, "PackRequest", asset, **xtras) + else: + raise ValueError("subject for PackRequest must be an instance of isAsset") @property def asset(self): diff --git a/lib/Bastion/Site.py b/lib/Bastion/Site.py index e195b64..cb57a57 100644 --- a/lib/Bastion/Site.py +++ b/lib/Bastion/Site.py @@ -6,7 +6,7 @@ import random import datetime -from Bastion.Common import entity, Thing, canTextify, DAYS, RDN, asPath, asLogLevel +from Bastion.Common import entity, Thing, canInStruct, DAYS, RDN, asPath, asLogLevel from Bastion.Condo import CxNode from Bastion.Model import ARK, isAsset, isZone, isSite #from .Curator import Asset @@ -23,20 +23,8 @@ DEFAULT_LOGGING_PATH = pathlib.Path("~/.bastion/log").expanduser() -class canConfigure: - """ - Trait for objects that can configure themselves from a nested dictionary. - """ - @classmethod - def fromConf(cls, condex): - raise NotImplementedError(".fromConf is subclass responsibility") - - def configured(condex): - raise NotImplementedError(".configured is subclass responsibility") - - -class Site(isSite, canConfigure, canTextify): +class Site(isSite, canConfigure): SITES = { } @staticmethod @@ -164,7 +152,7 @@ def asset(self, ark): # return None -class RetentionPolicy(canConfigure, canTextify): +class RetentionPolicy(canConfigure, canInStruct, canConStruct): """ How long to store an object, how many copies, and where the copies are deposited. """ diff --git a/lib/Bastion/Vaults/BFD.py b/lib/Bastion/Vaults/BFD.py index 7544e70..08f22a6 100644 --- a/lib/Bastion/Vaults/BFD.py +++ b/lib/Bastion/Vaults/BFD.py @@ -47,7 +47,7 @@ def scratch(self): @property def clerk(self): if getattr(self, '_clerk', None) is None: - self._clerk = Bastion.Clerks.BFD.Clerk(self, self.bank) + self._clerk = Bastion.Clerks.BFD.Clerk(self) return self._clerk @property @@ -60,6 +60,7 @@ def packer(self): def mover(self): if getattr(self, '_mover', None) is None: self._mover = Bastion.Movers.BFD.Mover(self) + return self._mover #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ #-- END Bastion.Model.Vault PROTOCOL | diff --git a/lib/Bastion/Vaults/CARP.py b/lib/Bastion/Vaults/CARP.py index f6dad62..6e42f89 100644 --- a/lib/Bastion/Vaults/CARP.py +++ b/lib/Bastion/Vaults/CARP.py @@ -55,7 +55,7 @@ def __init__(self, subject, *args, **kwargs): 'halo': str(asset.halo) } - isRequest.__init__("PackRequest", asset, **xtras) + isRequest.__init__(self, "PackRequest", asset, **xtras) @property def asset(self): diff --git a/lib/Bastion/Vaults/Common.py b/lib/Bastion/Vaults/Common.py index 3048917..28aa9fc 100644 --- a/lib/Bastion/Vaults/Common.py +++ b/lib/Bastion/Vaults/Common.py @@ -11,7 +11,7 @@ logger = logging.getLogger(__name__) -class isSpoolingVault(Bastion.Model.Vault): +class isSpoolingVault(Bastion.Model.isVault): """ I am a vault that uses a local scratch (spool) volume as a buffer to pack archives before transferring them to final storage location. @@ -34,13 +34,13 @@ def push(self, asset, basis = None, **kwargs): if packing.failed: return request.failed( PushReceipt([packing]) ) - movement = self.put(packing.tarp, packing.tag) + movement = self.put(packing.record.tarp, packing.record.tag) if movement.failed: return request.failed( PushReceipt([packing, movement]) ) #-- clean up! - if packing.tarp.exists(): - packing.tarp.unlink() + if packing.record.tarp.exists(): + packing.record.tarp.unlink() return request.succeeded( PushReceipt([packing, movement]) )