From b2a134f00d51d19d17d012167df999c5234fa88f Mon Sep 17 00:00:00 2001 From: Nathan Denny Date: Tue, 7 Jan 2025 14:07:05 -0500 Subject: [PATCH] debugging mostly; some refactoring, too; work in progress. --- bin/bastion.py | 21 ++---- lib/Bastion/Chronology.py | 1 + lib/Bastion/Clerks/BFD.py | 4 +- lib/Bastion/Common.py | 38 ++++------ lib/Bastion/Curator.py | 4 +- lib/Bastion/Humane.py | 2 +- lib/Bastion/Model.py | 5 +- lib/Bastion/Movers/BFD.py | 12 ++-- lib/Bastion/Movers/CARP.py | 6 +- lib/Bastion/Packers/CARP.py | 14 ++-- lib/Bastion/Site.py | 10 +-- lib/Bastion/Vaults/HPSS.py | 48 ++++++------- lib/Bastion/Vaults/SFTP.py | 139 ++---------------------------------- 13 files changed, 83 insertions(+), 221 deletions(-) diff --git a/bin/bastion.py b/bin/bastion.py index 8ffb46d..836f81e 100755 --- a/bin/bastion.py +++ b/bin/bastion.py @@ -4,7 +4,6 @@ import os import pathlib import logging -import traceback import socket import json @@ -23,14 +22,13 @@ sys.path.insert(0, str(LIB_PATH)) -from Bastion.Common import * +from Bastion.Common import Boggle, asPath from Bastion.Chronology import Quantim from Bastion.Site import Site -from Bastion.Condo import * -from Bastion.Actions import * +from Bastion.Condo import Condex from Bastion.Model import ARK -from Bastion.CARP import isRequest, isReceipt -#import Bastion.Vaults.HPSS +from Bastion.CARP import Request, isReceipt +import Bastion.Vaults.HPSS import Bastion.Vaults.BFD """ @@ -248,7 +246,7 @@ def emit(self, answer, ostream = None): ostream = ostream if (ostream is not None) else sys.stdout if 'emit' in answer.request.context: - form = opts['emit'].upper() + form = answer.request['emit'].upper() else: form = self.tongue @@ -603,18 +601,11 @@ def do_refresh_keytab(self, comargs, comdex, opts): if sys.argv[1:] != ['shell']: app.run( ) else: - vehicle = os.environ.get('BASTION_SITE', None) - if vehicle == 'rusina': + if os.environ.get('BASTION_SITE', None) == 'rusina': rusina = app.site('rusina') soundscapes = rusina.zone('soundscapes') asset = soundscapes['HackathonData'] vault = app.vault(asset.policy.vault) - elif vehicle == 'scout': - scout = app.site('scout') - ADS = scout.zone('ADS') - asset = ADS['FSIL'] - vault = app.vault(asset.policy.vault) - #-- does a full, level 0 backup #bastion backup site {site} diff --git a/lib/Bastion/Chronology.py b/lib/Bastion/Chronology.py index e632ea3..37c455c 100644 --- a/lib/Bastion/Chronology.py +++ b/lib/Bastion/Chronology.py @@ -175,3 +175,4 @@ def latest(self): @classmethod def now(cls): return cls(datetime.datetime.now()) + diff --git a/lib/Bastion/Clerks/BFD.py b/lib/Bastion/Clerks/BFD.py index 0a0deec..6f00cce 100644 --- a/lib/Bastion/Clerks/BFD.py +++ b/lib/Bastion/Clerks/BFD.py @@ -10,11 +10,13 @@ class Clerk(isClerk): def __init__(self, vault, root = None, **kwargs): - isClerk.__init__(self, vault) + isClerk.__init__(self) + assert isinstance(vault, isVault), "vault must be an instance of Bastion.Model.isVault" if root: assert isinstance(root, pathlib.PosixPath), "root must be an instance of PosixPath" + self.vault = vault self.root = root if root is not None else vault.bank #----------------------------------------- diff --git a/lib/Bastion/Common.py b/lib/Bastion/Common.py index 295e64d..1a64030 100644 --- a/lib/Bastion/Common.py +++ b/lib/Bastion/Common.py @@ -281,10 +281,23 @@ def Slug40(text): return base64.b32encode(bs).decode('utf-8') -class canConStruct: +class canStruct: """ - I am a trait for constructing instances of self from a given JSON normalized dictionary (JDN) + I am a trait for serialization using JSON and YAML. """ + def toJDN(self, **kwargs): + raise NotImplementedError(".toJDN is subclass responsibility") + + def toJSON(self, **kwargs): + jdn = self.toJDN(**kwargs) + return json.dumps(jdn, indent = 3, sort_keys = True) + + def toYAML(self, **kwargs): + jdn = self.toJDN(**kwargs) + return yaml.dump(jdn, default_flow_style = False, indent = 3) + + +class canConStruct: @classmethod def fromJDN(cls, jdn, **kwargs): raise NotImplementedError(".fromJDN is subclass responsibility") @@ -300,27 +313,6 @@ def fromYAML(cls, ydoc, **kwargs): return cls.fromJDN(jdn, **kwargs) -class canStruct: - """ - I am a trait for creating a data structure of self expressed in JSON dictionary normal (JDN) form. - """ - def toJDN(self, **kwargs): - raise NotImplementedError(".toJDN is subclass responsibility") - - #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ - #-- host class protocol. | - #-- host classes must implement the methods, above. | - #----------------------------------------------------- - - def toJSON(self, **kwargs): - jdn = self.toJDN(**kwargs) - return json.dumps(jdn, indent = 3, sort_keys = True) - - def toYAML(self, **kwargs): - jdn = self.toJDN(**kwargs) - return yaml.dump(jdn, default_flow_style = False, indent = 3) - - #-- Used as pre-defined symbol sets for the Boggle function, below. BOGGLES = { "Azed9": string.ascii_uppercase + string.ascii_lowercase + string.digits, diff --git a/lib/Bastion/Curator.py b/lib/Bastion/Curator.py index 92b84e1..dbdd2eb 100644 --- a/lib/Bastion/Curator.py +++ b/lib/Bastion/Curator.py @@ -5,12 +5,12 @@ """ import datetime -from Bastion.Common import Thing, Boggle, canStruct +from Bastion.Common import Thing, Boggle, canTextify from Bastion.Model import ARK, isAsset from Bastion.Chronology import Quantim -class BLONDE: +class BLONDE(canTextify): """ BLOb Name and Description Encoding I am a structured name describing a point in time for a single ARK. diff --git a/lib/Bastion/Humane.py b/lib/Bastion/Humane.py index 0ec444b..37410d3 100644 --- a/lib/Bastion/Humane.py +++ b/lib/Bastion/Humane.py @@ -27,4 +27,4 @@ def _humanize_timedelta(elapsed, **kwargs): return "{:.1f} seconds".format(dt) -humanize.spells.append( (datetime.timedelta, _humanize_timedelta) ) +humanize.magic.append( (datetime.timedelta, _humanize_timedelta) ) diff --git a/lib/Bastion/Model.py b/lib/Bastion/Model.py index e2f301c..a28f013 100644 --- a/lib/Bastion/Model.py +++ b/lib/Bastion/Model.py @@ -98,7 +98,7 @@ def perform(self, request): raise NotImplementedError -class isVault(canConfigure): +class isVault: """ I am the base class for all storage vaults. Users of Vaults primarily interact with the vault through the .push and .pull methods. @@ -266,9 +266,10 @@ def for_protocol(protocol): def handling(protocol): return isVault.PROTOCOLS[protocol] -#-- nickname +#-- legacy name preserved as an alt / nickname Vault = isVault + class isAsset: """ abstract Asset type, describes a local file in both the host file system and the logical (zone) space. diff --git a/lib/Bastion/Movers/BFD.py b/lib/Bastion/Movers/BFD.py index d92bc2a..f946841 100644 --- a/lib/Bastion/Movers/BFD.py +++ b/lib/Bastion/Movers/BFD.py @@ -10,11 +10,13 @@ class Mover(isMover): - def __init__(self, vault, bank = None, **kwargs): - isMover.__init__(self, vault) - if bank: - assert isinstance(bank, pathlib.Path), "bank must be an instance of Path" - self.bank = bank if bank is not None else vault.bank + def __init__(self, vault, **kwargs): + isMover.__init__(self) + self.vault = vault + self.bank = kwargs.get('bank', vault.bank) + + assert isinstance(self.vault, isVault), "vault must be an instance of Bastion.Model.isVault" + assert isinstance(self.bank, pathlib.PurePosixPath), "bank must be an instance of Path" #----------------------------------------- #-- BEGIN Bastion.Model.isMover PROTOCOL | diff --git a/lib/Bastion/Movers/CARP.py b/lib/Bastion/Movers/CARP.py index 63c30c7..94bbdee 100644 --- a/lib/Bastion/Movers/CARP.py +++ b/lib/Bastion/Movers/CARP.py @@ -6,10 +6,8 @@ import pathlib import logging -from Bastion.Common import Thing, Unknown, toJDN -from Bastion.Model import isAsset -from Bastion.Curator import Manifest, BLONDE, Snap -from Bastion.CARP import isRequest, isReceipt +from Bastion.Common import Thing, toJDN +from Bastion.CARP import isReceipt, isRequest logger = logging.getLogger(__name__) diff --git a/lib/Bastion/Packers/CARP.py b/lib/Bastion/Packers/CARP.py index 1f8c180..9133879 100644 --- a/lib/Bastion/Packers/CARP.py +++ b/lib/Bastion/Packers/CARP.py @@ -3,14 +3,14 @@ import logging from Bastion.Model import isAsset -from Bastion.CARP import isRequest +from Bastion.CARP import isRequest, isReceipt from Bastion.Curator import BLONDE logger = logging.getLogger(__name__) -class PackingReceipt: +class PackingReceipt(isReceipt): def __init__(self, asset, tag, packaged, tarp): #-- asset:Bastion.Model.isAsset #-- tag:pathlib.PurePosixPath @@ -20,10 +20,10 @@ def __init__(self, asset, tag, packaged, tarp): assert isinstance(tag, pathlib.PurePosixPath), "tag must be pathlib.PurePosixPath" assert isinstance(tarp, pathlib.PosixPath), "tarp must be pathlib.PosixPath" - self.asset = asset #-- the asset that was packed - self.tag = tag #-- the full name of the packed object - self.packaged = packaged #-- an inventory of what was packed - self.tarp = tarp #-- the path to the local package (TARfile Path → "tarp") + self.asset = asset #-- the asset that was packed + self.tag = tag #-- the full name of the packed object + self.packaged = packaged #-- an inventory of what was packed + self.tarp = tarp #-- the path to the local package (TARfile Path → "tarp") self.blonde = BLONDE.decode(tag.stem) #-- the BLONDE for the packed object def toJDN(self): @@ -48,7 +48,7 @@ class PackRequest(isRequest): """ def __init__(self, subject, *args, **kwargs): if isinstance(subject, isRequest): - isRequest.__init__("PackRequest", ID = subject.ID, opened = subject.when, context = subject.context) + isRequest.__init__(self, "PackRequest", ID = subject.ID, opened = subject.when, context = subject.context) elif isinstance(subject, isAsset): asset = subject diff --git a/lib/Bastion/Site.py b/lib/Bastion/Site.py index cb57a57..94b5b2a 100644 --- a/lib/Bastion/Site.py +++ b/lib/Bastion/Site.py @@ -6,7 +6,7 @@ import random import datetime -from Bastion.Common import entity, Thing, canInStruct, DAYS, RDN, asPath, asLogLevel +from Bastion.Common import entity, Thing, canStruct, DAYS, RDN, asPath, asLogLevel from Bastion.Condo import CxNode from Bastion.Model import ARK, isAsset, isZone, isSite #from .Curator import Asset @@ -24,7 +24,7 @@ -class Site(isSite, canConfigure): +class Site(isSite, canStruct): SITES = { } @staticmethod @@ -152,7 +152,7 @@ def asset(self, ark): # return None -class RetentionPolicy(canConfigure, canInStruct, canConStruct): +class RetentionPolicy(canStruct, canConStruct): """ How long to store an object, how many copies, and where the copies are deposited. """ @@ -205,7 +205,7 @@ def fromJDN(cls, jdn): -class Asset(isAsset, canTextify): +class Asset(isAsset, canStruct): def __init__(self, zone, *args, **kwargs): """ Asset(zone:Zone, name:str) @@ -372,7 +372,7 @@ def __iter__(self): return iter(sorted(self.xRDNs.values(), key = lambda x: str(x.halo))) -class Zone(isZone, canConfigure): +class Zone(isZone): """ I am a (resource) zone. A zone is a logical entry point to a collection of assets. diff --git a/lib/Bastion/Vaults/HPSS.py b/lib/Bastion/Vaults/HPSS.py index 5f18643..29ae992 100644 --- a/lib/Bastion/Vaults/HPSS.py +++ b/lib/Bastion/Vaults/HPSS.py @@ -8,9 +8,9 @@ import logging import getpass -from Bastion.Common import Thing, Unknown -import Bastion.Model -from Bastion.Curator import BLONDE +from Bastion.Common import Thing +from Bastion.Model import ARK, isVault +from Bastion.Curator import Manifest, BLONDE logger = logging.getLogger(__name__) @@ -205,13 +205,13 @@ def __getitem__(self, path): note = aline[11:].strip() return note - def __setitem__(self, path, note): - request = 'annotate -A "{}" {}'.format(note, str(path)) - proc = self.do(request) + #def __setitem__(self, path, note): + # request = 'annotate -A "{}" {}'.format(note, str(path)) + #proc = self.do(request) - def __delitem__(self, path): - request = 'annotate -e {}'.format(str(path)) - proc = self.do(request) + #def __delitem__(self, path): + # request = 'annotate -e {}'.format(str(path)) + #proc = self.do(request) class HSI: @@ -231,10 +231,10 @@ def do(self, command): self.procd = subprocess.run(comargs, capture_output = True, check = True) return self.procd - def mkdirs(self, path): - request = "mkdir -p {}".format( str(path) ) - procd = self.do(request) - return True + #def mkdirs(self, path): + #request = "mkdir -p {}".format( str(path) ) + #procd = self.do(request) + #return True def statx(self, target): """ @@ -251,14 +251,14 @@ def statx(self, target): lines = [line.strip() for line in proc.stdout.decode('utf-8').split('\n')] line = lines[0] tokens = [token.strip() for token in line.split()] - size = 0 + #size = 0 obj = target.toJDN() obj['xtype'] = tokens[0] if obj['xtype'] in ('FILE', 'HARDLINK'): - size = int(tokens[2]) + #size = int(tokens[2]) created_mdy = tokens[9] created_time = tokens[10] @@ -320,11 +320,11 @@ def lsx(self, path = None): -class Vault(Bastion.Model.Vault): +class Vault(isVault): PROTOCOL = 'HTAR' def __init__(self, name, **kwargs): - Bastion.Model.Vault.__init__(self, name, **kwargs) + isVault.__init__(self, name, **kwargs) self.server = kwargs.get('server', socket.gethostname()) self.login = kwargs.get('login', getpass.getuser()) @@ -462,12 +462,12 @@ def push(self, asset, **kwargs): 'stolo': "" } - exportBastion.Model.Vaults = { - "HPSS_AUTH_METHOD": "keytab", - "HPSS_PRINCIPAL": self.login, - "HPSS_KEYTAB_PATH": str(self.keytab.halo), - "HPSS_HOSTNAME": str(self.client) - } +# exports = { +# "HPSS_AUTH_METHOD": "keytab", +# "HPSS_PRINCIPAL": self.login, +# "HPSS_KEYTAB_PATH": str(self.keytab.halo), +# "HPSS_HOSTNAME": str(self.client) +# } #-- save the current working directory. ogdir = os.getcwd( ) @@ -505,7 +505,7 @@ def _manifest_ark(self, ark): #-- The manifest a catalog of all of the backup objects for the asset. fls = self.hsi.ls(self.root / ark.site / ark.zone / ark.asset) blondes = [fl.path.name for fl in fls] - manifest = Bastion.Curator.Manifest(ark, blondes) + manifest = Manifest(ark, blondes) return manifest diff --git a/lib/Bastion/Vaults/SFTP.py b/lib/Bastion/Vaults/SFTP.py index a513045..706728c 100644 --- a/lib/Bastion/Vaults/SFTP.py +++ b/lib/Bastion/Vaults/SFTP.py @@ -1,19 +1,9 @@ -import os import pathlib -import subprocess -import operator -import datetime -import json -import socket import logging import getpass -import shutil -import tarfile -import logging -from Bastion.Common import Thing, Unknown, asPath, asPurePath +from Bastion.Common import asPath, asPurePath import Bastion.Model -from Bastion.Curator import Manifest, BLONDE, Snap import Bastion.Movers.SFTP import Bastion.Clerks.SFTP @@ -49,139 +39,24 @@ def configured(self, conf): self.scratch = section.get(asPath, "scratch.path", "/tmp") #-- Configuration relevant to remote (bank) host. - self.host = section.get('remote.host') - self.login = section.get('remote.login', getpass.getuser()) - self.keypath = section.get(asPath, 'remote.key', pathlib.Path("~/.ssh/id").expanduser()) - self.root = section.get(asPurePath, 'remote.root', "/") + self.host = section.get('bank.host') + self.login = section.get('bank.login', getpass.getuser()) + self.keypath = section.get(asPath, 'bank.key', pathlib.Path("~/.ssh/id").expanduser()) + self.root = section.get(asPurePath, 'bank.root', "/") return self @property def mover(self): if getattr(self, '_mover', None) is None: - self._mover = Bastion.Movers.SFTP.Mover(self, self.host, self.login, self.keypath, root = self.root) + self._mover = Bastion.Movers.SFTP.Mover(self) return self._mover @property def clerk(self): if getattr(self, '_clerk', None) is None: - self._clerk = Clerk(self, target, self.host, self.login, self.keypath) + self._clerk = Bastion.Clerks.SFTP.Clerk(self) return self._clerk - @property - def bank(self): - client = SCURLer(self.login, self.host, self.root, keyfile = self.key) - -#--------------------------------------- -#-- BEGIN Bastion.Model.Vault PROTOCOL | -#↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ - def pack(self, asset, basis = None, **kwargs): - """ - Given a local asset, I package (.tar, .zip, etc) the asset into my scratch (spool) space. - Without a given basis, I package everything (i.e. a full backup). - When a basis is given as either a datetime or an anchor (BLONDE), I do a differential backup. - I answer an instance of Bastion.Model.PackingReceipt - """ - detail = 'F' - whence = None - basis = None - blonded = None - tarp = None - ark = asset.ARK - opts = { } - - if basis: - detail = 'D' - if isinstance(basis, BLONDE): - #-- I was given a BLONDE (a reference to a full backup) - anchor = basis - whence = anchor.when.earliest - genus = anchor.genus - blonded = BLONDE.forDiffBackup(anchor) - - if isinstance(basis, datetime.datetime): - whence = basis - genus = "___" - blonded = BLONDE(asset.ARK, detail, genus) - - opts['since'] = whence - - else: - blonded = BLONDE.forFullBackup(asset.ARK) - - package = "{}.tar".format(str(blonded)) - tag = pathlib.PurePosixPath(ark.site) / ark.zone / ark.asset / package - spooled = (self.scratch / tag) - spool = spooled.parent - - #-- assure that the scratch path exists and all of the subpaths that create the repo folder for this asset. - spool.mkdir(parents = True, exist_ok = True) - - #-- use the built-in python tar archiver, using "pax" (POSIX.1-2001) extensions. - pax((self.scratch / tag), asset.halo, **opts) - - #-- start a receipt for packing the asset. - receipt = Bastion.Model.PackingReceipt(asset, blonded, spooled) - - #-- some optional information that might be of use... - receipt['size'] = spooled.stat().st_size - receipt['tag'] = str(tag) - - #-- Answer the receipt. - return receipt - - - def put(self, obj, *args, **kwargs): - """ - performs a transfer of the file at host area local (halo) path to the tag at the destination. - put(receipt:PackingReceipt) - put(halo:PosixPath, tag:PurePosixPath) - """ - if isinstance(obj, PackingReceipt): - here = obj.tarp - there = obj.tag - elif isinstance(obj, pathlib.PosixPath): - here = obj - there = args[0] - assert isinstance(there, pathlib.PurePosixPath), "tag must be an instance of PurePosixPath" - else: - raise ValueError("obj must be an instance of either PackingReceipt or PosixPath") - - return self.mover.put(here, there) - - -#↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ -#-- END Bastion.Model.Vault PROTOCOL | -#------------------------------------- - def _copy_blonde(self, blonde, **kwargs): - raise NotImplementedError - - def _empty_scratch(self, ark, **kwargs): - raise NotImplementedError - - def _manifest_ark(self, ark): - #-- The contents of {root}/{site}/{zone}/{asset} are backup blobs - #-- Each name in this folder is a "BLOND" (BLOb Name and Descriptor) - #-- The manifest a catalog of all of the backup objects for the asset. - cell = self.bank / ark.site / ark.zone / ark.asset - if cell.exists(): - blondes = [ ] - for item in cell.iterdir(): - if not item.is_dir(): - blondes.append( BLONDE.decode(item.stem) ) - manifest = Bastion.Curator.Manifest(ark, blondes) - - return manifest - - def _manifest_site_zone_asset(self, site, zone, asset): - return self._manifest_ark( ARK(site, zone, asset) ) - - def _provision_ark(self, ark): - repo = self.bank / ark.site / ark.zone / ark.asset - return repo.mkdir(parents = True, exist_ok = True) - - def _provision_site_zone_asset(self, site, zone, asset_name): - return self._provision_ark( ARK(site, zone, asset_name) ) - Vault.register()