diff --git a/bin/bastion.py b/bin/bastion.py index a50d5e8..6d4c4c6 100755 --- a/bin/bastion.py +++ b/bin/bastion.py @@ -29,7 +29,8 @@ from Bastion.Condo import * from Bastion.Actions import * from Bastion.Model import ARK -import Bastion.HPSS +import Bastion.Vaults.HPSS +import Bastion.Vaults.BFD """ zone backup procedure... @@ -137,6 +138,9 @@ def logroot(self): def site(self, name): return Site(name).configured(self.conf) + def asset(self, ark): + return self.site(ark.site).asset(ark) + def vault(self, name, site = None): """ I answer an instance of a storage vault given a name and optional site. @@ -220,6 +224,8 @@ def run(self): except Exception as err: tb = traceback.format_exception(err) answer = CRASHED( ''.join(tb), tb ) + #-- always log crashes! + answer['context']['log.scope'] = '*' proc['task.ended'] = datetime.datetime.now().isoformat() @@ -260,7 +266,12 @@ def record(self, answer, opts): if 'log.scope' in answer['context']: #-- only write a log file if we have an explicit log.scope in the answer's context block. session = answer['context']['task.session'] - halo = self.logroot / answer['context']['log.scope'] / "{}.yaml".format(session) + + scope = self.logroot + if answer['context']['log.scope'] != '*': + scope = scope / answer['context']['log.scope'] + + halo = scope / "{}.yaml".format(session) halo.parent.mkdir(parents = True, exist_ok = True) with open(halo, 'wt') as fout: self.emit_YAML(answer, opts, fout) @@ -354,11 +365,16 @@ def do_bank_asset(self, comargs, comdex, opts): site = self.site(ark.site) asset = site.asset(ark) vault = self.vault(asset.policy.vault) - flag, stdout, stderr = vault.push(asset, client = self.hostname) - if flag: - return SUCCESS(stdout, {'stdout': stdout, 'stderr': stderr}) + blonde = vault.push(asset, client = self.hostname) + + extras = { + 'log.scope': "site/{}".format(ark.site) + } + + if blonde: + return SUCCESS("pushed full backup {}".format(str(blonde)), context = extras) else: - return FAILED(stdout, {'stdout': stdout, 'stderr': stderr}) + return FAILED("something went wrong!", context = extras) #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ #-- END bank (backup) operations | @@ -402,14 +418,14 @@ def do_update_site(self, comargs, comdex, opts): """ raise NotImplementedError - def do_update_zone(self, comargs, comdex): + def do_update_zone(self, comargs, comdex, opts): """ update zone {zone} * creates a differential backup of each asset in {zone} """ raise NotImplementedError - def do_update_asset(self, comargs, comdex): + def do_update_asset(self, comargs, comdex, opts): """ update asset {ARK} * asset given in ARK format @@ -417,7 +433,22 @@ def do_update_asset(self, comargs, comdex): * performs backup based on determined level * typically used to "automatically" do updates in a scheduled (cron) job """ - raise NotImplementedError + ark = ARK(comdex[2]) + asset = self.asset(ark) + vault = self.vault(asset.policy.vault) + banked = vault.manifest(ark) + if banked.anchors: + basis = banked.anchors[-1] + blonde = vault.push(asset, basis) + else: + blonde = vault.push(asset) + + extras = { + 'log.scope': "site/{}".format(ark.site) + } + + return SUCCESS("pushed update {}".format(str(blonde)), context = extras) + #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ #-- END update (automatic) operations | @@ -564,6 +595,11 @@ def do_refresh_keytab(self, comargs, comdex, opts): app = App().configured() if sys.argv[1:] != ['shell']: app.run( ) + else: + rusina = app.site('rusina') + soundscapes = rusina.zone('soundscapes') + asset = soundscapes['HackathonData'] + vault = app.vault(asset.policy.vault) #-- does a full, level 0 backup #bastion backup site {site} diff --git a/lib/Bastion/Chronology.py b/lib/Bastion/Chronology.py index d8cea91..c4d8522 100644 --- a/lib/Bastion/Chronology.py +++ b/lib/Bastion/Chronology.py @@ -144,6 +144,7 @@ def quaver(self): sQ = Quantim.EN36[msq] + Quantim.EN36[lsq] return self.separator.join([sY, sM, sD, sQ]) + @property def datetime(self): """ answers a python datetime.datetime object that is the midpoint of this quantum @@ -155,15 +156,25 @@ def datetime(self): elapsed_seconds = ((self.qM * Quantim.QUANTIM) + (Quantim.QUANTIM / 2)) * SECONDS return (y + elapsed_seconds) + @property def earliest(self): """ answers a python datetime.datetime object that is the earliest time within the range of this quantum """ y = datetime.datetime(self.dY + 2000, self.dM, self.dD, 0, 0, 0) - elapsed_seconds = self.qM * Quantim.QUANTIM + elapsed_seconds = (self.qM * Quantim.QUANTIM) * SECONDS + return (y + elapsed_seconds) + + def latest(self): + """ + answers a python datetime.datetime object that is the latest time possible within the range of this quantum. + """ + y = datetime.datetime(self.dY + 2000, self.dM, self.dD, 0, 0, 0) + elapsed_seconds = ((self.qM * Quantim.QUANTIM) + (Quantim.QUANTIM - 1)) * SECONDS return (y + elapsed_seconds) + @classmethod def now(cls): return cls(datetime.datetime.now()) diff --git a/lib/Bastion/Curator.py b/lib/Bastion/Curator.py index 9902b7f..4ffd4e6 100644 --- a/lib/Bastion/Curator.py +++ b/lib/Bastion/Curator.py @@ -6,7 +6,7 @@ import pathlib from .Common import * -from .Model import ARK +from .Model import ARK, isAsset from .Chronology import Quantim @@ -18,12 +18,14 @@ class BLONDE(canTextify): def __init__(self, asset, detail, basis = None, when = None): if isinstance(asset, ARK): self.badge = asset.badge + elif isinstance(asset, isAsset): + self.badge = asset.ARK.badge elif isinstance(asset, str): self.badge = asset else: raise ValueError - when = datetime.datetime.utcnow() if when is None else when + when = datetime.datetime.now() if when is None else when self.when = Quantim(when) self.detail = detail @@ -31,7 +33,7 @@ def __init__(self, asset, detail, basis = None, when = None): self.RDN = str(self) #-- Automatically create a basis reference for full backups. - if self.detail == 'F': + if (self.detail == 'F') and (self.basis is None): self.basis = Boggle(3) @property @@ -64,7 +66,7 @@ def decode(cls, blonde): 'detail': blonde[16], 'basis': blonde[17:20], }) - return cls(ds.badge, ds.when, ds.detail, ds.basis) + return cls(ds.badge, ds.detail, ds.basis, ds.when) @classmethod def forFullBackup(cls, ark, **kwargs): @@ -140,9 +142,9 @@ def __init__(self, asset, items): self._snaps = None self._anchors = None #-- a map of anchor layer -> blob name - blondes = [BLONDE(item) for item in items] + blondes = list(items) self._items = tuple(sorted([blonde for blonde in blondes if (blonde.badge == self.badge)], key = lambda b: b.RDN)) - self._anchors = dict([(item.layer, item) for item in self._items if item.isAnchor]) + self._anchors = dict([(item.basis, item) for item in self._items if item.isAnchor]) def __iter__(self): return iter(self._items) @@ -203,7 +205,7 @@ def anchors(self): """ I answer a chronologically (earliest -> latest) tuple of BLONDEs for my anchor items. """ - return tuple(sorted(self._anchors.values, key = lambda item: item.RDN)) + return tuple(sorted(self._anchors.values(), key = lambda item: item.RDN)) #------------------------ #-- canTextify protocol | diff --git a/lib/Bastion/Model.py b/lib/Bastion/Model.py index 84b283a..032811d 100644 --- a/lib/Bastion/Model.py +++ b/lib/Bastion/Model.py @@ -123,18 +123,19 @@ def provision(self, *args): """ raise NotImplementedError - def push(self, asset, **kwargs): + def push(self, asset, basis = None, **kwargs): """ Given an asset, I push a backup of the asset to this vault. - push(asset, detail = 'FULL') - push(asset, detail = 'DIFF') + push(asset) - creates a full backup in this vault, creating a new base for differentials + push(asset, basis) - creates a differential backup relative to the given basis. + {basis} - can be a datetime or a BLONDE. """ raise NotImplementedError def pull(self, blonde, **kwargs): raise NotImplementedError - def put(self, halo, tag): + def put(self, halo, tag, **kwargs): """ Given path to a local file (aka Host Asset LOcation), move the file from the local scope to this vault and store @@ -142,7 +143,7 @@ def put(self, halo, tag): """ raise NotImplementedError - def get(self, tag, halo): + def get(self, tag, halo, **kwargs): """ Given a tag (the path relative to the root of this vault), download the object and store it in the local file designated by halo. @@ -205,7 +206,7 @@ def CURIE(self): @property def ARK(self): - return ARK(self.zone.site.name, RDN(self.zone), self.name) + return ARK(RDN(self.zone.site), RDN(self.zone), self.name) @property def badge(self): diff --git a/lib/Bastion/Site.py b/lib/Bastion/Site.py index aa307eb..9917b42 100644 --- a/lib/Bastion/Site.py +++ b/lib/Bastion/Site.py @@ -300,11 +300,11 @@ def configured(self, aconf): @property def created(self): - return datetime.datetime.fromtimestamp( self.path.stat().st_stime ) + return datetime.datetime.fromtimestamp( self.halo.stat().st_ctime ) @property def modified(self): - return datetime.datetime.fromtimestamp( self.path.stat().st_mtime ) + return datetime.datetime.fromtimestamp( self.halo.stat().st_mtime ) diff --git a/lib/Bastion/Utils.py b/lib/Bastion/Utils.py new file mode 100644 index 0000000..4f5694f --- /dev/null +++ b/lib/Bastion/Utils.py @@ -0,0 +1,69 @@ +""" +Bastion.Utils + +I mostly contain "helper" functions that are agnostic to the vault protocol, etc. +""" +import datetime +import pathlib +import tarfile +import logging + +import Bastion.Model + +logger = logging.getLogger(__name__) + +class ifFileChanged: + """ + A class that can create callable instances to check + tarinfo metadata blocks for file modifications since a given data. + this is a filter for use with the python tarfile module and follows + the filter convention used where the filter modifies in-place the + given tarinfo object, or returns None to signal that the file should + be skipped. + e.g. + filter = ifChangedSince( datetime.datetime(2024,1,1) ) + if filter(tarinfo): + #-- add to tar file + else: + #-- return None + """ + def __init__(self, when): + self.whence = when + + def __call__(self, tarinfo): + if tarinfo.type not in (tarfile.REGTYPE, tarfile.AREGTYPE): + return tarinfo + + then = datetime.datetime.fromtimestamp(tarinfo.mtime) + logger.debug("comparing file {} mod'd at {} to change limit at {}".format(tarinfo.name, then.isoformat(), self.whence.isoformat())) + if then > self.whence: + return tarinfo + else: + #-- returning None is rejecting this file for inclusion + #-- in the accumulating tar file. + return None + +def pax(tarp, asset, **kwargs): + """ + pax uses the python tarfile module to create a tar file using the + extended POSIX.1-2001 (aka "PAX") format. + gnutar (as of the date of writing, 2024-09-04) can read PAX format, but still defaults to creating archives using its own modified USTAR format. + {tarp} is the halo (path) to the local file where the tar will be built. + {asset} can be a file path or an instance of Bastion.Model.isAsset (e.g. Bastion.Site.Asset) + differential backups can be done by supplying the "since" keyword set the datetime which is the earliest allowed modification time for files to be admitted into the tar. + """ + if isinstance(asset, Bastion.Model.isAsset): + src = asset.halo + else: + src = pathlib.Path(asset) + + with tarfile.open(tarp, "w", format = tarfile.PAX_FORMAT) as tar: + if 'since' in kwargs: + when = kwargs['since'] + tar.add(src, filter = ifFileChanged(when)) + else: + tar.add(src) + + #-- if no exceptions were generated during the tar construction, + #-- then we get here and we can return a happy True! + return True diff --git a/lib/Bastion/Vaults/BFD.py b/lib/Bastion/Vaults/BFD.py index db11355..a8f53f6 100644 --- a/lib/Bastion/Vaults/BFD.py +++ b/lib/Bastion/Vaults/BFD.py @@ -7,11 +7,17 @@ import socket import logging import getpass +import shutil +import tarfile +import logging from Bastion.Common import Thing, Unknown import Bastion.Model from Bastion.Curator import Manifest, BLONDE, Snap +from Bastion.Utils import pax + +logger = logging.getLogger(__name__) class Vault(Bastion.Model.Vault): PROTOCOL = 'BFD' @@ -19,21 +25,19 @@ class Vault(Bastion.Model.Vault): def __init__(self, name, **kwargs): Bastion.Model.Vault.__init__(self, name, **kwargs) self.bfd = pathlib.Path("/") - self.root = self.bfd / "bastion" / "bank" - self.scratch = self.bfd / "bastion" / "scratch" - self.tarx = 'tar' - + self.bank = self.bfd / "bank" + self.scratch = self.bfd / "scratch" def configured(self, conf): confkey = "vaults.{}".format(self.name) if confkey in conf: section = conf[confkey] - self.bfd = pathlib.Path( section['bfd.path'] ) - self.root = self.bfd / "bastion" / "bank" - self.scratch = self.bfd / "bastion" / "scratch" + self.bfd = pathlib.Path( section['root'] ) + self.bank = self.bfd / "bank" + self.scratch = self.bfd / "scratch" - if 'root.path' in section: - self.root = pathlib.Path( section['root.path'] ) + if 'bank.path' in section: + self.bank = pathlib.Path( section['bank.path'] ) if 'scratch.path' in section: self.scratch = pathlib.Path( section['scratch.path'] ) @@ -110,48 +114,86 @@ def provision(self, *args): else: raise ValueError - def push(self, asset, **kwargs): - #-- tar asset to scratch - #-- move from scratch to asset repository - raise NotImplementedError + def push(self, asset, basis = None, **kwargs): + """ + Given an asset, I push a backup of the asset to this vault. + push(asset) - creates a full backup in this vault, creating a new base for differentials + push(asset, basis) - creates a differential backup relative to the given basis. + {asset} - an instance of Bastion.Model.isAsset + {basis} - can be a datetime or a BLONDE. + """ + detail = 'F' + whence = None + branch = None + blonded = None + tarp = None + ark = asset.ARK + opts = { } + + if basis: + detail = 'D' + if isinstance(basis, BLONDE): + whence = basis.when.earliest + branch = basis.anchor + if isinstance(basis, datetime.datetime): + whence = basis + branch = "___" + blonded = BLONDE(asset.ARK, detail, branch) + tarp = "{}.tar".format(str(blonded)) + opts['since'] = whence + + else: + blonded = BLONDE(asset.ARK, detail, branch) + tarp = "{}.tar".format(str(blonded)) + + + #-- assure that the scratch and bank paths exist. + (self.scratch / ark.site / ark.zone / ark.asset).mkdir(parents = True, exist_ok = True) + (self.bank / ark.site / ark.zone / ark.asset).mkdir(parents = True, exist_ok = True) + + tag = "{}/{}/{}/{}".format(ark.site, ark.zone, ark.asset, tarp) + + pax(self.scratch / tag, asset.halo, **opts) + + self.put(self.scratch / tag, tag) + + #-- clean up! + (self.scratch / tag).unlink() + + return blonded def pull(self, ark, **kwargs): raise NotImplementedError - def put(self, halo, asARK, **kwargs): - raise NotImplementedError + def put(self, halo, tag, **kwargs): + here = halo + there = self.bank / tag + logger.debug("put source {} to {}".format(str(here), str(there))) + shutil.copy(here, there) - def get(self, ark, time, halo): - raise NotImplementedError + def get(self, tag, halo, **kwargs): + shutil.copystat(self.bank / tag, halo) #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ #-- END Bastion.Model.Vault PROTOCOL | #------------------------------------- - def _tar_asset(self, asset, **kwargs): - ark = asset.ARK - bucket = self.scratch / ark.site / ark.zone / ark.asset - bucket.mkdir(parents = True, exist_ok = True) - subprocess.run(cmd_tar, - raise NotImplementedError - def _copy_blonde(self, blonde, **kwargs): raise NotImplementedError def _empty_scratch(self, ark, **kwargs): - pass - + raise NotImplementedError def _manifest_ark(self, ark): #-- The contents of {root}/{site}/{zone}/{asset} are backup blobs #-- Each name in this folder is a "BLOND" (BLOb Name and Descriptor) #-- The manifest a catalog of all of the backup objects for the asset. - cell = self.root / ark.site / ark.zone / ark.asset + cell = self.bank / ark.site / ark.zone / ark.asset if cell.exists(): blondes = [ ] for item in cell.iterdir(): if not item.is_dir(): - blondes.append( item.name ) + blondes.append( BLONDE.decode(item.stem) ) manifest = Bastion.Curator.Manifest(ark, blondes) return manifest @@ -159,12 +201,10 @@ def _manifest_ark(self, ark): def _manifest_site_zone_asset(self, site, zone, asset): return self._manifest_ark( ARK(site, zone, asset) ) - def _provision_ark(self, ark): - repo = self.root / ark.site / ark.zone / ark.asset + repo = self.bank / ark.site / ark.zone / ark.asset return repo.mkdir(parents = True, exist_ok = True) - def _provision_site_zone_asset(self, site, zone, asset_name): return self._provision_ark( ARK(site, zone, asset_name) ) diff --git a/lib/Bastion/HPSS.py b/lib/Bastion/Vaults/HPSS.py similarity index 100% rename from lib/Bastion/HPSS.py rename to lib/Bastion/Vaults/HPSS.py diff --git a/lib/Bastion/Vaults/Local.py b/lib/Bastion/Vaults/Local.py deleted file mode 100644 index ac8850e..0000000 --- a/lib/Bastion/Vaults/Local.py +++ /dev/null @@ -1,8 +0,0 @@ -""" -Bastion.Vault.Local - -I am a vault for storing backups to a local file system. -""" - -from .Common import * -