From a8bb02c75e2785b16acaa83c9b38a88316e13526 Mon Sep 17 00:00:00 2001 From: Nathan Denny Date: Thu, 11 Jul 2024 05:00:16 -0400 Subject: [PATCH] Continuing development, still at an incomplete stage. --- bin/bastion.py | 34 +++++++++- lib/Bastion/Chronology.py | 127 ++++++++++++++++++++++++----------- lib/Bastion/HPSS.py | 60 +++++++++-------- lib/Bastion/Model.py | 8 +++ lib/Bastion/Vaults/Common.py | 1 + 5 files changed, 162 insertions(+), 68 deletions(-) diff --git a/bin/bastion.py b/bin/bastion.py index f03d483..4097725 100755 --- a/bin/bastion.py +++ b/bin/bastion.py @@ -112,6 +112,9 @@ def site(self, name): return Site(name).configured(self.conf) def vault(self, name, site = None): + """ + I answer an instance of a storage vault given a name and optional site. + """ if ((name[0] == '{') and (name[-1] == '}')): name = name[1:-1] if name in self.conf['vaults']: @@ -169,18 +172,45 @@ def do_help(self, comargs, comdex): #---------------------- #-- backup operations | #---------------------- - def do_backup_zone(self, zone): + def do_backup_site(self, comargs, comdex): + """ + backup site {site} + * creates a full backup for each asset in site. + """ raise NotImplementedError - def do_update_zone(self, zone): + def do_update_site(self, comargs, comdex): + """ + update site {site} + * creates a differential backup for each asset in {site} + """ + raise NotImplementedErro + + def do_backup_zone(self, comargs, comdex): + """ + backup zone {zone} + * creates a full backup for each asset in {zone}. + """ + raise NotImplementedError + + def do_update_zone(self, comargs, comdex): + """ + update zone {zone} + * creates a differential backup of each asset in {zone} + """ raise NotImplementedError def do_update_asset(self, comargs, comdex): + """ + update asset {ARK} + * creates a differential backup of {ARK} + """ raise NotImplementedError def do_backup_asset(self, comargs, comdex): """ backup asset {ARK} + * creates a full backup of {ARK} """ ark = ARK(comdex[2]) site = self.site(ark.site) diff --git a/lib/Bastion/Chronology.py b/lib/Bastion/Chronology.py index 9ecd72d..140486e 100644 --- a/lib/Bastion/Chronology.py +++ b/lib/Bastion/Chronology.py @@ -7,36 +7,50 @@ logger = logging.getLogger(__name__) - -def quantim3(then = None, **kwargs): +SECOND = datetime.timedelta(seconds = 1) +SECONDS = SECOND +MINUTE = datetime.timedelta(minutes = 1) +MINUTES = MINUTE +HOUR = datetime.timedelta(hours = 1) +HOURS = HOUR + +def quantim(then = None, **kwargs): + """ + I generate a quantized time string. + I default to YYYMDDQQ format, but can do years as either 2, 3, or 4 digits of precision. + """ marker = kwargs.get('separator', '') + yform = kwargs.get('precision', 3) + QUANTIM = 86400.0 / (36**2) when = then if (then is not None) else datetime.datetime.now() + ds = (when.hour * 3600) + (when.minute * 60) + when.second + (when.microsecond / 1000000) dq = round( ds / QUANTIM ) lsq = dq % 36 msq = dq // 36 - xmap = list(string.digits + string.ascii_uppercase) - y3 = "{:03d}".format(when.year - 2000) - doy = "{:03d}".format(when.timetuple().tm_yday) - qt = xmap[msq] + xmap[lsq] - return marker.join([y3, doy, qt]) + xmap = tuple(string.digits + string.ascii_uppercase) + if yform == 2: + yn = "{:02d}".format(when.year - 2000) + elif yform == 3: + yn = "{:03d}".format(when.year - 2000) + else: + raise ValueError("year format must be one of 2, 3, or 4 digits") -def quiver(then = None): - QUANTIM = 86400.0 / (36**2) - when = then if (then is not None) else datetime.datetime.now() - ds = (when.hour * 3600) + (when.minute * 60) + when.second + (when.microsecond / 1000000) - dq = round( ds / QUANTIM ) - lsq = dq % 36 - msq = dq // 36 - xmap = list(string.digits + string.ascii_uppercase) + mo = "{:X}".format(when.month) + dy = "{:02d}".format(when.day) qt = xmap[msq] + xmap[lsq] - y2 = "{:02d}".format(when.year - 2000) - mo = "{:X}".format(when.month) + return marker.join([yn, mo, dy, qt]) - return "{}{}{}{}".format(y2, mo, when.day, qt) + +def quaver(then = None): + """ + I am a quick quantized version timestamp. + I use the YYMDDQQ format. + """ + return quantim(then, precision = 2) class Quantim: @@ -48,23 +62,30 @@ class Quantim: With two digits, the day is divided into 1,296 quantums, each of which is ~ 66.67 seconds. """ - EN36 = list(string.digits + string.ascii_uppercase) + EN36 = tuple(string.digits + string.ascii_uppercase) DE36 = dict([(c, i) for i, c in enumerate(EN36)]) - QUANTUM = 86400.0 / (36**2) + QUANTIM = 86400.0 / (36**2) def __init__(self, whence, separator = None): + self.dYr = None + self.dMo = None + self.dDy = None + self.dHM = None + self._when = None + self.separator = separator if (separator is not None) else '' if isinstance(whence, datetime.datetime): - year_starts = datetime.datetime(whence.year, 1, 1, 0, 0, 0) - adnl_seconds = (whence - year_starts).seconds + self._when = whence - self.dY = whence.year - 2000 - self.dD = whence.timetuple().tm_yday - self.qM = round((whence.hour * 3600) + (whence.minute * 60) + whence.second + (whence.microsecond / 1000000)) // Quantim.QUANTUM + self.dY = whence.year - 2000 + self.dM = whence.month + self.dD = whence.day + self.qM = round( ((whence.hour * 3600) + (whence.minute * 60) + whence.second + (whence.microsecond / 1000000)) / Quantim.QUANTUM ) elif isinstance(whence, Quantim): self.dY = whence.dY + self.dM = whence.dM self.dD = whence.dD self.qM = whence.qM @@ -78,15 +99,21 @@ def __init__(self, whence, separator = None): else: raise Exception("Quantim:__init__ parse error for '{}'".format(whence)) else: - if len(whence) == 8: - yW = whence[0:3] - dW = whence[3:6] - qW = whence[6:8] + #-- There are three format options: YYYMDDQQ, YYMDDQQ + if len(whence) == 7: + wY = whence[0:2] + wDMQ = whence[2:] + elif len(whence) == 8: + wY = whence[0:3] + wDMQ = whence[3:] else: raise Exception("Quantim:__init__ parse error for '{}'".format(whence)) - self.dY = int(yW) - self.dD = int(dW) + qW = wDMQ[3:5] + + self.dY = int(wY) + self.dM = int(wDMQ[0], 16) + self.dD = int(wDMQ[1:3]) self.qM = (Quantim.DE36[qW[0]] * 36) + Quantim.DE36[qW[1]] else: @@ -97,16 +124,38 @@ def __str__(self): lsq = self.qM % 36 msq = self.qM // 36 xmap = list(string.digits + string.ascii_uppercase) - yW = "{:03d}".format(self.dY) - dW = "{:03d}".format(self.dD) - qW = xmap[msq] + xmap[lsq] - return self.separator.join([yW, dW, qW]) + sY = "{:03d}".format(self.dY) + sM = "{:X}".format(self.dM) + sD = "{:02d}".format(self.dD) + sQ = Quantim.EN36[msq] + Quantim.EN36[lsq] + return self.separator.join([sY, sM, sD, sQ]) + + @property + def quaver(self): + """ + I am the QUAntim VERsioning stamp. + I am used a compact timestamp. + I use only 2-digit years, so anything after 2099 won't work. :) + """ + if (self.dY >= 100): + raise ValueError("quavers are only defined on years 2000-2099") + + lsq = self.qM % 36 + msq = self.qM // 36 + xmap = list(string.digits + string.ascii_uppercase) + sY = "{:03d}".format(self.dY) + sM = "{:X}".format(self.dM) + sD = "{:02d}".format(self.dD) + sQ = Quantim.EN36[msq] + Quantim.EN36[lsq] + return self.separator.join([sY, sM, sD, sQ]) def datetime(self): - y = datetime.datetime(self.dY + 2000, 1, 1, 0, 0, 0) - elapsed_days = self.dD * DAYS - elapsed_seconds = ((self.qM * Quantim.QUANTUM) + (Quantim.QUANTUM / 2)) * SECONDS - return (y + elapsed_days + elapsed_seconds) + if self._when is not None: + return self._when + else: + y = datetime.datetime(self.dY + 2000, self.dM, self.dD, 0, 0, 0) + elapsed_seconds = ((self.qM * Quantim.QUANTIM) + (Quantim.QUANTIM / 2)) * SECONDS + return (y + elapsed_seconds) @classmethod def now(cls): diff --git a/lib/Bastion/HPSS.py b/lib/Bastion/HPSS.py index 1f26a3b..6bb983b 100644 --- a/lib/Bastion/HPSS.py +++ b/lib/Bastion/HPSS.py @@ -349,12 +349,9 @@ def hsi(self): self._hsi = HSI(xpath = self.xpath, login = self.login, keytab = self.keytab) return self._hsi - @property - def sites(self): - #-- sites are top level elements relative to the root of the vault. - fls = self.hsi.ls(self.root) - return tuple(sorted([fl.path.name for fl in fls if fl.isdir()])) - +#--------------------------------------- +#-- BEGIN Bastion.Model.Vault PROTOCOL | +#↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ @property def ARKs(self): arks = [ ] @@ -364,6 +361,12 @@ def ARKs(self): arks.append( ARK(site, zone, asset) ) return tuple(sorted(arks)) + @property + def sites(self): + #-- sites are top level elements relative to the root of the vault. + fls = self.hsi.ls(self.root) + return tuple(sorted([fl.path.name for fl in fls if fl.isdir()])) + def zones(self, site): #-- a zone will be a subdirectory (subfolder) of the given site. #-- look for all of the subfolders of root / site @@ -392,21 +395,6 @@ def manifest(self, *args): else: raise ValueError - - def _manifest_ark(self, ark): - #-- The contents of {root}/{site}/{zone}/{asset} are backup blobs - #-- Each name in this folder is a "BLOND" (BLOb Name and Descriptor) - #-- The manifest a catalog of all of the backup objects for the asset. - fls = self.hsi.ls(self.root / ark.site / ark.zone / ark.asset) - blondes = [fl.path.name for fl in fls] - manifest = Bastion.Curator.Manifest(ark, blondes) - - return manifest - - def _manifest_site_zone_asset(self, site, zone, asset): - return self._manifest_ark( ARK(site, zone, asset) ) - - def provision(self, *args): """ provision(ark) - ensures that the site, zone, and asset folders exist. @@ -420,12 +408,6 @@ def provision(self, *args): else: raise ValueError - def _provision_ark(self, ark): - self.hsi.mkdirs(self.root / ark.site / ark.zone / ark.asset) - - def _provision_site_zone_asset(self, site, zone, asset_name): - return self._provision_ark( ARK(site, zone, asset_name) ) - def push(self, asset, **kwargs): detail = kwargs.get('detail', 'F') localf = asset.path @@ -457,5 +439,29 @@ def push(self, asset, **kwargs): flag = True if (proc.returncode == 0) else False return (flag, stdout, stderr) +#↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ +#-- END Bastion.Model.Vault PROTOCOL | +#------------------------------------- + + def _manifest_ark(self, ark): + #-- The contents of {root}/{site}/{zone}/{asset} are backup blobs + #-- Each name in this folder is a "BLOND" (BLOb Name and Descriptor) + #-- The manifest a catalog of all of the backup objects for the asset. + fls = self.hsi.ls(self.root / ark.site / ark.zone / ark.asset) + blondes = [fl.path.name for fl in fls] + manifest = Bastion.Curator.Manifest(ark, blondes) + + return manifest + + def _manifest_site_zone_asset(self, site, zone, asset): + return self._manifest_ark( ARK(site, zone, asset) ) + + + def _provision_ark(self, ark): + self.hsi.mkdirs(self.root / ark.site / ark.zone / ark.asset) + + def _provision_site_zone_asset(self, site, zone, asset_name): + return self._provision_ark( ARK(site, zone, asset_name) ) + #hsi = HSI("/opt/hsi/bin/hsi", login = "ndenny") diff --git a/lib/Bastion/Model.py b/lib/Bastion/Model.py index a35dcc2..cf9c531 100644 --- a/lib/Bastion/Model.py +++ b/lib/Bastion/Model.py @@ -108,6 +108,14 @@ def manifest(self, ark): """ raise NotImplementedError + def provision(self, *args): + """ + provision(ark) - ensures that the site, zone, and asset folders exist. + provision(site, zone, asset_name) - an alias for provision(ark) + provision(asset) - given an instance of Asset, provision the necessary site, zone, and asset folders. + """ + raise NotImplementedError + def push(self, asset, **kwargs): """ Given an asset, I push a backup of the asset to this vault. diff --git a/lib/Bastion/Vaults/Common.py b/lib/Bastion/Vaults/Common.py index e451cbc..6f0f633 100644 --- a/lib/Bastion/Vaults/Common.py +++ b/lib/Bastion/Vaults/Common.py @@ -60,6 +60,7 @@ def __init__(self, site, zone, archive, branch): def snaps(self): raise NotImplementedError + class isVault: """ I am an abstract base type for specialized Vault classes.