From 1e66ed172196e9ea5246ce4f85566f3cede0a713 Mon Sep 17 00:00:00 2001 From: Nathan Denny Date: Mon, 21 Apr 2025 09:11:28 -0400 Subject: [PATCH] many updates ... the "enroll assets" feature seems to be working, now. --- bin/bastion.py | 45 ++++++++++++++++- lib/Bastion/Chronology.py | 98 +++++++++++++------------------------ lib/Bastion/Clerks/SFTP.py | 8 --- lib/Bastion/Model.py | 16 ++++-- lib/Bastion/NetOps/sCURL.py | 24 ++++++++- lib/Bastion/Packers/TARs.py | 4 +- 6 files changed, 114 insertions(+), 81 deletions(-) diff --git a/bin/bastion.py b/bin/bastion.py index 55425de..2040db2 100755 --- a/bin/bastion.py +++ b/bin/bastion.py @@ -808,8 +808,49 @@ def do_enroll_assets(self, request): On successful completion, I create (or overwrite) a configuration file that is the asset catalog for the given zone. By default, the asset catalog is written to ~/.bastion/conf-{site}-{zone}-catalog.yaml, unless otherwise specified by the arg: "catalog.path" - """ - raise NotImplementedError + * enroll assets {site} {zone} + * enroll assets {site} {zone} -catalog.path:{path} + * enroll assets {ARK} + -- note that all enrolled assets are assumed to use the same policy as their parent zone. + """ + #-- scan the given zone for all assets. + #-- I then write the new configuration file to the given path. + #-- If no path is given, I write the file to ~/.bastion/conf-{site}-{zone}.yaml + #-- If the file already exists, I overwrite it. + #-- If the file doesn't exist, I create it. + #-- If the file is not writable, I crash. + if len(request.args) == 1: + #-- We were given a single ARK argument. + ark = ARK(request.args[0]) + elif len(request.args) == 2: + #-- We were given two arguments (site, zone) + ark = ARK(request.args[0], request.args[1]) + else: + raise ValueError("do_enroll_assets expects zone to be given as a CURIE'd ARK") + + site = self.site(ark.site) + zone = site.zone(ark) + + #-- Create a new configuration file for the zone. + catalog_path = request.context.get('catalog.path', "~/.bastion/conf-{}-{}.yaml".format(ark.site, ark.zone)) + catalog_path = pathlib.Path(catalog_path).expanduser() + + #-- Scan the zone for all assets. + assets = [] + for entry in zone.halo.iterdir(): + if entry.is_dir(): + assets.append(entry.name) + print(assets) + + #-- Write the updated configuration to the file. + catalog = Condex() + catalog["assets.{}.{}".format(ark.site, ark.zone)] = assets + catalog_path.parent.mkdir(parents=True, exist_ok=True) + with open(catalog_path, 'w') as fout: + yaml = YAML() + yaml.dump(catalog.toJDN(), fout) + + return request.succeeded(None, report="Assets enrolled and catalog written to {}".format(catalog_path)) if __name__ == '__main__': diff --git a/lib/Bastion/Chronology.py b/lib/Bastion/Chronology.py index 8aed592..363501b 100644 --- a/lib/Bastion/Chronology.py +++ b/lib/Bastion/Chronology.py @@ -9,45 +9,6 @@ logger = logging.getLogger(__name__) -def quantim(then = None, **kwargs): - """ - I generate a quantized time string. - I default to YYYMDDQQ format, but can do years as either 2, 3, or 4 digits of precision. - """ - marker = kwargs.get('separator', '') - yform = kwargs.get('precision', 3) - - QUANTIM = 86400.0 / (36**2) - when = then if (then is not None) else datetime.datetime.now() - - ds = (when.hour * 3600) + (when.minute * 60) + when.second + (when.microsecond / 1000000) - dq = round( ds / QUANTIM ) - lsq = dq % 36 - msq = dq // 36 - xmap = tuple(string.digits + string.ascii_uppercase) - - if yform == 2: - yn = "{:02d}".format(when.year - 2000) - elif yform == 3: - yn = "{:03d}".format(when.year - 2000) - else: - raise ValueError("year format must be one of 2, 3, or 4 digits") - - mo = "{:X}".format(when.month) - dy = "{:02d}".format(when.day) - qt = xmap[msq] + xmap[lsq] - - return marker.join([yn, mo, dy, qt]) - - -def quaver(then = None): - """ - I am a quick quantized version timestamp. - I use the YYMDDQQ format. - """ - return quantim(then, precision = 2) - - class Quantim: """ Quantized Time (Quantim). @@ -87,9 +48,10 @@ def __init__(self, whence, separator = None): elif isinstance(whence, str): if self.separator: words = whence.split(self.separator) - if len(words) == 4: - wY = words[0] - wDMQ = ''.join(words[1:]) + if len(words) == 3: + yW = words[0] + dW = words[1] + qW = words[2] else: raise Exception("Quantim:__init__ parse error for '{}'".format(whence)) else: @@ -117,6 +79,7 @@ def __init__(self, whence, separator = None): def __str__(self): lsq = self.qM % 36 msq = self.qM // 36 + xmap = list(string.digits + string.ascii_uppercase) sY = "{:03d}".format(self.dY) sM = "{:X}".format(self.dM) sD = "{:02d}".format(self.dD) @@ -126,7 +89,7 @@ def __str__(self): @property def quaver(self): """ - I am the QUAntim VERsioning stamp. + I am the [QUA]ntim [VER]sioning stamp. I am used a compact timestamp. I use only 2-digit years, so anything after 2099 won't work. :) """ @@ -135,17 +98,14 @@ def quaver(self): lsq = self.qM % 36 msq = self.qM // 36 - sY = "{:03d}".format(self.dY) + xmap = list(string.digits + string.ascii_uppercase) + sY = "{:02d}".format(self.dY) sM = "{:X}".format(self.dM) sD = "{:02d}".format(self.dD) sQ = Quantim.EN36[msq] + Quantim.EN36[lsq] return self.separator.join([sY, sM, sD, sQ]) - @property def datetime(self): - """ - answers a python datetime.datetime object that is the midpoint of this quantum - """ if self._when is not None: return self._when else: @@ -154,26 +114,38 @@ def datetime(self): return (y + elapsed_seconds) @property - def earliest(self): - """ - answers a python datetime.datetime object that is the earliest time within the range of this quantum + def quake(self): """ - y = datetime.datetime(self.dY + 2000, self.dM, self.dD, 0, 0, 0) - elapsed_seconds = (self.qM * Quantim.QUANTIM) * SECONDS - return (y + elapsed_seconds) - - @property - def latest(self): - """ - answers a python datetime.datetime object that is the latest time possible within the range of this quantum. + I am the [QUA]ntim [K]ilo-year [E]ncoding + I am used a compact timestamp. + I use only 3-digit years, so anything after 2999 won't work. :) """ - y = datetime.datetime(self.dY + 2000, self.dM, self.dD, 0, 0, 0) - elapsed_seconds = ((self.qM * Quantim.QUANTIM) + (Quantim.QUANTIM - 1)) * SECONDS - return (y + elapsed_seconds) - + if (self.dY >= 1000): + raise ValueError("quakes are only defined on years 2000-2999") + lsq = self.qM % 36 + msq = self.qM // 36 + xmap = list(string.digits + string.ascii_uppercase) + sY = "{:03d}".format(self.dY) + sM = "{:X}".format(self.dM) + sD = "{:02d}".format(self.dD) + sQ = Quantim.EN36[msq] + Quantim.EN36[lsq] + return self.separator.join([sY, sM, sD, sQ]) @classmethod def now(cls): return cls(datetime.datetime.now()) + @classmethod + def fromtimestamp(cls, timestamp): + """ + Convert a POSIX timestamp to a Quantim instance. + """ + return cls(datetime.datetime.fromtimestamp(timestamp)) + + +def quaver(): + return Quantim.now().quaver + +def quake(): + return Quantim.now().quake diff --git a/lib/Bastion/Clerks/SFTP.py b/lib/Bastion/Clerks/SFTP.py index 42f01c8..1168bed 100644 --- a/lib/Bastion/Clerks/SFTP.py +++ b/lib/Bastion/Clerks/SFTP.py @@ -51,12 +51,8 @@ def assets(self, site, zone): zroot = self.scurler / site / zone if zroot.exists( ): for alien in zroot: -<<<<<<< HEAD - assets.append(alien.name) -======= if alien.is_folder: assets.append(alien.name) ->>>>>>> 9fcbbffdc039d6ad10d92101171b6841f6fa7756 return tuple(sorted(assets)) def manifest(self, *args): @@ -82,10 +78,6 @@ def manifest(self, *args): cell = self.scurler / ark.site / ark.zone / ark.asset if cell.exists: for alien in cell.lsall(): -<<<<<<< HEAD - print("ALIEN: ", alien) -======= ->>>>>>> 9fcbbffdc039d6ad10d92101171b6841f6fa7756 if not alien.is_folder: blondes.append( BLONDE.decode(alien.stem) ) manifest = Manifest(ark, blondes) diff --git a/lib/Bastion/Model.py b/lib/Bastion/Model.py index 25a803c..707e56e 100644 --- a/lib/Bastion/Model.py +++ b/lib/Bastion/Model.py @@ -328,7 +328,7 @@ def badge(self): @property def halo(self): """ - host asset location (halo) + host area location (halo) The local (relative to the host) file system path to the asset. """ return self.zone.root / pathlib.Path(self.name) @@ -351,6 +351,14 @@ def __init__(self, site, name, root): self.name = name self.root = root if (root is None) else pathlib.Path(root) + @property + def halo(self): + """ + host area location (halo) + The local (relative to the host) file system path to the zone. + """ + return self.root + @property def RDN(self): """ @@ -369,7 +377,7 @@ class isSite: pass -class isClerk(isActor): +class isClerk(isPerformer): """ abstract class for metadata and file management specific to the capabilities of a given vault type. """ @@ -415,7 +423,7 @@ def manifest(self, ark): raise NotImplementedError -class isPacker(isActor): +class isPacker(isPerformer): def __init__(self, vault): assert isinstance(vault, isVault), "vault must be an instance of Bastion.Model.isVault" @@ -444,7 +452,7 @@ def unpack(self, halo, root, **kwargs): -class isMover(isActor): +class isMover(isPerformer): """ abstract class for file movement in to and out of a specific vault type. """ diff --git a/lib/Bastion/NetOps/sCURL.py b/lib/Bastion/NetOps/sCURL.py index 47dccbd..d13ec4c 100644 --- a/lib/Bastion/NetOps/sCURL.py +++ b/lib/Bastion/NetOps/sCURL.py @@ -162,7 +162,7 @@ def toDEX(self): dex[k] = v return dex - def toJDN(self): + def toJDN(self) -> dict: jdn = self.toDEX() jdn['path'] = str(jdn['path']) if 'mdate' in jdn: @@ -222,6 +222,14 @@ def lsall(self): def mkdir(self): return self.scurler.mkdir(self.rpath) + def rename(self, newname): + """ + Rename the remote object to the new name within the same parent directory. + """ + newpath = self.rpath.parent / newname + self.scurler.rename(self.rpath, newpath) + self.rpath = newpath + def is_file(self): if self.permits is not None: return (self.permits[0] != 'd') @@ -446,6 +454,18 @@ def rm(self, rpath): Given an absolute path on the remote host (rpath), I execute an "rm {}" command on the remote host. """ self.quote("rm", str(rpath)) + + def rename(self, rsrcpath, rdestpath): + """ + Given an absolute path on the remote host (rsrcpath), I execute a remote "rename" operation to the absolute path rdestpath. + """ + self.quote("rename", str(rsrcpath), str(rdestpath)) + + def rmdir(self, rpath): + """ + Given an absolute path on the remote host (rpath), I execute a remote "rmdir" operation. + """ + self.quote("rmdir", str(rpath)) def ls(self, rpath = None): """ @@ -471,7 +491,7 @@ def lsall(self, rpath = None): """ Given an absolute path on the remote host (rpath), I execute a remote "ls" operation. """ - rqURL = self.sfURL(rpath) + rqURL = self.sfURL(rpath) #-- the "request" URL lsURL = str(rqURL) lsURL = lsURL if lsURL.endswith('/') else "{}/".format(lsURL) diff --git a/lib/Bastion/Packers/TARs.py b/lib/Bastion/Packers/TARs.py index 625bb58..64c3ebe 100644 --- a/lib/Bastion/Packers/TARs.py +++ b/lib/Bastion/Packers/TARs.py @@ -8,7 +8,7 @@ import tarfile import logging -from Bastion.Chronology import quantim +from Bastion.Chronology import Quantim from Bastion.Common import asPath, prefer from Bastion.Model import isAsset, isPacker from Bastion.CARP import isRequest, Report @@ -166,6 +166,6 @@ def catalog(tarp): with tarfile.open(tarp) as tark: for info in tark.getmembers(): sz = info.size - mq = quantim( datetime.datetime.fromtimestamp(info.mtime) ) + mq = Quantim.fromtimestamp(info.mtime).quaver cats.append( [sz, mq, info.name] ) return cats