From 694e0a78852b1335d61953b1f00c7554aa6172c6 Mon Sep 17 00:00:00 2001 From: Nathan Denny Date: Thu, 5 Jun 2025 12:29:20 -0400 Subject: [PATCH] Added an idiom class for pattern matching vs. command line; extending isRequest to read command line; re-established hasCanonicalForm and fromCanonicalForm as trait classes to signal toJDN capabilities; tightened up requirements for records (in results) to have hasCanonicalForm traits; tar catalog is now an object (not a list). --- bin/bastion.py | 44 ++++++++----- lib/Bastion/CARP.py | 128 ++++++++++++++++++++++++++++++++++-- lib/Bastion/Common.py | 4 +- lib/Bastion/Packers/CARP.py | 4 +- lib/Bastion/Packers/TARs.py | 79 ++++++++++++++++++---- lib/Bastion/Site.py | 7 ++ 6 files changed, 229 insertions(+), 37 deletions(-) diff --git a/bin/bastion.py b/bin/bastion.py index 682de5b..acc3ae4 100755 --- a/bin/bastion.py +++ b/bin/bastion.py @@ -48,6 +48,19 @@ a. differential if drift < policy b. anchor (full) if drift >= policy """ +class Commander: + def __init__(self, **kwargs): + self.simulate = kwargs.get('simulate', False) + + def do(self, cmd): + if self.simulate: + logger.debug("simulating... {}".format(cmd)) + xcode = 0 + else: + logger.debug("executing... {}".format(cmd)) + xcode = os.system(cmd) + return xcode + class ProtectionRequest(Request): """ @@ -981,33 +994,33 @@ def do_become_conductor(self, request): I enter into a forever loop, napping for 15 second intervals, then waking to check the time and do any necessary housekeeping and backup operations. """ simulated = ('simulate!' in request.args.values()) or ('simulated!' in request.args.values()) + commander = Commander(simulate = simulated) lastCheckIn = datetime.datetime(1, 1, 1, 0, 0, 0) #-- 1 JAN 1 CE + while ALWAYS: now = datetime.datetime.now() logger.debug("tick-tock {}".format(now.isoformat())) + if (now - lastCheckIn) < (24*HOURS): time.sleep(15.0) #-- Sleep 15 seconds. + else: #-- A brand new day! + for site in self.sites: #-- Update the asset list for all of the sites that I'm managing... for zone in site.zones: - if zone.dynamic: + if zone.isDynamic: ecommand = '{}/bastion.py enroll assets "{}"'.format(str(BIN_PATH), str(zone.ARK)) - if not simulated: - logger.debug("executing... {}".format(ecommand)) - os.system(ecommand) - else: - logger.debug("simulating... {}".format(ecommand)) + commander.do( ecommand ) + for site in self.sites: #-- Queue assets for updates for all of the sites that I'm managing... for zone in site.zones: + #-- Assets are queued by zone... qcommand = '{}/bastion.py queue assets "{}"'.format(str(BIN_PATH), str(zone.ARK)) - if not simulated: - logger.debug("executing... {}".format(qcommand)) - os.system(qcommand) - else: - logger.debug("simulating... {}".format(qcommand)) + commander.do( qcommand ) + for site in self.sites: #-- Spawn a subprocess to pop all of the objects in the queue for each siteXzone combo.... for zone in site.zones: @@ -1016,11 +1029,8 @@ def do_become_conductor(self, request): #-- if IONICE is defined, then run the bank operations with IO niced to "Idle", #-- i.e. disk access for banking is only done when no other processes need the disk. bcommand = "{} -c 3 -t {}".format(str(IONICE), bcommand) - if not simulated: - logger.debug("executing... {}".format(bcommand)) - os.system(bcommand) - else: - logger.debug("simulating... {}".format(bcommand)) + commander.do( bcommand ) + lastCheckIn = datetime.datetime.now() @@ -1091,4 +1101,4 @@ def do_become_conductor(self, request): #bastion perform request filed #bastion perform request queued -#bastion become conductor \ No newline at end of file +#bastion become conductor diff --git a/lib/Bastion/CARP.py b/lib/Bastion/CARP.py index 082c357..ce759b5 100644 --- a/lib/Bastion/CARP.py +++ b/lib/Bastion/CARP.py @@ -28,7 +28,7 @@ from collections.abc import Sequence -from Bastion.Common import toJDN, isBoxedObject, isTagged +from Bastion.Common import toJDN, hasCanonicalForm, isBoxedObject, isTagged class ReplyStatus(tuple): @@ -134,22 +134,98 @@ def is_inconclusive(self): ResultStatus = ReplyStatus #-- Alias for ReplyStatus -class isRequest: +class Idiom(tuple): + def __new__(cls, idiom): + if isinstance(idiom, Idiom): + return idiom + elif isinstance(idiom, str): + tokens = [token.strip() for token in idiom.split()] + return tuple.__new__(cls, tokens) + else: + raise ValueError + + def slots(self): + if not hasattr(self, "_slots"): + slots = [ ] + for token in self: + if (token[0] == '_') and (token[-1] == '_'): + slots.append(token[1:-1]) + self._slots = tuple(slots) + return self._slots + + @property + def priority(self): + weight = len(self) + density = len([token for token in self if ((token[0] != '_') and (token[0] != '...'))]) + return (weight, density) + + def __str__(self): + return " ".join(self) + + def __repr__(self): + return "🧠:{}".format(str(self)) + + def __mod__(self, other): + """ + congruence operator. + i.e. Idiom("_ help") % "get help" -> (Idiom("_help"), ["get"]) + """ + words = [word.strip() for word in other.split()] + + if len(self) > len(words): + #-- I have more tokens than other, I can't possibly be a match for other. + return None + + planks = [ ] + args = { } + + for i, token in enumerate(self): + word = words.pop(0) + if token == '...': + remainder = [word] + words + planks.extend(remainder) + args['...'] = " ".join(remainder) + return (self, args, tuple(fillers)) + elif token == '_': + planks.append(word) + elif (len(token) > 1) and (token[0] == '_') and (token[-1] == '_'): + slot = token[1:-1] + args[slot] = word + planks.append(word) + elif token != word: + return None + + remainder = words + + return (self, args, tuple(planks), remainder) + +#-- Some ideas for testing Idiom... +#menu = [Idiom(m) for m in ["help", "help ...", "bank asset _ark_", "update asset _ark_"]] +#menu = sorted(menu, key = lambda i: i.priority, reverse = True) + +class isRequest(hasCanonicalForm): def __init__(self, action, *args, **kwargs): self.ID = kwargs.get('ID', str(uuid.uuid4())) self.action = action self.when = kwargs.get('opened', datetime.datetime.now()) self.args = dict(enumerate(args)) self.context = { } + self.flags = set() - excluded = {'ID', 'opened', 'context'} + excluded = {'ID', 'opened', 'context', 'flags'} for k, v in kwargs.items(): if k not in excluded: self.context[k] = v + if 'context' in kwargs: for k, v in kwargs['context'].items(): self.context[k] = v + if 'flags' in kwargs: + self.flags.update(kwargs['flags']) + for flag in self.flags: + self.context[flag] = True + @property def lede(self): return ' '.join( [self.action] + [self.args[i] for i in sorted(self.args.keys())] ) @@ -166,7 +242,8 @@ def toJDN(self, **kwargs): 'action': self.action, 'args': { }, 'opened': self.when.isoformat(), - 'context': { } + 'context': { }, + 'flags': list(self.flags) } if self.args: for k, v in self.args.items(): @@ -189,10 +266,49 @@ def crashed(self, record = None, *args, **kwargs): def inconclusive(self, record = None, *args, **kwargs): return Report.Inconclusive(self, record, *args, **kwargs) + @classmethod + def fromCLI(cls, argls = None, **kwargs) + """ + Convenience constructor for creating a request directly from the command line (sys.argv) + e.g. + request = Request.fromCLI() + """ + flags = [] + opts = { } + idioms = kwargs.get('idioms', []) + + if argls is None: + argls = sys.argv[1:] + + if '--' in argls: + i = argls.index('--') + comargls = argls[:i] + optl = argls[i+1:] + for word in optls: + if word[-1] == '!': + flag = word[:-1] + flags.append(flag) + if ':' in word: + j = word.index(':') + var = word[:j] + val = word[j+1:] + opts[var] = val + + #-- by default, action is the concatenation of the command arg list. + action = " ".join(comargls) + #-- However, if we have a list of idioms to match... + if idioms: + menu = [Idiom(m) for m in in idioms] + for idiom in menu: + if idiom % opts: + + return cls(action, *comargls, context = opts, flags = flags) + + #-- legacy alias Request = isRequest -class isResult: +class isResult(hasCanonicalForm): def __init__(self, request, status, record, **kwargs): #-- POSITIONAL ARGS... #-- request is a REQUIRED 2-tuple of (action, [arg1, ...]) @@ -345,7 +461,7 @@ def __init__(self, request, *args, **kwargs): -class isReceipt: +class isReceipt(hasCanonicalForm): """ I am an abstract base class for receipts. Mostly, I'm used for run-time type checking. diff --git a/lib/Bastion/Common.py b/lib/Bastion/Common.py index c5764ba..ffef0b0 100644 --- a/lib/Bastion/Common.py +++ b/lib/Bastion/Common.py @@ -349,7 +349,7 @@ def Slug40(text): return base64.b32encode(bs).decode('utf-8') -class canStruct: +class hasCanonicalForm: """ I am a trait for serialization using JSON and YAML. """ @@ -365,7 +365,7 @@ def toYAML(self, **kwargs): return yaml.dump(jdn, default_flow_style = False, indent = 3) -class canConStruct: +class fromCanonicalForm: @classmethod def fromJDN(cls, jdn, **kwargs): raise NotImplementedError(".fromJDN is subclass responsibility") diff --git a/lib/Bastion/Packers/CARP.py b/lib/Bastion/Packers/CARP.py index a6a991a..0b89930 100644 --- a/lib/Bastion/Packers/CARP.py +++ b/lib/Bastion/Packers/CARP.py @@ -2,6 +2,7 @@ import datetime import logging +from Bastion.Common import toJDN, hasCanonicalForm from Bastion.Model import isAsset from Bastion.CARP import isRequest, isReceipt from Bastion.Curator import BLONDE, Snap @@ -19,6 +20,7 @@ def __init__(self, asset, tag, packaged, tarp): assert isinstance(asset, isAsset), "asset must be an instance of Bastion.Model.isAsset" assert isinstance(tag, pathlib.PurePosixPath), "tag must be pathlib.PurePosixPath" assert isinstance(tarp, pathlib.PosixPath), "tarp must be pathlib.PosixPath" + assert isinstance(packaged, hasCanonicalForm), "package must have canonical form (e.g. toJDN)" self.asset = asset #-- the asset that was packed self.tag = tag #-- the full name of the packed object @@ -30,7 +32,7 @@ def toJDN(self): jdn = { 'asset': str(self.asset.CURIE), 'tag': str(self.tag), - 'packaged': list(self.packaged), + 'packaged': toJDN(self.packaged), 'blonde': str(self.blonde), 'tarp': str(self.tarp) } diff --git a/lib/Bastion/Packers/TARs.py b/lib/Bastion/Packers/TARs.py index 64c3ebe..acf0a92 100644 --- a/lib/Bastion/Packers/TARs.py +++ b/lib/Bastion/Packers/TARs.py @@ -6,10 +6,11 @@ import datetime import pathlib import tarfile +import operator import logging +from Bastion.Common import asPath, prefer, hasCanonicalForm from Bastion.Chronology import Quantim -from Bastion.Common import asPath, prefer from Bastion.Model import isAsset, isPacker from Bastion.CARP import isRequest, Report from Bastion.Packers.CARP import PackRequest, PackingReceipt @@ -35,14 +36,17 @@ class ifFileChanged: """ def __init__(self, when): self.whence = when + self.tally = 0 def __call__(self, tarinfo): if tarinfo.type not in (tarfile.REGTYPE, tarfile.AREGTYPE): + self.tally += 1 return tarinfo then = datetime.datetime.fromtimestamp(tarinfo.mtime) logger.debug("comparing file {} mod'd at {} to change limit at {}".format(tarinfo.name, then.isoformat(), self.whence.isoformat())) if then > self.whence: + self.tally += 1 return tarinfo else: #-- returning None is rejecting this file for inclusion @@ -58,6 +62,7 @@ def pax(tarp, folder, **kwargs): {tarp} is the halo (path) to the local file where the tar will be built. {asset} can be a file path or an instance of Bastion.Model.isAsset (e.g. Bastion.Site.Asset) differential backups can be done by supplying the "since" keyword set the datetime which is the earliest allowed modification time for files to be admitted into the tar. + I answer the tally of files added to the archive tar. """ #-- If tar runs into any problems, it should throw an exception. with tarfile.open(tarp, "w", format = tarfile.PAX_FORMAT) as tar: @@ -156,16 +161,68 @@ def unpack(self, subject, *args, **kwargs): raise NotImplementedError(".unpack is subclass responsibility") +class TarCatalogEntry(tuple, hasCanonicalForm): + def __new__(cls, info): + if isinstance(info, TarCatalogEntry): + return info + else: + #-- info is a tarfile.TarInfo object. + return tuple.__new__(cls, (pathlib.PurePosixPath(info.name),info.mtime, info.size)) + + path = property(operator.itemgetter(0)) + mtime = property(operator.itemgetter(1)) + size = property(operator.itemgetter(2)) + + @property + def mq(self): + if not hasattr(self, "_mq"): + self._mq = Quantim.fromtimestamp(self.mtime).quaver + return self._mq + + def toJDN(self, **kwargs): + return {'path': str(self.path), 'modified': str(self.mq), 'size': self.size} + + +class TarCatalog(hasCanonicalForm): + def __init__(self, entries): + #-- entries is a collection of tarfile.TarInfo objects + self.entries = tuple([TarCatalogEntry(entry) for entry in entries]) + + def __iter__(self): + return iter(self.entries) + + def __getitem__(self, i): + return self.entries[i] + + def __len__(self): + return len(self.entries) + + def paths(self): + return [entry.path for entry in self] + + def toJDN(self, **kwargs): + jdn = {} + jdn['entries'] = [entry.toJDN(**kwargs) for entry in self] + return jdn + + @property + def size(self): + return sum([entry.size for entry in self]) + + @classmethod + def ofTarFile(cls, tarp): + """ + I answer the catalog of a tar file at the given path, tarp. + The catalog is a list of entries of the form (size, modified, path) + where size is in bytes, modified is the quantim of the file's mod timestamp, and path is a pathlib.Path object + """ + with tarfile.open(tarp) as tark: + cat = cls(tark.getmembers()) + return cat + + def catalog(tarp): """ - I answer the catalog of a tar file at the given path, tarp. - The catalog is a list of entries of the form (size, modified, path) - where size is in bytes, modified is the quantim of the file's mod timestamp, and path is a pathlib.Path object + I answer the catalog of the tar file at the given path, tarp. """ - cats = [ ] - with tarfile.open(tarp) as tark: - for info in tark.getmembers(): - sz = info.size - mq = Quantim.fromtimestamp(info.mtime).quaver - cats.append( [sz, mq, info.name] ) - return cats + return TarCatalog.ofTarFile(tarp) diff --git a/lib/Bastion/Site.py b/lib/Bastion/Site.py index 2cbf723..510d2a5 100644 --- a/lib/Bastion/Site.py +++ b/lib/Bastion/Site.py @@ -442,6 +442,13 @@ def __init__(self, site, name, root = None): self.policy = self.site.policy #--inherit my site's default policy self.dynamic = False #-- when True, I should automatically enroll subfolders as assets. + @property + def isDynamic(self): + """ + Answers True if this zone's asset list can change and False if this zone's asset list is immutable. + """ + return self.dynamic + @property def assets(self): return self.site.assets(self.name)