diff --git a/bin/bastion.py b/bin/bastion.py index 46cfa58..2f91aee 100755 --- a/bin/bastion.py +++ b/bin/bastion.py @@ -29,7 +29,7 @@ from Bastion.Condo import * from Bastion.Actions import * from Bastion.Model import ARK -from Bastion.CARP import Request +from Bastion.CARP import Request, isReceipt import Bastion.Vaults.HPSS import Bastion.Vaults.BFD @@ -134,9 +134,6 @@ def sites(self): yield self.site(nick) def run(self): - #-- generate a session ID. - self.session = "{}{}".format(Quantim.now().quaver, Boggle(5)) - #-- scan the command line for options of the form "-{opt}:{value}" #-- options are removed from the command sequence opts = { } @@ -153,9 +150,6 @@ def run(self): else: comargs.append(arg) - #-- create a convenient positional argument index. - comdex = dict(enumerate(comargs)) - menu = [ ("help", self.do_help), @@ -182,6 +176,13 @@ def run(self): ("refresh keytab", self.do_refresh_keytab), ] + #-- Look for an explicitly declared session ID in opts; + #-- and fall back to the default if no ID is declared. + if 'session.ID' in opts: + self.session = opts['session.ID'] + else: + self.session = "{}{}".format(Quantim.now().quaver, Boggle(5)) + #-- find the action to be performed. #-- scan through the menu of idioms and associated methods #-- attempt to match the command line arguments with an idiom @@ -193,14 +194,16 @@ def run(self): tokens = idiom.split() if tokens == comargs[:len(tokens)]: action = method - request = Request(idiom, comargs[len(tokens):], ID = self.session) + request = Request(idiom, comargs[len(tokens):], ID = self.session, context = opts) #-- execute the action within crash guardrails try: answer = action(request) + if not isinstance(answer, isReceipt): + raise ValueError("actions must respond with an instance of CARP.isReceipt") except Exception as err: tb = traceback.format_exception(err) - answer = CRASHED( ''.join(tb), tb ) + answer = request.crashed( ''.join(tb), tb ) #-- always log crashes! answer.context['log.scope'] = '*' @@ -209,10 +212,10 @@ def run(self): answer.context['log.scope'] = opts['log.scope'] #-- write the answer to the log, if there is a given log.scope - self.record(answer, opts) + self.remember(answer) #-- write the answer to STDOUT using the current "tongue" (emission format) - self.emit(answer, opts) + self.emit(answer) #-- explicitly exit based on the answer to the request #-- status codes follow the general "theory of reply codes" @@ -222,47 +225,47 @@ def run(self): else: sys.exit(1) - def record(self, answer, opts): + def remember(self, answer): """ Given an answer structure and opts (as a dict), I serialize the answer to YAML and save the answer as a file in the logs. I need answer to include "log.scope" in the context block. If no "log.scope" is declared, I silently ignore the request to record. """ - if 'log.scope' in answer.context: + if 'log.scope' in answer.request.context: #-- only write a log file if we have an explicit log.scope in the answer's context block. session = answer.request.ID scope = self.logroot - if answer.context['log.scope'] != '*': - scope = scope / answer.context['log.scope'] + if answer.request.context['log.scope'] != '*': + scope = scope / answer.request.context['log.scope'] halo = scope / "{}.yaml".format(session) halo.parent.mkdir(parents = True, exist_ok = True) with open(halo, 'wt') as fout: - self.emit_YAML(answer, opts, fout) + self.emit_YAML(answer, fout) - def emit(self, answer, opts, ostream = None): + def emit(self, answer, ostream = None): ostream = ostream if (ostream is not None) else sys.stdout - if 'emit' in opts: + if 'emit' in answer.request.context: form = opts['emit'].upper() else: form = self.tongue do_emit = self.emitters.get(form, self.emitters['YAML']) - return do_emit(answer, opts, ostream) + return do_emit(answer, ostream) - def emit_YAML(self, answer, opts, ostream): + def emit_YAML(self, answer, ostream): yaml = YAML() yaml.default_flow_style = False - answer.report = PreservedScalarString(answer.report']) + answer.report = PreservedScalarString(answer.report) yaml.dump(answer, ostream) - def emit_JSON(self, answer, opts, ostream): + def emit_JSON(self, answer, ostream): json.dump(answer, ostream, sort_keys = True, indent = 3) - def emit_PROSE(self, answer, opts, ostream): + def emit_PROSE(self, answer, ostream): request = answer.request requested = ' '.join([request.action] + request.args) ostream.write("request: {}\n".format(requested)) @@ -324,7 +327,35 @@ def do_bank_zone(self, request): * zone can be given as two arguments (site, zone) * zone can be given as a single argument in ARK format """ - raise NotImplementedError + if len(request.args) == 1: + #-- We were given a single ARK argument. + ark = ARK(request.args[0]) + elif len(request.args) == 2: + #-- We were given two arguments (site, zone) + ark = ARK(request.args[0], request.args[1]) + else: + raise ValueError("do_bank_zone expects zone to be given as a CURIE'd ARK") + + site = self.site(ark.site) + zone = site.zone(ark.zone) + + receipts = [ ] + for asset in zone: + vault = self.vault(asset.policy.vault) + receipt = vault.push(asset) + receipts.append(receipt) + + all_succeeded = all(receipt.indicates_success for receipt in receipts]) + all_failed = all(receipt.indicates_failure for receipt in receipts]) + + pushed = [receipt.body['blonde'] for receipt in receipts if receipt.indicates_success] + + if all_succeeded: + return request.succeeded("all {} assets successfully pushed".format(len(receipts)), receipts) + elif all_failed: + return request.failed("push FAILED for ALL {} ASSETS".format(len(receipts)), receipts) + else: + return request.inconclusive("{}/{} assets successfully pushed".format(len(pushed), len(receipts))) def do_bank_asset(self, request): """ @@ -337,16 +368,13 @@ def do_bank_asset(self, request): vault = self.vault(asset.policy.vault) receipt = vault.push(asset, client = self.hostname) - - extras = { - 'log.scope': "site/{}".format(ark.site) - } + request.context['log.scope'] = 'log.scope': "site/{}".format(ark.site) if receipt.indicates_success: blonde = receipt.body['blonde'] - return request.succeeded("pushed full backup of {} to {}".format(str(ark), str(blonde)), receipt, context = extras) + return request.succeeded("pushed full backup of {} to {}".format(str(ark), str(blonde)), receipt) else: - return request.failed("while pushing full backup of {}, something went wrong!".format(str(ark)), receipt, context = extras) + return request.failed("while pushing full backup of {}, something went wrong!".format(str(ark)), receipt) #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ #-- END bank (backup) operations | diff --git a/lib/Bastion/CARP.py b/lib/Bastion/CARP.py index bfebf65..f5b365a 100644 --- a/lib/Bastion/CARP.py +++ b/lib/Bastion/CARP.py @@ -113,9 +113,8 @@ class isRequest: def __init__(self, action, *args, **kwargs): self.ID = kwargs.get('ID', str(uuid.uuid4())) self.action = action - self.args = list(args) self.when = kwargs.get('when', datetime.datetime.now()) - self.argdex = dict(enumerate(args)) + self.args = dict(enumerate(args)) self.context = { } if 'context' in kwargs: @@ -178,7 +177,7 @@ def toJDN(self, **kwargs): 'message': self.status.gloss, 'lede': self.lede, 'answered': self.when.isoformat(), - 'context': {}, + 'context': { }, 'body': payload } #-- Populate the result's context (if any) @@ -224,7 +223,10 @@ def succeeded(self, doc, data = None, **kwargs): return Report(self, ReplyStatus.Ok, doc, data, **kwargs) def failed(self, doc, data = None, **kwargs): - return Report(self, ReplyStatus.Failed, lede, doc, data, **kwargs)) + return Report(self, ReplyStatus.Failed, doc, data, **kwargs)) def crashed(self, doc, data = None, **kwargs): - return Report(self, ReplyStatus.Crashed, lede, doc, data, **kwargs)) + return Report(self, ReplyStatus.Crashed, doc, data, **kwargs)) + + def inconclusive(self, doc, data = None, **kwargs) + return Report(self, ReplyStatus.Inconclusive, doc, data, **kwargs) diff --git a/lib/Bastion/Model.py b/lib/Bastion/Model.py index d360d75..74cbf4b 100644 --- a/lib/Bastion/Model.py +++ b/lib/Bastion/Model.py @@ -24,6 +24,8 @@ def __new__(cls, *args): site = site[1:] zone = arg.path.parts[0] asset = arg.path.relative_to( pathlib.PurePosixPath(zone) ) + if asset == pathlib.PurePosixPath("."): + asset = None return ARK(site, zone, asset) elif isinstance(arg, str): @@ -32,11 +34,17 @@ def __new__(cls, *args): elif isinstance(arg, isAsset): return ARK( RDN(arg.site), RDN(arg.zone), RDN(arg.asset) ) + if len(args) == 2: + #-- site, zone, / + site = args[0] + zone = args[1] + return ARK(site, zone, None) + if len(args) == 3: site, zone, asset = args s = RDN(site) z = RDN(zone) - a = pathlib.PurePosixPath(asset) + a = asset st = s if (s[0] == '@') else "@{}".format(s) return tuple.__new__(cls, [st, z, a]) @@ -62,7 +70,10 @@ def zolo(self): """ zolo ("zone location") is the logical path to this including its zone. """ - return pathlib.PurePosixPath(self.zone) / self.asset + if self.asset: + return pathlib.PurePosixPath(self.zone) / self.asset + else: + return pathlib.PurePosixPath(self.zone) @property def CURIE(self): @@ -76,73 +87,6 @@ def badge(self): return Slug40(str(self.CURIE)) -class isOpReceipt: - def __init__(self, status, message, opts): - self.succeeded = succeeded - self.opts = { } - for k, v in opts.items(): - self.opts[k] = v - - @property - def opts(self): - return set(self.opts.keys()) - - def __getitem__(self, k): - return self.opts[k] - - def __setitem__(self, k, v): - self.opts[k] = v - return self - - - -class TransferReceipt(isOpReceipt): - """ - Generic class for returning detailed information about the transfer (put) operation. - """ - def __init__(self, halo, tag, started, ended, opts): - isOpReceipt.__init__(self, opts) - self.succeeded = False - self.source = halo - self.tag = tag - self.started = started - self.ended = ended - self.message = "" - - def toJDN(self, **kwargs): - dex = { } - dex['source'] = str(self.source) - dex['tag'] = str(self.tag) - dex['started'] = self.started.isoformat() - dex['ended'] = self.ended.isoformat() - dex['opts'] = { } - for k in opts: - dex['opts'][k] = self.opts[k] - return dex - - -class PackingReceipt(isOpReceipt): - """ - Generic class for returning detailed information about the pack operation. - """ - def __init__(self, succeeded, asset, blonde, spooled, opts): - isOpReceipt.__init__(self, succeeded, opts) - - self.asset = asset - self.blonde = blonde - self.spool = spooled - - def toJDN(self, **kwargs): - dex = { } - dex['asset'] = str(self.asset) - dex['blonde'] = str(self.blonde) - dex['spooled'] = str(self.spooled) - dex['opts'] = { } - for opt in self.opts: - dex['opts'][opt] = self.opts[opt] - return dex - - class isVault: """ abstract class for all vaults. diff --git a/lib/Bastion/Vaults/BFD.py b/lib/Bastion/Vaults/BFD.py index 5086014..877b441 100644 --- a/lib/Bastion/Vaults/BFD.py +++ b/lib/Bastion/Vaults/BFD.py @@ -184,18 +184,33 @@ def push(self, asset, basis = None, **kwargs): {asset} - an instance of Bastion.Model.isAsset {basis} - can be a datetime or a BLONDE. """ - blonde, tag, spool, package = self.pack(asset, basis, **kwargs) + request = CARP.Request("{}.push".format(self.PROTOCOL)) + request.args['asset'] = CURIE(asset) + request.args['vault'] = self.name + if basis: + if isinstance(basis, datetime.datetime): + request.args['basis'] = basis.isoformat() + elif isinstance(basis, BLONDE): + request.args['basis'] = str(basis) + else: + raise ValueError('.push takes basis as an instance of BLONDE or datetime') - #-- assure that the bank exists. - (self.bank / tag).parent.mkdir(parents = True, exist_ok = True) + packing_receipt = self.pack(asset, basis, **kwargs) - transferred, receipt = self.put(self.scratch / tag, tag) + if packing_receipt.indicates_success: + #-- assure that the bank exists. + (self.bank / tag).parent.mkdir(parents = True, exist_ok = True) - if transferred: - #-- clean up! - (self.scratch / tag).unlink() + copy_receipt = self.put(self.scratch / tag, tag) + + if copy_receipt.indicates_success: + #-- clean up! + (self.scratch / tag).unlink() + else: + #-- HALT AND CATCH FIRE!!! + #-- WORK HERE WORK HERE> - return (transferred, blonde, receipt) + return receipt def pull(self, ark, **kwargs):