diff --git a/bin/bastion.py b/bin/bastion.py index 7074482..a50d5e8 100755 --- a/bin/bastion.py +++ b/bin/bastion.py @@ -6,6 +6,7 @@ import logging import traceback import socket +import json from ruamel.yaml import YAML from ruamel.yaml.scalarstring import PreservedScalarString @@ -22,11 +23,12 @@ sys.path.insert(0, str(LIB_PATH)) -from Bastion.Common import * -from Bastion.Site import Site -from Bastion.Condo import * -from Bastion.Actions import * -from Bastion.Model import ARK +from Bastion.Common import * +from Bastion.Chronology import Quantim +from Bastion.Site import Site +from Bastion.Condo import * +from Bastion.Actions import * +from Bastion.Model import ARK import Bastion.HPSS """ @@ -40,41 +42,35 @@ a. differential if drift < policy b. anchor (full) if drift >= policy """ - -def SUCCESS(doc, obj = None): - return { +def CARP(code, msg, doc, obj = None, **kwargs): + answer = { 'reply': { - 'status': '200', - 'message': 'Ok' + 'status': code, + 'message': msg }, + 'context': {}, 'report': doc, 'data': obj } + if 'context' in kwargs: + for k, v in kwargs['context'].items(): + answer['context'][k] = v -def FAILED(doc, obj = None): - return { - 'reply': { - 'status': '400', - 'message': 'Bad Request' - }, - 'report': doc, - 'data': obj - } + return answer -def CRASHED(doc, obj = None): - return { - 'reply': { - 'status': '500', - 'message': 'Internal Application Error (crash)' - }, - 'report': doc, - 'data': obj - } +def SUCCESS(doc, obj = None, **kwargs): + return CARP('200', 'Ok', doc, obj, **kwargs) + +def FAILED(doc, obj = None, **kwargs): + return CARP('400', 'Bad Request', doc, obj, **kwargs) +def CRASHED(doc, obj = None, **kwargs): + return CARP('500', 'Internal Application Error (crash!)', doc, obj, **kwargs) + class App: CONF_SEARCH_ORDER = [ @@ -101,6 +97,15 @@ def critical(self, msg): def __init__(self): self.conf = Condex() + self.emitters = { + 'YAML': self.emit_YAML, + 'JSON': self.emit_JSON, + 'PROSE': self.emit_PROSE + } + + #-- Generate a unique session name based on the current time and a random string. + self.session = "{}-{}".format( str(Quantim.now()), Boggle() ) + def configured(self): for folder in App.CONF_SEARCH_ORDER: folder = folder.expanduser() @@ -116,6 +121,19 @@ def hostname(self): else: return socket.getfqdn() + @property + def tongue(self): + #-- the configured emission format. + tongue = self.conf.get("bastion.format", "YAML").upper() + if tongue not in ('JSON', "YAML", "PROSE"): + #-- if I can't interpret the spec'd format, fall back to YAML. + tongue = "YAML" + return tongue + + @property + def logroot(self): + return self.conf.get(asPath, "bastion.logging.path", "/var/log/bastion") + def site(self, name): return Site(name).configured(self.conf) @@ -127,131 +145,292 @@ def vault(self, name, site = None): name = name[1:-1] if name in self.conf['vaults']: protocol = self.conf['vaults'][name]['protocol'] - if protocol == 'HPSS': - opts = { } - if site is not None: - opts['client'] = site.host - return Bastion.HPSS.Vault(name, **opts).configured(self.conf) - else: - raise NotImplementedError + cls = Bastion.Model.Vault.handling(protocol) + return cls(name).configured(self.conf) else: return None def run(self): - comargs = sys.argv[1:] - comdex = dict(enumerate(sys.argv[1:])) + #-- scan the command line for options of the form "-{opt}:{value}" + #-- options are removed from the command sequence + opts = { } + comargs = [ ] + for arg in sys.argv[1:]: + if arg[0] == '-': + tokens = arg.split(':') + opt = tokens[0][1:] + if len(tokens) > 1: + val = tokens[1] + else: + val = True + opts[opt] = val + else: + comargs.append(arg) + + #-- create a convenient positional argument index. + comdex = dict(enumerate(comargs)) menu = [ - ("help", self.do_help), - ("backup asset", self.do_backup_asset), - ("update asset", self.do_update_asset), - ("backup zone", self.do_backup_zone), - ("update zone", self.do_update_zone), - ("export zone assets", self.do_export_zone_assets), - ("list zone assets", self.do_list_zone_assets) + ("help", self.do_help), + + ("bank site", + "backup site", self.do_bank_site), + ("bank zone", + "backup zone", self.do_bank_zone), + ("bank asset", + "backup asset", self.do_bank_asset), + + ("amend asset", self.do_amend_asset), + + ("update asset", self.do_update_asset), + ("update zone", self.do_update_zone), + ("update site", self.do_update_site), + + ("export sites provisioned", + "list sites provisioned", self.do_export_sites_provisioned), + ("export zones provisioned", + "list zones provisioned", self.do_export_zones_provisioned), + ("export manifest", + "show manifest", self.do_export_manifest), + + ("refresh keytab", self.do_refresh_keytab), ] + #-- find the action to be performed. + #-- scan through the menu of idioms and associated methods + #-- attempt to match the command line arguments with an idiom action = self.do_help #-- default is to show the help. - for request, method in menu: - tokens = request.split() - if tokens == comargs[:len(tokens)]: - action = method + for entry in menu: + idioms = entry[:-1] + method = entry[-1] + for idiom in idioms: + tokens = idiom.split() + if tokens == comargs[:len(tokens)]: + action = method + + #-- create a process tracking structure. + proc = { + 'task.started': datetime.datetime.now().isoformat(), + 'task.session': opts.get('task.session', self.session) + } + + #-- execute the action within crash guardrails try: - answer = action(comargs, comdex) + answer = action(comargs, comdex, opts) except Exception as err: tb = traceback.format_exception(err) answer = CRASHED( ''.join(tb), tb ) - yaml = YAML() - yaml.default_flow_style = False - answer['report'] = PreservedScalarString(answer['report']) - yaml.dump(answer, sys.stdout) + proc['task.ended'] = datetime.datetime.now().isoformat() + + #-- include the user's request as part of the answered report + answer['request'] = ' '.join(comargs) + + #-- embed the process tracking info as part of the reply context + for k, v in proc.items(): + answer['context'][k] = v + + #-- check for override on the logging scope. + if "log.scope" in opts: + answer['context']['log.scope'] = opts['log.scope'] + + #-- write the answer to the log, if there is a given log.scope + self.record(answer, opts) + + #-- write the answer to STDOUT using the current "tongue" (emission format) + self.emit(answer, opts) + + #-- explicitly exit based on the answer to the request + #-- status codes follow the general "theory of reply codes" + #-- e.g. those used in SMTP, HTTP, etc. if answer['reply']['status'][0] in ('1','2','3'): + #-- codes in 100, 200, 300 blocks indicate success sys.exit(0) else: + #-- codes in 400, 500 (others?) indicate failure sys.exit(1) - #---------------------- - #-- basic operations | - #---------------------- - def do_help(self, comargs, comdex): - raise NotImplementedError - - #---------------------- - #-- backup operations | - #---------------------- - def do_backup_site(self, comargs, comdex): + def record(self, answer, opts): """ - backup site {site} - * creates a full backup for each asset in site. + Given an answer structure and opts (as a dict), + I serialize the answer to YAML and save the answer as a file in the logs. + I need answer to include "log.scope" in the context block. + If no "log.scope" is declared, I silently ignore the request to record. """ - raise NotImplementedError + if 'log.scope' in answer['context']: + #-- only write a log file if we have an explicit log.scope in the answer's context block. + session = answer['context']['task.session'] + halo = self.logroot / answer['context']['log.scope'] / "{}.yaml".format(session) + halo.parent.mkdir(parents = True, exist_ok = True) + with open(halo, 'wt') as fout: + self.emit_YAML(answer, opts, fout) + + def emit(self, answer, opts, ostream = None): + ostream = ostream if (ostream is not None) else sys.stdout + + if 'emit' in opts: + form = opts['emit'].upper() + else: + form = self.tongue - def do_update_site(self, comargs, comdex): - """ - update site {site} - * creates a differential backup for each asset in {site} - """ - raise NotImplementedErro + do_emit = self.emitters.get(form, self.emitters['YAML']) + return do_emit(answer, opts, ostream) + + def emit_YAML(self, answer, opts, ostream): + yaml = YAML() + yaml.default_flow_style = False + answer['report'] = PreservedScalarString(answer['report']) + yaml.dump(answer, ostream) + + def emit_JSON(self, answer, opts, ostream): + json.dump(answer, ostream, sort_keys = True, indent = 3) + + def emit_PROSE(self, answer, opts, ostream): + ostream.write("request: {request}\n".format(**answer)) + ostream.write("reply: {status} {message}\n".format(**answer['reply'])) + ostream.write("----\n") + ostream.write(answer['report']) + ostream.write("\n") - def do_backup_zone(self, comargs, comdex): + #---------------------- + #-- basic operations | + #---------------------- + def do_help(self, comargs, comdex, opts): + #-- generate a document by scanning all of my "do_*" methods. + menu = [ ] + for attr in dir(self): + if attr.startswith("do_"): + if callable(getattr(self, attr)): + menu.append(attr) + + menu = sorted(menu) + + stanzas = [ ] + for attr in menu: + action = getattr(self, attr) + about = action.__doc__ if (action.__doc__ is not None) else "" + about = about.strip() + if about: + stanza = [ ] + lines = about.split('\n') + for line in lines: + line = line.strip() + if line.startswith("* "): + stanza.append("\t{}".format(line)) + else: + stanza.append(line) + stanzas.append( '\n'.join(stanza) ) + + doc = '\n----\n'.join(stanzas) + + return SUCCESS(doc) + + #-------------------------------------------------------- + #-- BEGIN bank (backup) operations | + #-- all "bank" operations create full (level 0) backups | + #↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ + def do_bank_site(self, comargs, comdex, opts): """ - backup zone {zone} - * creates a full backup for each asset in {zone}. + bank site {site} + * creates a full backup for each asset in site. """ raise NotImplementedError - def do_update_zone(self, comargs, comdex): + def do_bank_zone(self, comargs, comdex, opts): """ - update zone {zone} - * creates a differential backup of each asset in {zone} + bank zone {site} {zone} + bank zone {ARK} + * creates a full backup for each asset in the given zone + * zone can be given as two arguments (site, zone) + * zone can be given as a single argument in ARK format """ - raise NotImplementedError - def do_update_asset(self, comargs, comdex): + def do_bank_asset(self, comargs, comdex, opts): """ - update asset {ARK} - * creates a differential backup of {ARK} + backup asset {ARK} + * creates a full backup of {ARK} """ ark = ARK(comdex[2]) site = self.site(ark.site) asset = site.asset(ark) vault = self.vault(asset.policy.vault) - flag, stdout, stderr = vault.push(asset, detail = 'D', client = self.hostname) + flag, stdout, stderr = vault.push(asset, client = self.hostname) if flag: - return SUCCESS(stdout, {'stdout': stdout}) + return SUCCESS(stdout, {'stdout': stdout, 'stderr': stderr}) else: - return FAILED(stdout, {'stdout': stdout}) + return FAILED(stdout, {'stdout': stdout, 'stderr': stderr}) + + #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ + #-- END bank (backup) operations | + #----------------------------------- - def do_backup_asset(self, comargs, comdex): + #----------------------------------------------------------------- + #-- BEGIN amend (differential) operations | + #-- all "amend" operations create differential (level 1) backups | + #↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ + def do_amend_asset(self, comargs, comdex, opts): """ - backup asset {ARK} - * creates a full backup of {ARK} + amend asset {ARK} + * creates a differential backup of {ARK} """ ark = ARK(comdex[2]) site = self.site(ark.site) asset = site.asset(ark) vault = self.vault(asset.policy.vault) - flag, stdout, stderr = vault.push(asset, client = self.hostname) + flag, stdout, stderr = vault.push(asset, detail = 'D', client = self.hostname) + flag, stdout, stderr = vault.push(asset, detail = 'D') if flag: - return SUCCESS(stdout, {'stdout': stdout}) + return SUCCESS(stdout, {'stdout': stdout, 'stderr': stderr}) else: - return FAILED(stdout, {'stdout': stdout}) + return FAILED(stdout, {'stdout': stdout, 'stderr': stderr}) - #---------------------- - #-- export operations | - #---------------------- - def do_export_site_list(self, comargs, comdex): - name = comdex[3] - vault = self.vault(name) - sitels = list(vault.sites) - report = { - 'vault': name, - 'sites': list(vault.sites) - } - return SUCCESS(report) + #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ + #-- END amend (differential) operations | + #------------------------------------------ + + #------------------------------------------------------------------ + #-- BEGIN update (automatic) operations | + #-- all "update" operations use the subject's policy and manifest | + #-- to determine the level of backup to be performed. | + #-- This is the typical case for a scheduled (e.g. cron) job | + #-- to automatically create full and differential backups | + #↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ + def do_update_site(self, comargs, comdex, opts): + """ + update site {site} + * creates a differential backup for each asset in {site} + """ + raise NotImplementedError + + def do_update_zone(self, comargs, comdex): + """ + update zone {zone} + * creates a differential backup of each asset in {zone} + """ + raise NotImplementedError - def do_export_zone_assets(self, comargs, comdex): + def do_update_asset(self, comargs, comdex): + """ + update asset {ARK} + * asset given in ARK format + * determines backup level based on asset's policy and current manifest + * performs backup based on determined level + * typically used to "automatically" do updates in a scheduled (cron) job + """ + raise NotImplementedError + + #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ + #-- END update (automatic) operations | + #------------------------------------------ + + + #---------------------------------------------- + #-- BEGIN "export" operations | + #-- "export" ops are informational only | + #-- none of these ops will modify any storage | + #-- aka "list" or "show" | + #↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ + def do_export_zone_assets(self, comargs, comdex, opts): """ export zone assets {ark} """ @@ -272,39 +451,136 @@ def do_export_zone_assets(self, comargs, comdex): return SUCCESS(report.strip(), assets) - def do_list_zone_assets(self, comargs, comdex): - return self.do_export_zone_assets(comargs, comdex) + def do_export_manifest(self, comargs, comdex, opts): + """ + export manifest {ARK} + export manifest {ARK} {vault} + * lists all banked objects for the given asset + * asset must be given in ARK format + * vault is assumed from the asset's policy + * vault can be explicitly named to override policy + """ + #-- ARK of interest? + ark = ARK(comdex[2]) + site = self.site(ark.site) + asset = site.asset(ark) + + #-- were we given an explicit vault? + there = self.comdex.get(3, None) + + #-- if we were given a specific vault, use that, otherwise look at the policy for the asset. + vault = self.vault(there) if (there is not None) else self.vault(asset.policy.vault) + + manifest = vault.manifest(ark) + + spool = ["# manifest for {} banked in {}".format(str(ark), vault.name)] + for item in manifest: + spool.append("* {}".format(str(item))) + report = '\n'.join(spool) + + return SUCCESS(report, manifest.toJDN()) + def do_export_sites_provisioned(self, comargs, comdex, opts): + """ + export sites provisioned {vault} + * lists all sites that are provisioned in the given vault + """ + vault = self.vault(comdex[3]) + sites = list(vault.sites) + + spool = ["# sites banked in {}".format(vault.name)] + for site in sites: + spool.append("* {}".format(site)) + report = '\n'.join(spool) + data = {vault.name: sites} - def do_refresh_keytab(self, comargs, comdex): + return SUCCESS(report, data) + + def do_export_zones_provisioned(self, comargs, comdex, opts): + """ + export zones provisioned {vault} {site} + * lists all zones provisioned in the given vault for the named site + """ + vault = self.vault(comdex[3]) + site = comdex.get(4, None) + + if site is None: + return FAILED("must specify a site provisioned in {}".format(vault.name)) + if site not in vault.sites: + return FAILED("site {} is not known in {}".format(site, vault.name)) + + zones = vault.zones(site) + + spool = ["# {} zones banked in {}".format(site, vault.name)] + if zones: + for zone in zones: + spool.append("* {}".format(zone)) + else: + spool.append("no zones are currently banked") + report = "\n".join(spool) + data = {vault.name: {site: list(zones)}} + + return SUCCESS(report, data) + + #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ + #-- END "export" operations | + #-------------------------------- + + def do_refresh_keytab(self, comargs, comdex, opts): """ refresh keytab {vault} * uses ssh+scp to regenerate the private keytab for the named vault. """ - vault = self.vault(comdex[2]) - vault.refresh_keytab() + subject = comdex[2] + vault = self.vault(subject) + + extras = { + 'log.scope': "vault/{}".format(subject) + } + + if not vault: + #-- We didn't get a reference to an actual vault. + #-- The vault requested isn't declared. + #-- FAIL! + extras['log.scope'] = "vault" + return FAILED("vault {} is not declared".format(subject), context = extras) + + + if not callable( getattr(vault, 'refresh_keytab', None) ): + #-- FAIL immediately since this vault doesn't declare + #-- a method to refresh keytabs ... probably because the + #-- vault doesn't use keytab files. + return FAILED("vault {} doesn't support keytab refresh".format(), context = extras) + flag, stdout, stderr = vault.refresh_keytab() if flag: - return SUCCESS(stdout, {'stdout': stdout, 'stderr': stderr}) + return SUCCESS(stdout, {'stdout': stdout, 'stderr': stderr}, context = extras) else: - return FAILED(stdout, {'stdout': stdout, 'stderr': stderr}) - + return FAILED(stdout, {'stdout': stdout, 'stderr': stderr}, context = extras) if __name__ == '__main__': app = App().configured() - if sys.argv[1] != 'shell': + if sys.argv[1:] != ['shell']: app.run( ) +#-- does a full, level 0 backup #bastion backup site {site} #bastion backup zone {ark} #bastion backup asset {ark} +#bastion bank site {site} +#bastion bank zone {ark} +#bastion bank asset {ark} +#-- does either level 0 or level 1 backup based on policy and what is already banked #bastion update site {site} #bastion update zone {ark} #bastion update asset {ark} +#-- does ONLY a differential backup of ark +#bastion amend asset {ark} + #bastion refresh keytab {vault} #bastion restore zone {ark} @@ -318,3 +594,13 @@ def do_refresh_keytab(self, comargs, comdex): #bastion [list|export] anchors {ark} #bastion [list|export] snaps {ark} + +#bastion [list|export] sites banked {vault} +#bastion [list|export] zones banked {vault} {site} +#bastion [list|export] assets banked {vault} {ark} + +#bastion [show|export] manifest {ark} [{vault}] + +#bastion JSON! list sites banked fortress +#bastion YAML! list sites banked fortress +#bastion PROSE! list sites banked fortress diff --git a/environment.yml b/environment.yml index ccf0a00..28ad03e 100644 --- a/environment.yml +++ b/environment.yml @@ -6,4 +6,4 @@ dependencies: - python=3.10 - pyyaml - ruamel.yaml -prefix: /home/ndenny/miniconda3/envs/bastion +prefix: /home/parselmouth/.conda/envs/bastion diff --git a/lib/Bastion/Chronology.py b/lib/Bastion/Chronology.py index 418bbda..d8cea91 100644 --- a/lib/Bastion/Chronology.py +++ b/lib/Bastion/Chronology.py @@ -76,7 +76,7 @@ def __init__(self, whence, separator = None): self.dY = whence.year - 2000 self.dM = whence.month self.dD = whence.day - self.qM = round( ((whence.hour * 3600) + (whence.minute * 60) + whence.second + (whence.microsecond / 1000000)) / Quantim.QUANTUM ) + self.qM = round( ((whence.hour * 3600) + (whence.minute * 60) + whence.second + (whence.microsecond / 1000000)) / Quantim.QUANTIM ) elif isinstance(whence, Quantim): self.dY = whence.dY @@ -145,6 +145,9 @@ def quaver(self): return self.separator.join([sY, sM, sD, sQ]) def datetime(self): + """ + answers a python datetime.datetime object that is the midpoint of this quantum + """ if self._when is not None: return self._when else: @@ -152,6 +155,15 @@ def datetime(self): elapsed_seconds = ((self.qM * Quantim.QUANTIM) + (Quantim.QUANTIM / 2)) * SECONDS return (y + elapsed_seconds) + def earliest(self): + """ + answers a python datetime.datetime object that is the earliest time within the range of this quantum + """ + y = datetime.datetime(self.dY + 2000, self.dM, self.dD, 0, 0, 0) + elapsed_seconds = self.qM * Quantim.QUANTIM + return (y + elapsed_seconds) + + @classmethod def now(cls): return cls(datetime.datetime.now()) diff --git a/lib/Bastion/HPSS.py b/lib/Bastion/HPSS.py index 94d58e2..2a0cc09 100644 --- a/lib/Bastion/HPSS.py +++ b/lib/Bastion/HPSS.py @@ -321,17 +321,40 @@ def lsx(self, path = None): class Vault(Bastion.Model.Vault): + PROTOCOL = 'HPSS' + def __init__(self, name, **kwargs): - self.name = name + Bastion.Model.Vault.__init__(self, name, **kwargs) + self.server = kwargs.get('server', socket.gethostname()) self.login = kwargs.get('login', getpass.getuser()) - self.keytab = pathlib.Path( kwargs.get('keytab', "~/.private/hpss.unix.keytab") ).expanduser() - self.root = kwargs.get('root', None) + self.root = kwargs.get('root', pathlib.PurePosixPath("")) self.hpath = pathlib.Path( kwargs.get('hpath', '/opt/hsi') ) self.xpath = pathlib.Path( kwargs.get('xpath', (self.hpath / 'bin' / 'hsi')) ) self.xhtar = self.hpath / 'bin' / 'htar' self._hsi = None - self.client = kwargs.get('client', socket.gethostname()) + + #-- determining the client can be a little complex... + #-- in order of preference... + #-- 1. (highest) client kwarg given, or ... + #-- 2. shell variable HPSS_HOSTNAME is set, or finally ... + #-- 3. guess using socket.gethostname + if 'client' in kwargs: + self.client = kwargs['client'] + elif 'HPSS_HOSTNAME' in os.environ: + self.client = os.environ['HPSS_HOSTNAME'] + else: + self.client = socket.gethostname( ) + + #-- this describes the keytab authentication file ... + #-- ... and how to regenerate the keytab. + self.keytab = Thing() + self.keytab.halo = pathlib.Path( kwargs.get('keytab', "~/.private/hpss.unix.keytab") ).expanduser() + self.keytab.regen = Thing() + self.keytab.regen.host = None + self.keytab.regen.user = getpass.getuser() + self.keytab.regen.key = pathlib.Path("~/.ssh/id_rsa") + self.keytab.regen.command = 'keytab' self.keytab = Thing() self.keytab.halo = pathlib.Path( kwargs.get('keytab', "~/.private/hpss.unix.keytab") ).expanduser() @@ -360,7 +383,7 @@ def configured(self, conf): @property def hsi(self): if self._hsi is None: - self._hsi = HSI(xpath = self.xpath, login = self.login, keytab = self.keytab) + self._hsi = HSI(xpath = self.xpath, login = self.login, keytab = self.keytab.halo) return self._hsi #--------------------------------------- @@ -424,7 +447,6 @@ def provision(self, *args): def push(self, asset, **kwargs): detail = kwargs.get('detail', 'F') - localf = asset.path ark = asset.ARK #-- First we must assure that the vault is provisioned for storing this asset. @@ -436,24 +458,41 @@ def push(self, asset, **kwargs): 'zone': ark.zone, 'asset': ark.asset, 'blonde': BLONDE(ark, detail), - 'localf': localf + 'halo': asset.halo, + 'stolo': "" } - exports = { + exportBastion.Model.Vaults = { "HPSS_AUTH_METHOD": "keytab", "HPSS_PRINCIPAL": self.login, - "HPSS_KEYTAB_PATH": str(self.keytab), - "HPSS_HOSTNAME": str(self.client) + "HPSS_KEYTAB_PATH": str(self.keytab.halo), + "HPSS_HOSTNAME": str(self.client) } - comargs = [str(opts['htar']), "-c", "-f", "{site}/{zone}/{asset}/{blonde}.tar".format(**opts), "-v", "-Hverify=1", "{localf}".format(**opts)] - proc = subprocess.run(comargs, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, check = False, env = exports) + #-- save the current working directory. + ogdir = os.getcwd( ) + #-- change working directory to the root of the zone. + os.chdir( asset.zone.root ) + #-- compute stolo (storage location path) + opts['stolo'] = self.root / ark.site / ark.zone / ark.asset / "{blonde}.tar".format(**opts) + #-- performat the htar operation. + comargs = [str(opts['htar']), "-c", "-f", "{stolo}".format(**opts), "-v", "-Hverify=1", "{halo}".format(**opts)] + + logger.info("cwd is {}".format(os.getcwd())) + logger.info(">>> {}".format(" ".join(comargs))) + + proc = subprocess.run("/usr/bin/env;{}".format(" ".join(comargs)), shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE, check = False) stdout = proc.stdout.decode('utf-8') -# stderr = proc.stderr.decode('utf-8') - stderr = stdout + stderr = proc.stderr.decode('utf-8') + #stderr = stdout for line in stdout.split('\n'): logger.info(line) flag = True if (proc.returncode == 0) else False + + #-- change back to the original working directory. + os.chdir( ogdir ) + + #-- answer our caller with some details of the operation. return (flag, stdout, stderr) #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ @@ -473,19 +512,59 @@ def _manifest_ark(self, ark): def _manifest_site_zone_asset(self, site, zone, asset): return self._manifest_ark( ARK(site, zone, asset) ) - def _provision_ark(self, ark): self.hsi.mkdirs(self.root / ark.site / ark.zone / ark.asset) def _provision_site_zone_asset(self, site, zone, asset_name): return self._provision_ark( ARK(site, zone, asset_name) ) + + +class Fortress(Vault): + PROTOCOL = 'HPSS/Purdue/Fortress' + + def __init__(self, name, **kwargs): + Vault.__init__(self, name, **kwargs) + def refresh_keytab(self): """ Use ssh+scp to regenerate the authenticating keytab file. """ - regencmd = "ssh {}@{} {}" - proc = subprocess.run(comargs, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, check = False, env = exports) - + opts = { + 'secret': self.keytab.regen.key, + 'login': self.keytab.regen.user, + 'generator': self.keytab.regen.host, + 'action': self.keytab.regen.command, + 'halo': self.keytab.halo + } + #-- Use the host and action to regenerate the required keytab file. + #cmd_regen = "ssh -i {secret} {login}@{generator} {action}".format(**opts) + cmd_regen = ["ssh", "-i", self.keytab.regen.key, "{login}@{generator}".format(**opts), self.keytab.regen.command] + regen = subprocess.run(cmd_regen, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, check = False, text = True) + logger.debug(regen.stdout) + if regen.returncode != 0: + #-- something went wrong, signal failure. + return (False, regen.stdout, regen.stderr) + + #-- Now that the keytab should have been generated, + #-- we need to use scp to download the keytab. + #cmd_scp = "scp -i {secret} {login}@{generator}:~/.private/hpss.unix.keytab {halo}".format(**opts) + cmd_scp = ["scp", "-i", self.keytab.regen.key, "{login}@{generator}:~/.private/hpss.unix.keytab".format(**opts), self.keytab.halo] + scp = subprocess.run(cmd_scp, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, check = False, text = True) + logger.debug(scp.stdout) + if regen.returncode != 0: + #-- something went wrong, signal failure. + return (False, regen.stdout, regen.stderr) + + #-- If we get to here, it would appear that both the regen and scp commands succeeded. + #-- signal success! + return (True, "keytab regenerated and downloaded to {}".format(self.keytab.halo), "") + + +#-- Register both the base HPSS vault and the Purdue RCAC specific instance. +Vault.register() +Fortress.register() + +#fortress = Vault.for_protocol("HPSS/Purdue/Fortress").configured(app.conf) #hsi = HSI("/opt/hsi/bin/hsi", login = "ndenny") diff --git a/lib/Bastion/Model.py b/lib/Bastion/Model.py index 6cd7bc2..84b283a 100644 --- a/lib/Bastion/Model.py +++ b/lib/Bastion/Model.py @@ -81,6 +81,10 @@ class Vault: """ I am the base class for all storage vaults. """ + PROTOCOLS = { } + + def __init__(self, name, **kwargs): + self.name = name @property def sites(self): @@ -130,15 +134,35 @@ def push(self, asset, **kwargs): def pull(self, blonde, **kwargs): raise NotImplementedError - def upload(self, path, ark): + def put(self, halo, tag): + """ + Given path to a local file (aka Host Asset LOcation), + move the file from the local scope to this vault and store + the object at tag (the path relative to the root of this vault) + """ raise NotImplementedError - def download(self, ark, time, lpath): + def get(self, tag, halo): + """ + Given a tag (the path relative to the root of this vault), + download the object and store it in the local file designated by halo. + """ raise NotImplementedError def configured(self, conf): raise NotImplementedError + @classmethod + def register(cls): + Vault.PROTOCOLS[cls.PROTOCOL] = cls + + @staticmethod + def for_protocol(protocol): + return Vault.PROTOCOLS[protocol] + + @staticmethod + def handling(protocol): + return Vault.PROTOCOLS[protocol]