From 8e5ec090337adaf99b618f07cefe2403282622c2 Mon Sep 17 00:00:00 2001 From: Nathan Denny Date: Fri, 12 Jul 2024 16:15:10 -0400 Subject: [PATCH] intermediate code push (for reference point) WIP. --- bin/bastion.py | 32 ++++++++----- environment.yml | 2 +- lib/Bastion/Chronology.py | 2 +- lib/Bastion/HPSS.py | 97 +++++++++++++++++++++++++++++++++------ lib/Bastion/Model.py | 10 ++++ 5 files changed, 116 insertions(+), 27 deletions(-) diff --git a/bin/bastion.py b/bin/bastion.py index ec50cbb..ddf3a85 100755 --- a/bin/bastion.py +++ b/bin/bastion.py @@ -119,13 +119,8 @@ def vault(self, name, site = None): name = name[1:-1] if name in self.conf['vaults']: protocol = self.conf['vaults'][name]['protocol'] - if protocol == 'HPSS': - opts = { } - if site is not None: - opts['client'] = site.host - return Bastion.HPSS.Vault(name, **opts).configured(self.conf) - else: - raise NotImplementedError + cls = Bastion.Model.Vault.handling(protocol) + return cls(name).configured(self.conf) else: return None @@ -209,11 +204,11 @@ def do_update_asset(self, comargs, comdex): site = self.site(ark.site) asset = site.asset(ark) vault = self.vault(asset.policy.vault) - flag, stdout, stderr = vault.push(asset, detail = ) + flag, stdout, stderr = vault.push(asset, detail = 'D') if flag: - return SUCCESS(stdout, {'stdout': stdout}) + return SUCCESS(stdout, {'stdout': stdout, 'stderr': stderr}) else: - return FAILED(stdout, {'stdout': stdout}) + return FAILED(stdout, {'stdout': stdout, 'stderr': stderr}) def do_backup_asset(self, comargs, comdex): """ @@ -226,9 +221,9 @@ def do_backup_asset(self, comargs, comdex): vault = self.vault(asset.policy.vault) flag, stdout, stderr = vault.push(asset) if flag: - return SUCCESS(stdout, {'stdout': stdout}) + return SUCCESS(stdout, {'stdout': stdout, 'stderr': stderr}) else: - return FAILED(stdout, {'stdout': stdout}) + return FAILED(stdout, {'stdout': stdout, 'stderr': stderr}) #---------------------- #-- export operations | @@ -267,6 +262,19 @@ def do_export_zone_assets(self, comargs, comdex): def do_list_zone_assets(self, comargs, comdex): return self.do_export_zone_assets(comargs, comdex) + def do_refresh_keytab(self, comargs, comdex): + """ + refresh keytab {vault} + * uses ssh+scp to regenerate the private keytab for the named vault. + """ + vault = self.vault(comdex[2]) + vault.refresh_keytab() + flag, stdout, stderr = vault.refresh_keytab() + if flag: + return SUCCESS(stdout, {'stdout': stdout, 'stderr': stderr}) + else: + return FAILED(stdout, {'stdout': stdout, 'stderr': stderr}) + if __name__ == '__main__': diff --git a/environment.yml b/environment.yml index ccf0a00..28ad03e 100644 --- a/environment.yml +++ b/environment.yml @@ -6,4 +6,4 @@ dependencies: - python=3.10 - pyyaml - ruamel.yaml -prefix: /home/ndenny/miniconda3/envs/bastion +prefix: /home/parselmouth/.conda/envs/bastion diff --git a/lib/Bastion/Chronology.py b/lib/Bastion/Chronology.py index 418bbda..e89b6e6 100644 --- a/lib/Bastion/Chronology.py +++ b/lib/Bastion/Chronology.py @@ -76,7 +76,7 @@ def __init__(self, whence, separator = None): self.dY = whence.year - 2000 self.dM = whence.month self.dD = whence.day - self.qM = round( ((whence.hour * 3600) + (whence.minute * 60) + whence.second + (whence.microsecond / 1000000)) / Quantim.QUANTUM ) + self.qM = round( ((whence.hour * 3600) + (whence.minute * 60) + whence.second + (whence.microsecond / 1000000)) / Quantim.QUANTIM ) elif isinstance(whence, Quantim): self.dY = whence.dY diff --git a/lib/Bastion/HPSS.py b/lib/Bastion/HPSS.py index dd6c3eb..b20e559 100644 --- a/lib/Bastion/HPSS.py +++ b/lib/Bastion/HPSS.py @@ -321,17 +321,39 @@ def lsx(self, path = None): class Vault(Bastion.Model.Vault): + PROTOCOL = 'HPSS' + def __init__(self, name, **kwargs): self.name = name self.server = kwargs.get('server', socket.gethostname()) self.login = kwargs.get('login', getpass.getuser()) - self.keytab = pathlib.Path( kwargs.get('keytab', "~/.private/hpss.unix.keytab") ).expanduser() - self.root = kwargs.get('root', None) + self.root = kwargs.get('root', pathlib.PurePosixPath("")) self.hpath = pathlib.Path( kwargs.get('hpath', '/opt/hsi') ) self.xpath = pathlib.Path( kwargs.get('xpath', (self.hpath / 'bin' / 'hsi')) ) self.xhtar = self.hpath / 'bin' / 'htar' self._hsi = None - self.client = kwargs.get('client', socket.gethostname()) + + #-- determining the client can be a little complex... + #-- in order of preference... + #-- (highest) client kwarg given + #-- shell variable HPSS_HOSTNAME is set + #-- guess using socket.gethostname + if 'client' in kwargs: + self.client = kwargs['client'] + elif 'HPSS_HOSTNAME' in os.environ: + self.client = os.environ['HPSS_HOSTNAME'] + else: + self.client = socket.gethostname( ) + + #-- this describes the keytab authentication file ... + #-- ... and how to regenerate the keytab. + self.keytab = Thing() + self.keytab.halo = pathlib.Path( kwargs.get('keytab', "~/.private/hpss.unix.keytab") ).expanduser() + self.keytab.regen = Thing() + self.keytab.regen.host = None + self.keytab.regen.user = getpass.getuser() + self.keytab.regen.key = pathlib.Path("~/.ssh/id_rsa") + self.keytab.regen.command = 'keytab' def configured(self, conf): confkey = "vaults.{}".format(self.name) @@ -341,12 +363,18 @@ def configured(self, conf): self.login = section['login'] if 'root' in section: self.root = pathlib.PurePosixPath( section['root'] ) + if 'key' in section: + self.keytab.halo = section.get('key.path', self.keytab.halo) + self.keytab.regen.host = section.get('key.refresh.ssh.host', self.keytab.regen.host) + self.keytab.regen.user = section.get('key.refresh.ssh.user', self.keytab.regen.user) + self.keytab.regen.key = section.get('key.refresh.ssh.key', self.keytab.regen.key) + self.keytab.regen.command = section.get('key.refresh.ssh.command', self.keytab.regen.command) return self @property def hsi(self): if self._hsi is None: - self._hsi = HSI(xpath = self.xpath, login = self.login, keytab = self.keytab) + self._hsi = HSI(xpath = self.xpath, login = self.login, keytab = self.keytab.halo) return self._hsi #--------------------------------------- @@ -410,7 +438,6 @@ def provision(self, *args): def push(self, asset, **kwargs): detail = kwargs.get('detail', 'F') - localf = asset.path ark = asset.ARK #-- First we must assure that the vault is provisioned for storing this asset. @@ -422,24 +449,42 @@ def push(self, asset, **kwargs): 'zone': ark.zone, 'asset': ark.asset, 'blonde': BLONDE(ark, detail), - 'localf': localf + 'halo': asset.halo, + 'stolo': "" } - exports = { + exportBastion.Model.Vaults = { "HPSS_AUTH_METHOD": "keytab", "HPSS_PRINCIPAL": self.login, - "HPSS_KEYTAB_PATH": str(self.keytab), - "HPSS_HOSTNAME": str(self.client) + "HPSS_KEYTAB_PATH": str(self.keytab.halo), + "HPSS_HOSTNAME": str(self.client) } - comargs = [str(opts['htar']), "-c", "-f", "{site}/{zone}/{asset}/{blonde}.tar".format(**opts), "-v", "-Hverify=1", "{localf}".format(**opts)] - proc = subprocess.run(comargs, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, check = False, env = exports) + #-- save the current working directory. + ogdir = os.getcwd( ) + #-- change working directory to the root of the zone. + os.chdir( asset.zone.root ) + #-- compute stolo (storage location path) + opts['stolo'] = self.root / ark.site / ark.zone / ark.asset / "{blonde}.tar".format(**opts) + #-- performat the htar operation. + comargs = [str(opts['htar']), "-c", "-f", "{stolo}".format(**opts), "-v", "-Hverify=1", "{halo}".format(**opts)] + + logger.info("cwd is {}".format(os.getcwd())) + logger.info(">>> {}".format(" ".join(comargs))) + +# proc = subprocess.run(comargs, stdout = subprocess.PIPE, stderr = subprocess.PIPE, check = False, env = exports) + proc = subprocess.run("/usr/bin/env;{}".format(" ".join(comargs)), shell = True, stdout = subprocess.PIPE, stderr = subprocess.PIPE, check = False, env = exports) stdout = proc.stdout.decode('utf-8') -# stderr = proc.stderr.decode('utf-8') - stderr = stdout + stderr = proc.stderr.decode('utf-8') + #stderr = stdout for line in stdout.split('\n'): logger.info(line) flag = True if (proc.returncode == 0) else False + + #-- change back to the original working directory. + os.chdir( ogdir ) + + #-- answer our caller with some details of the operation. return (flag, stdout, stderr) #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ @@ -466,5 +511,31 @@ def _provision_ark(self, ark): def _provision_site_zone_asset(self, site, zone, asset_name): return self._provision_ark( ARK(site, zone, asset_name) ) + def refresh_keytab(self): + raise NotImplementedError + + + +class Fortress: + PROTOCOL = 'rcac.purdue.edu' + + def refresh_keytab(self): + """ + Use ssh+scp to regenerate the authenticating keytab file. + """ + opts = { + 'secret': self.keytab.regen.key, + 'login': self.keytab.regen.user, + 'generator': self.keytab.regen.host, + 'action': self.keytab.regen.command, + 'halo': self.keytab.hoflo + } + regencmd = "ssh -i {secret} {login}@{generator} {action}".format(**opts) + proc = subprocess.run(comargs, stdout = subprocess.PIPE, stderr = subprocess.STDOUT, check = False, env = exports) + getcmd = "scp -i {secret} {login}@{generator}:~/.private/hpss.unix.keytab {halo}".format(**opts) + + +#fortress = Vault.for_protocol("rcac.purdue.edu").configured(app.conf) + #hsi = HSI("/opt/hsi/bin/hsi", login = "ndenny") diff --git a/lib/Bastion/Model.py b/lib/Bastion/Model.py index 01147f9..2f6d693 100644 --- a/lib/Bastion/Model.py +++ b/lib/Bastion/Model.py @@ -78,6 +78,7 @@ class Vault: """ I am the base class for all storage vaults. """ + PROTOCOLS = { } @property def sites(self): @@ -136,6 +137,15 @@ def download(self, ark, time, lpath): def configured(self, conf): raise NotImplementedError + @staticmethod + def register(cls): + if not isinstance(cls, Vault): + raise Exception("Vault protocol handlers must be instances of Vault") + Vault.PROTOCOLS[cls.PROTOCOL] = cls + + @staticmethod + def for_protocol(protocol): + return Vault.PROTOCOLS[protocol]