From b7f6f4429d4db68f60fea946aa4c7fe83985f4a7 Mon Sep 17 00:00:00 2001 From: Nathan Denny Date: Wed, 5 Mar 2025 08:33:46 -0500 Subject: [PATCH] more debugging, particularly in SFTP areas --- bin/bastion.py | 2 +- lib/Bastion/CARP.py | 2 +- lib/Bastion/Clerks/SFTP.py | 3 ++- lib/Bastion/Movers/CARP.py | 43 +++++++++++++++++++++++++++++++++- lib/Bastion/Movers/SFTP.py | 23 ++++++++++++------- lib/Bastion/NetOps/sCURL.py | 46 +++++++++++++++++++++++++++++++------ lib/Bastion/Vaults/SFTP.py | 4 ++++ 7 files changed, 104 insertions(+), 19 deletions(-) diff --git a/bin/bastion.py b/bin/bastion.py index 404123c..52e3fc3 100755 --- a/bin/bastion.py +++ b/bin/bastion.py @@ -450,7 +450,7 @@ def do_amend_asset(self, request): site = self.site(ark) asset = site.asset(ark) vault = self.vault(asset.policy.vault) - result = vault.ammend(asset) + result = vault.amend(asset) request['log.scope'] = "site/{}".format(ark.site) diff --git a/lib/Bastion/CARP.py b/lib/Bastion/CARP.py index 5728a90..45cf2e4 100644 --- a/lib/Bastion/CARP.py +++ b/lib/Bastion/CARP.py @@ -394,7 +394,7 @@ class isWorkflowReceipt(isReceipt, isTagged): """ def __init__(self, *args): self.results = [ ] - self._tags = { } + self._tags = { } for result in args: self.append(result) diff --git a/lib/Bastion/Clerks/SFTP.py b/lib/Bastion/Clerks/SFTP.py index 42c62f8..b24fbd9 100644 --- a/lib/Bastion/Clerks/SFTP.py +++ b/lib/Bastion/Clerks/SFTP.py @@ -76,9 +76,10 @@ def manifest(self, *args): blondes = [ ] cell = self.scurler / ark.site / ark.zone / ark.asset print("CELL:", cell) + print(type(cell)) if cell.exists: for alien in cell.lsall(): - print(alien) + print("ALIEN: ", alien) if not alien.is_dir(): blondes.append( BLONDE.decode(alien.stem) ) manifest = Manifest(ark, blondes) diff --git a/lib/Bastion/Movers/CARP.py b/lib/Bastion/Movers/CARP.py index 94bbdee..3f5b121 100644 --- a/lib/Bastion/Movers/CARP.py +++ b/lib/Bastion/Movers/CARP.py @@ -8,11 +8,52 @@ from Bastion.Common import Thing, toJDN from Bastion.CARP import isReceipt, isRequest +from Bastion.Model import isAsset, ARK logger = logging.getLogger(__name__) +class ProvisionRequest(isRequest): + def __init__(self, *args, **kwargs): + ark = None + if len(args) == 1: + arg = args[0] + if isinstance(arg, ProvisionRequest): + self.ark = arg.ARK + elif isinstance(arg, isRequest): + self.ark = arg['ARK'] + elif isinstance(arg, isAsset): + ark = arg.ARK + elif isinstance(arg, ARK): + ark = arg + elif len(args) == 3: + ark = ARK(*args) + else: + raise Exception("ProvisionRequest cannot be constructed") + + xtras = { + 'asset': ark + } + isRequest.__init__(self, "ProvisionRequest", **xtras) + + @property + def ARK(self): + return self.context['asset'] + + +class ProvisionReceipt(isReceipt): + def __init__(self, log): + if isinstance(log, str): + self.logged = [log] + else: + self.logged = list(log) + + def toJDN(self, **kwargs): + jdn = {'logged': [toJDN(item) for item in self.logged]} + return jdn + + class PutReceipt(isReceipt): def __init__(self, halo, rendpoint, tag): assert isinstance(halo, pathlib.Path), "halo must be an instance of Path" @@ -44,7 +85,7 @@ def toJDN(self, **kwargs): jdn = { 'source': { 'endpoint': toJDN(self.target.endpoint), - 'tag': toJDN(self.target.tag) + 'tag': toJDN(self.target.tag) }, 'target': { 'path': str(self.local) diff --git a/lib/Bastion/Movers/SFTP.py b/lib/Bastion/Movers/SFTP.py index 459579a..6e01c2e 100644 --- a/lib/Bastion/Movers/SFTP.py +++ b/lib/Bastion/Movers/SFTP.py @@ -1,10 +1,10 @@ import pathlib import logging -import Bastion.Model +import Bastion.Model from Bastion.Common import asPath, asPurePath from Bastion.NetOps.sCURL import SCURLer -from Bastion.Movers.CARP import PutRequest, PutReceipt +from Bastion.Movers.CARP import PutRequest, PutReceipt, ProvisionRequest, ProvisionReceipt logger = logging.getLogger(__name__) @@ -12,21 +12,28 @@ class Mover(Bastion.Model.isMover): def __init__(self, vault, **kwargs): Bastion.Model.isMover.__init__(self, vault) - self.scurler = SCURLer(self.vault.sfURL, keyfile = self.vault.keypath) + self.server = self.vault.server @property def bank(self): - return self.scurler + return self.server #----------------------------------------- #-- BEGIN Bastion.Model.isMover PROTOCOL | #↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ def provision(self, *args): """ - provision(ark) - ensures that the site, zone, and asset folders exist. - provision(site, zone, asset_name) - an alias for provision(ark) + provision(request:ProvisionRequest) + provision(request:isRequest) + provision(ark:ARK) - ensures that the site, zone, and asset folders exist. + provision(asset:isAsset) + provision(site:str, zone:str, asset_name:str) - an alias for provision(ark) """ - pass + request = ProvisionRequest(*args) + ark = request.ARK + self.server.mkdir(pathlib.PurePosixPath(ark.site) / ark.zone / ark.asset) + #-↑↑↑↑↑↑↑↑↑↑ right now, this will always "succeed" even when it doesn't. + return request.succeeded(ProvisionReceipt(str(ark)), report = "provisioned vault for {}".format(str(ark))) def put(self, halo, tag, **kwargs): here = halo @@ -38,7 +45,7 @@ def put(self, halo, tag, **kwargs): try: #-- execute sCURL operation, here. - copied = self.scurler.put(halo, tag) + copied = self.server.put(halo, tag) except Exception as err: #-- shutil should throw an error if there was a problem copying. report = request.failed(err) diff --git a/lib/Bastion/NetOps/sCURL.py b/lib/Bastion/NetOps/sCURL.py index af1df55..8877a59 100644 --- a/lib/Bastion/NetOps/sCURL.py +++ b/lib/Bastion/NetOps/sCURL.py @@ -68,6 +68,9 @@ def __truediv__(self, subpath): path = self.path / subpath return sfURL(self.host, path, user = self.user, port = self.port) + @property + def server(self): + return sfURL(self.host, "/", user = self.user) class Alien: """ @@ -306,11 +309,12 @@ def __init__(self, *args, **kwargs): #-------------------------------------------------------------- #-- Configure while looking for additional keyword arguments. | #-------------------------------------------------------------- - self.keypath = kwargs.get('keyfile', None) + #self.keypath = kwargs.get('keyfile', None) self.silent = kwargs.get('silent', True) #-- default to silent mode self.verbose = kwargs.get('verbose', False) #-- default to terse output mode self.mkdirs = kwargs.get('mkdirs', True) #-- default to creating missing directories in upload paths. self.CURL = kwargs.get('curl', "/usr/bin/curl") + self.trusted = kwargs.get('trusted', False) self.CURL = pathlib.Path(self.CURL) @@ -324,15 +328,21 @@ def __init__(self, *args, **kwargs): #-- ... with additional flags and arguments. | #------------------------------------------------------------------------- basecom = [str(self.CURL)] - if self.keypath is not None: - basecom.extend(['--key', str(self.keypath)]) + #if self.keypath is not None: + # basecom.extend(['--key', str(self.keypath)]) if self.silent: basecom.append('-s') if self.verbose: basecom.append('-v') + if not self.trusted: + basecom.append('-k') self.basecom = tuple(basecom) + @property + def server(self): + return SCURLer(self.host, self.user) + def URL(self, reqpath = None): return str( self.sfURL(reqpath) ) @@ -456,14 +466,36 @@ def mkdir(self, rpath): NOTE: when uploading files to a remote host, folders are typically automatically created during the upload, making the method somewhat redundant or unnecessary for upload operations. """ - url = self.sfURL(rpath) + rpath = pathlib.PurePosixPath(rpath) + myURL = self.sfURL().server + + mkcom = list(self.basecom) + mkcom.append( str(myURL) ) + + prepath = self.root + + for part in rpath.parts: + prepath = prepath / part + mkcom.append("-Q") + mkcom.append("*mkdir {}".format(str(prepath))) + + logger.debug("{}".format(mkcom)) + p = subprocess.run(mkcom, capture_output = True) + self.lastop = p + if p.returncode == 0: + return True + else: + return False + + def quote(self, cmd, *args): + recom = ' '.join( [cmd] + list(args) ) mkcom = list(self.basecom) - mkcom.append(str(url)) + mkcom.append(str(self.sfURL().server)) mkcom.append('-Q') - mkcom.append('mkdir {}'.format(str(url.path))) - logger.debug("mkdir {}".format(str(url.path))) + mkcom.append('{}'.format(recom)) + logger.debug("{}".format(mkcom)) p = subprocess.run(mkcom, capture_output = True) self.lastop = p if p.returncode == 0: diff --git a/lib/Bastion/Vaults/SFTP.py b/lib/Bastion/Vaults/SFTP.py index b4e68f4..ac4d139 100644 --- a/lib/Bastion/Vaults/SFTP.py +++ b/lib/Bastion/Vaults/SFTP.py @@ -54,6 +54,10 @@ def sfURL(self): """ return Bastion.NetOps.sCURL.sfURL(self.host, self.root, user = self.login, port = self.port) + @property + def server(self): + return Bastion.NetOps.sCURL.SCURLer(self.sfURL) + @property def mover(self): if getattr(self, '_mover', None) is None: