From 1bf509e99ea376521dc28b6450375bcd0162e0d8 Mon Sep 17 00:00:00 2001 From: Nathan Denny Date: Mon, 14 Apr 2025 09:27:27 -0400 Subject: [PATCH] evolving code, particularly with SCURLer ability to execute SFTP "quote" commands --- lib/Bastion/CARP.py | 5 +- lib/Bastion/Model.py | 24 +++-- lib/Bastion/Movers/CARP.py | 37 +++++++ lib/Bastion/NetOps/sCURL.py | 206 ++++++++++++++++++++++-------------- 4 files changed, 184 insertions(+), 88 deletions(-) diff --git a/lib/Bastion/CARP.py b/lib/Bastion/CARP.py index 45cf2e4..082c357 100644 --- a/lib/Bastion/CARP.py +++ b/lib/Bastion/CARP.py @@ -1,7 +1,7 @@ """ Bastion.CARP -Common Action-Result/Report Protocol +Common Action-Request/Result/Report Protocol Basic concepts... requests (instances of isRequest or subclasses) encapsulates details of the action requested. @@ -354,7 +354,7 @@ def toJDN(self, **kwargs): raise NotImplementedError(".toJDN is subclass responsibility") -class isBagReceipt: +class isBagReceipt(isReceipt): """ I am a thin wrapper around a sequence of results. """ @@ -370,7 +370,6 @@ def append(self, result, *args): def __iter__(self): return iter(self.results) - def __getitem__(self, i): return self.results[i] diff --git a/lib/Bastion/Model.py b/lib/Bastion/Model.py index 2b30d85..25a803c 100644 --- a/lib/Bastion/Model.py +++ b/lib/Bastion/Model.py @@ -87,10 +87,10 @@ def badge(self): return Slug40(str(self.CURIE)) -class isActor: +class isPerformer: """ - Abstract class for actors. - Actors are the "do-ers" in the model. + Abstract class for performers. + Performers are the "do-ers" in the model. All action methods should return an instance of Bastion.CARP.Report """ def perform(self, request): @@ -109,24 +109,24 @@ def __init__(self, name, **kwargs): self.name = name #---------------------------------------------------------------------------- - #-- BEGIN ACTOR ACCESSORS | + #-- BEGIN PERFORMER ACCESSORS | #↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ @property def clerk(self): - #-- Answers the clerk for this vault. + #-- Answers the (possibly delegated) clerk performer for this vault. raise NotImplementedError @property def mover(self): - #-- Answers a mover for this vault. + #-- Answers the (possibly delegated) mover performer for this vault. raise NotImplementedError @property def packer(self): - #-- Answers a packer (if needed) for this vault. + #-- Answers the (possibly delegated) packer peformer (if needed) for this vault. raise NotImplementedError #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ - #-- END ACTOR ACCESSORS | + #-- END PERFOMER ACCESSORS | #---------------------------------------------------------------------------- #---------------------------------------------------------------------------- @@ -192,6 +192,14 @@ def get(self, tag, halo, **kwargs): download the object and store it in the local file designated by halo. """ return self.mover.get(tag, halo, **kwargs) + + def remove(self, tag, **kwargs): + """ + Given a tag (the path relative to the root of this vault), + remove the object from the (remote) storage. + """ + return self.mover.remove(tag, **kwargs) + #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ #-- END MOVER DELEGATES | #---------------------------------------------------------------------------- diff --git a/lib/Bastion/Movers/CARP.py b/lib/Bastion/Movers/CARP.py index 3f5b121..39f8b98 100644 --- a/lib/Bastion/Movers/CARP.py +++ b/lib/Bastion/Movers/CARP.py @@ -94,6 +94,22 @@ def toJDN(self, **kwargs): return jdn +class RemoveReceipt(isReceipt): + def __init__(self, rendpoint, tag): + assert isinstance(tag, (pathlib.PurePosixPath, str)), "tag must be an instance of str or PurePosixPath" + self.target = Thing(endpoint = rendpoint, tag = tag) + + def toJDN(self, **kwargs): + jdn = { + 'target': { + 'endpoint': toJDN(self.target.endpoint), + 'tag': toJDN(self.target.tag) + } + } + return jdn + + + class PutRequest(isRequest): """ A request for putting a given file into a target endpoint. @@ -142,3 +158,24 @@ def __init__(self, rendpoint, tag, halo): } isRequest.__init__(self, "GetRequest", **xtras) + + +class RemoveRequest(isRequest): + """ + A request for removing a given object from an endpoint (remote storage). + """ + def __init__(self, rendpoint, tag): + """ + rednpoint - is a string or object that describes the remote endpoint. + tag - the tag that selects the object for removal, this is typically a path relative to the (remote) storage root. + """ + assert isinstance(tag, (pathlib.PurePosixPath, str)), "tag must be an instance of str or PurePosixPath" + + self.bank = Thing(endpoint = rendpoint, tag = tag) + + xtras = { + "bank.endpoint": toJDN(self.bank.endpoint), + "bank.tag": toJDN(self.bank.tag) + } + + isRequest.__init__(self, "RemoveRequest", **xtras) diff --git a/lib/Bastion/NetOps/sCURL.py b/lib/Bastion/NetOps/sCURL.py index f8e905e..47dccbd 100644 --- a/lib/Bastion/NetOps/sCURL.py +++ b/lib/Bastion/NetOps/sCURL.py @@ -8,9 +8,12 @@ import datetime import getpass import operator +import urllib.parse import logging +from Bastion.Common import Unknown + logger = logging.getLogger(__name__) logging.basicConfig(level = logging.DEBUG) @@ -40,23 +43,39 @@ def __new__(cls, *args, **kwargs): #-- Can be called as... #-- sfCURL(instance_of_sfCURL) #-- sfCURL(host, path, **kwargs) - if (len(args) == 1) and isinstance(args[0], cls): - return args[0] + #-- sfCURL("sftp://user@host/path") + arg = args[0] + + if isinstance(arg, cls) and (len(args) == 1): + return arg - host = args[0] + host = None path = pathlib.PurePosixPath("/") port = kwargs.get('port', 22) user = kwargs.get('user', getpass.getuser()) - if len(args) > 1: - path = pathlib.PurePosixPath(args[1]) + if arg.startswith("sftp://"): + conargs = urllib.parse.urlparse(arg) + + #-- Interpret my construction argument as an SFTP URL. + host = conargs.hostname + path = pathlib.PurePosixPath(conargs.path) if conargs.path else pathlib.PurePosixPath("/") + port = conargs.port if (conargs.port is not None) else 22 + user = conargs.username if (conargs.username is not None) else getpass.getuser() + else: + #-- I was just given a hostname and nothing else. + host = arg + if len(args) > 1: + path = pathlib.PurePosixPath(args[1]) return tuple.__new__(cls, (host, port, user, path)) - host = property(operator.itemgetter(0)) - port = property(operator.itemgetter(1)) - user = property(operator.itemgetter(2)) - path = property(operator.itemgetter(3)) + host = property(operator.itemgetter(0)) + hostname = property(operator.itemgetter(0)) + port = property(operator.itemgetter(1)) + user = property(operator.itemgetter(2)) + username = property(operator.itemgetter(2)) + path = property(operator.itemgetter(3)) def __str__(self): if self.port != 22: @@ -72,6 +91,38 @@ def __truediv__(self, subpath): def server(self): return sfURL(self.host, "/", user = self.user) + +class Catalog: + """ + I am a collection of aliens retrieved via an lsall() request. + """ + def __init__(self, sfURL, collected, **kwargs): + self.sfURL = sfURL + self._entries = tuple([asset for asset in collected if isinstance(asset, Alien)]) + self._index = dict([(entry.rpath.name, entry) for entry in self._entries]) + + def root(self): + return self.sfURL.rpath + + def server(self): + return self.sfURL.server + + def __len__(self): + return len(self._entries) + + def __iter__(self): + return iter(self._entries) + + def entries(self): + return tuple(sorted(self._index.keys())) + + def __getitem__(self, nm): + return self._index[nm] + + def __contains__(self, nm): + return (nm in self._index) + + class Alien: """ I am an accessor to a remotely hosted object (i.e. a resource path on a remote host). @@ -118,6 +169,10 @@ def toJDN(self): jdn['mdate'] = jdn['mdate'].isoformat() return jdn + @property + def sfURL(self): + return self.scurler.sfURL(self.rpath) + @property def URL(self): return self.scurler.URL(self.rpath) @@ -166,6 +221,18 @@ def lsall(self): def mkdir(self): return self.scurler.mkdir(self.rpath) + + def is_file(self): + if self.permits is not None: + return (self.permits[0] != 'd') + else: + return Unknown + + def is_dir(self): + if self.permits is not None: + return (self.permits[0] == 'd') + else: + return Unknown #----------------------------------------- #-- BEGIN pathlib.Path emulation methods | @@ -288,7 +355,9 @@ def parsel(entry, rpath = None): } if rpath is not None: - parsed['rpath'] = rpath / name + parsed['rpath'] = pathlib.PurePosixPath(rpath) / name + else: + parsed['rpath'] = name return parsed @@ -345,6 +414,16 @@ def __init__(self, *args, **kwargs): def server(self): return SCURLer(self.host, self.user) + def curl(self, *args): + cmd = list(self.basecom) + list(args) + logger.debug("{}".format(cmd)) + proc = subprocess.run(cmd, capture_output = True) + self.lastop = proc + if proc.returncode != 0: + logger.debug("error {}".format(proc.returncode)) + logger.debug(proc.stderr.decode('utf-8')) + return proc + def URL(self, reqpath = None): return str( self.sfURL(reqpath) ) @@ -362,54 +441,53 @@ def sfURL(self, reqpath = None): rpath = self.root return sfURL(self.host, rpath, user = self.user, port = self.port) + def rm(self, rpath): + """ + Given an absolute path on the remote host (rpath), I execute an "rm {}" command on the remote host. + """ + self.quote("rm", str(rpath)) + def ls(self, rpath = None): """ Given an absolute path on the remote host (rpath), I execute a remote "ls" operation. """ - lsurl = "{}/".format( self.URL(rpath) ) + lsURL = self.sfURL(rpath) lscom = list(self.basecom) lscom.append('--list-only') - lscom.append(lsurl) + lscom.append(str(lsURL)) logger.debug("{}".format(lscom)) - p = subprocess.run(lscom, capture_output = True) - self.lastop = p + p = self.curl('--list-only', str(lsURL)) if p.returncode == 0: entries = p.stdout.decode('utf-8').split('\n') - catalog = [(rpath / entry) for entry in entries] + catalog = [(lsURL.path / entry) for entry in entries] return catalog else: - print(p.stderr) raise NotADirectoryError def lsall(self, rpath = None): """ Given an absolute path on the remote host (rpath), I execute a remote "ls" operation. """ - lsurl = "{}/".format( self.URL(rpath) ) + rqURL = self.sfURL(rpath) + lsURL = str(rqURL) + lsURL = lsURL if lsURL.endswith('/') else "{}/".format(lsURL) - lscom = list(self.basecom) - lscom.append(lsurl) - - logger.debug("{}".format(lscom)) - - p = subprocess.run(lscom, capture_output = True) - self.lastop = p + p = self.curl(lsURL) if p.returncode == 0: entries = p.stdout.decode('utf-8').split('\n') - catalog = [ ] + collected = [ ] for entry in entries: entry = entry.strip() if entry: info = parsel(entry, rpath) if info['name'] not in ('.', '..'): #-- We ommit the special directory references: '.' and '..', but allow all others. - catalog.append( Alien.fromLS(self, info) ) - return catalog + collected.append( Alien.fromLS(self, info) ) + return Catalog(rqURL, collected) else: - logger.debug(p.stderr) raise NotADirectoryError def put(self, lpath, rpath): @@ -418,20 +496,14 @@ def put(self, lpath, rpath): I upload the local file (at lpath) to the remote location specified by rpath. If needed, I automatically make any intermediate folders in the remote path. """ - pturl = self.URL(rpath) - ptcom = list(self.basecom) - ptcom.append('--upload-file') - ptcom.append(str(lpath)) - ptcom.append(str(pturl)) + ptURL = self.sfURL(rpath) + ptargs = ['--upload-file', str(lpath), str(ptURL)] if self.mkdirs: - ptcom.append('--ftp-create-dirs') - logger.debug("{}".format(ptcom)) - p = subprocess.run(ptcom, capture_output = True) - self.lastop = p - if p.returncode == 0: - return True - else: - return False + ptargs.append('--ftp-create-dirs') + + p = self.curl(*ptargs) + + return True if (p.returncode == 0) else False def get(self, rpath, lpath = None): """ @@ -448,19 +520,12 @@ def get(self, rpath, lpath = None): else: target = lpath - gturl = self.URL(rpath) - gtcom = list(self.basecom) - gtcom.append(gturl) - gtcom.append('-o') - gtcom.append(str(target)) + gtURL = self.sfURL(rpath) + gtargs = [str(gtURL), "-o", str(target)] - logger.debug("{}".format(gtcom)) - p = subprocess.run(gtcom, capture_output = True) - self.lastop = p - if p.returncode == 0: - return (True, target) - else: - return (False, None) + p = self.curl(*gtargs) + + return True if (p.returncode == 0) else False def mkdir(self, rpath): """ @@ -471,39 +536,26 @@ def mkdir(self, rpath): rpath = pathlib.PurePosixPath(rpath) myURL = self.sfURL().server - mkcom = list(self.basecom) - mkcom.append( str(myURL) ) + mkargs = [str(myURL)] prepath = self.root for part in rpath.parts: prepath = prepath / part - mkcom.append("-Q") - mkcom.append("*mkdir {}".format(str(prepath))) + mkargs.append("-Q") + mkargs.append("*mkdir {}".format(str(prepath))) - logger.debug("{}".format(mkcom)) - p = subprocess.run(mkcom, capture_output = True) - self.lastop = p - if p.returncode == 0: - return True - else: - return False + p = self.curl(*mkargs) - def quote(self, cmd, *args): - recom = ' '.join( [cmd] + list(args) ) + return True if (p.returncode == 0) else False - mkcom = list(self.basecom) - mkcom.append(str(self.sfURL().server)) - mkcom.append('-Q') - mkcom.append('{}'.format(recom)) + def quote(self, qcmd, *args): + qcom = ' '.join( [qcmd] + list(args) ) - logger.debug("{}".format(mkcom)) - p = subprocess.run(mkcom, capture_output = True) - self.lastop = p - if p.returncode == 0: - return True - else: - return False + qargs = [str(self.sfURL().server), "-Q", qcom] + p = self.curl(*qargs) + + return True if (p.returncode == 0) else False def __truediv__(self, rpath): return Alien(self, self.root / rpath)