From b0ced79620c0cde7afd405653c1724b79c4f634c Mon Sep 17 00:00:00 2001 From: Nathan Denny Date: Mon, 28 Oct 2024 09:38:32 -0400 Subject: [PATCH] Working towards SFTP interface to (remote) BFD-like repo. --- lib/Bastion/Model.py | 67 +++++++++++++++++++++++++++++++++++++ lib/Bastion/Movers/sCURL.py | 24 +++++++------ lib/Bastion/Vaults/SFTP.py | 47 ++++++++++++++------------ 3 files changed, 107 insertions(+), 31 deletions(-) diff --git a/lib/Bastion/Model.py b/lib/Bastion/Model.py index de0269f..d360d75 100644 --- a/lib/Bastion/Model.py +++ b/lib/Bastion/Model.py @@ -76,6 +76,73 @@ def badge(self): return Slug40(str(self.CURIE)) +class isOpReceipt: + def __init__(self, status, message, opts): + self.succeeded = succeeded + self.opts = { } + for k, v in opts.items(): + self.opts[k] = v + + @property + def opts(self): + return set(self.opts.keys()) + + def __getitem__(self, k): + return self.opts[k] + + def __setitem__(self, k, v): + self.opts[k] = v + return self + + + +class TransferReceipt(isOpReceipt): + """ + Generic class for returning detailed information about the transfer (put) operation. + """ + def __init__(self, halo, tag, started, ended, opts): + isOpReceipt.__init__(self, opts) + self.succeeded = False + self.source = halo + self.tag = tag + self.started = started + self.ended = ended + self.message = "" + + def toJDN(self, **kwargs): + dex = { } + dex['source'] = str(self.source) + dex['tag'] = str(self.tag) + dex['started'] = self.started.isoformat() + dex['ended'] = self.ended.isoformat() + dex['opts'] = { } + for k in opts: + dex['opts'][k] = self.opts[k] + return dex + + +class PackingReceipt(isOpReceipt): + """ + Generic class for returning detailed information about the pack operation. + """ + def __init__(self, succeeded, asset, blonde, spooled, opts): + isOpReceipt.__init__(self, succeeded, opts) + + self.asset = asset + self.blonde = blonde + self.spool = spooled + + def toJDN(self, **kwargs): + dex = { } + dex['asset'] = str(self.asset) + dex['blonde'] = str(self.blonde) + dex['spooled'] = str(self.spooled) + dex['opts'] = { } + for opt in self.opts: + dex['opts'][opt] = self.opts[opt] + return dex + + class isVault: """ abstract class for all vaults. diff --git a/lib/Bastion/Movers/sCURL.py b/lib/Bastion/Movers/sCURL.py index 746bbff..ffcd13b 100644 --- a/lib/Bastion/Movers/sCURL.py +++ b/lib/Bastion/Movers/sCURL.py @@ -229,9 +229,13 @@ class SCURLer: I am a client for executing single SFTP operations on a remote host via CURL. I execute all of my SFTP operations by executing CURL in a subshell. """ - def __init__(self, user, host, **kwargs): + def __init__(self, user, host, root = None, **kwargs): self.host = host self.user = user + self.root = pathlib.PurePosixPath("/") + + if root is not None: + self.root = pathlib.PurePosixPath(root) #-------------------------------------------------------------- #-- Configure while looking for additional keyword arguments. | @@ -259,18 +263,18 @@ def __init__(self, user, host, **kwargs): self.basecom = tuple(basecom) - def URL(self, rpath = None): + def URL(self, reqpath = None): """ - Given an absolute path on the remote host (rpath), I construct an SFTP URL to the remote path. + Given a path (relative to my root) on the remote host (rpath), I construct an SFTP URL to the remote path. """ - if rpath is None: - return "sftp://{}@{}/".format(self.user, self.host) + if reqpath is not None: + reqpath = pathlib.PurePosixPath(reqpath) + if reqpath.is_absolute(): + raise ValueError("rpath {} must be a relative path".format(str(reqpath))) + rpath = self.root / reqpath else: - rpath = pathlib.PurePosixPath(rpath) - if rpath.is_absolute(): - return "sftp://{}@{}{}".format(self.user, self.host, str(rpath)) - else: - raise ValueError("rpath {} must be an absolute path".format(str(rpath))) + rpath = self.root + return "sftp://{}@{}{}".format(self.user, self.host, str(rpath)) def ls(self, rpath): """ diff --git a/lib/Bastion/Vaults/SFTP.py b/lib/Bastion/Vaults/SFTP.py index 7e48eb8..9f30b0a 100644 --- a/lib/Bastion/Vaults/SFTP.py +++ b/lib/Bastion/Vaults/SFTP.py @@ -28,6 +28,7 @@ def asPurePath(x): return pathlib.PurePosixPath(x) + class Vault(Bastion.Model.Vault): PROTOCOL = 'SFTP' @@ -37,7 +38,7 @@ def __init__(self, name, **kwargs): self.scratch = pathlib.Path("/tmp") self.host = None self.login = getpass.getuser() - self.key = pathlib.Path("~/.ssh/id") + self.keypath = pathlib.Path("~/.ssh/id") self.bank = None def configured(self, conf): @@ -51,17 +52,16 @@ def configured(self, conf): self.scratch = local.get(asPath, "scratch", "/tmp") #-- Configuration relevant to remote (bank) host. - self.host = remote.get('host') - self.login = remote.get('login', getpass.getuser()) - self.key = remote.get(asPath, 'key', pathlib.Path("~/.ssh/id").expanduser()) - self.root = remote.get(asPurePath, 'root', "/") + self.host = remote.get('host') + self.login = remote.get('login', getpass.getuser()) + self.keypath = remote.get(asPath, 'key', pathlib.Path("~/.ssh/id").expanduser()) + self.root = remote.get(asPurePath, 'root', "/") return self @property def bank(self): - client = SCURLer(self.login, self.host, keyfile = self.key) - return (client / self.root) + client = SCURLer(self.login, self.host, self.root, keyfile = self.key) #--------------------------------------- #-- BEGIN Bastion.Model.Vault PROTOCOL | @@ -149,7 +149,7 @@ def pack(self, asset, basis = None, **kwargs): Given a local asset, I package (.tar, .zip, etc) the asset into my scratch (spool) space. Without a given basis, I package everything (i.e. a full backup). When a basis is given as either a datetime or an anchor (BLONDE), I do a differential backup. - I answer the BLONDE for the package, relative to the local scratch (spool) space. + I answer an instance of Bastion.Model.PackingReceipt """ detail = 'F' whence = None @@ -180,7 +180,8 @@ def pack(self, asset, basis = None, **kwargs): package = "{}.tar".format(str(blonded)) tag = pathlib.PurePosixPath(ark.site) / ark.zone / ark.asset / package - spool = (self.scratch / tag).parent + spooled = (self.scratch / tag) + spool = spooled.parent #-- assure that the scratch path exists and all of the subpaths that create the repo folder for this asset. spool.mkdir(parents = True, exist_ok = True) @@ -188,8 +189,15 @@ def pack(self, asset, basis = None, **kwargs): #-- use the built-in python tar archiver, using "pax" (POSIX.1-2001) extensions. pax((self.scratch / tag), asset.halo, **opts) - #-- answer the BLONDE of the newly created package. - return (blonded, tag, spool, package) + #-- start a receipt for packing the asset. + receipt = Bastion.Model.PackingReceipt(asset, blonded, spooled) + + #-- some optional information that might be of use... + receipt['size'] = spooled.stat().st_size + receipt['tag'] = str(tag) + + #-- Answer the receipt. + return receipt def push(self, asset, basis = None, **kwargs): @@ -200,12 +208,9 @@ def push(self, asset, basis = None, **kwargs): {asset} - an instance of Bastion.Model.isAsset {basis} - can be a datetime or a BLONDE. """ - blonde, tag, spool, package = self.pack(asset, basis, **kwargs) + packed = self.pack(asset, basis, **kwargs) - #-- assure that the bank exists. - (self.bank / tag).parent.mkdir(parents = True, exist_ok = True) - - transferred, receipt = self.put(self.scratch / tag, tag) + xferrd = self.put(packed.spooled, packed.opts['tag']) if transferred: #-- clean up! @@ -223,13 +228,13 @@ def put(self, halo, tag, **kwargs): there = self.bank / tag logger.debug("put source {} to {}".format(str(here), str(there))) started = datetime.datetime.now() - shutil.copy(here, there) + completed = datetime.datetime.now() receipt = { - 'tag': str(tag), - 'source': str(here), - 'destination': str(there), - 'started': started.isoformat(), + 'tag': str(tag), + 'source': str(here), + 'URL': str(there), + 'started': started.isoformat(), 'completed': completed.isoformat() } return (True, receipt)