Working towards SFTP interface to (remote) BFD-like repo.

lib/Bastion/Model.py

@@ -76,6 +76,73 @@ def badge(self):
 		return Slug40(str(self.CURIE))
 
 
+class isOpReceipt:
+	def __init__(self, status, message, opts):
+		self.succeeded = succeeded
+		self.opts = { }
+		for k, v in opts.items():
+			self.opts[k] = v
+
+	@property
+	def opts(self):
+		return set(self.opts.keys())
+
+	def __getitem__(self, k):
+		return self.opts[k]
+
+	def __setitem__(self, k, v):
+		self.opts[k] = v
+		return self
+
+
+
+class TransferReceipt(isOpReceipt):
+	"""
+	Generic class for returning detailed information about the transfer (put) operation.
+	"""
+	def __init__(self, halo, tag, started, ended, opts):
+		isOpReceipt.__init__(self, opts)
+		self.succeeded = False
+		self.source    = halo
+		self.tag       = tag
+		self.started   = started
+		self.ended     = ended
+		self.message   = ""
+
+	def toJDN(self, **kwargs):
+		dex = { }
+		dex['source']  = str(self.source)
+		dex['tag']     = str(self.tag)
+		dex['started'] = self.started.isoformat()
+		dex['ended']   = self.ended.isoformat()
+		dex['opts']    = { }
+		for k in opts:
+			dex['opts'][k] = self.opts[k]
+		return dex
+
+
+class PackingReceipt(isOpReceipt):
+	"""
+	Generic class for returning detailed information about the pack operation.
+	"""
+	def __init__(self, succeeded, asset, blonde, spooled, opts):
+		isOpReceipt.__init__(self, succeeded, opts)
+
+		self.asset  = asset
+		self.blonde = blonde
+		self.spool  = spooled
+
+	def toJDN(self, **kwargs):
+		dex = { }
+		dex['asset']    = str(self.asset)
+		dex['blonde']   = str(self.blonde)
+		dex['spooled']  = str(self.spooled)
+		dex['opts']     = { }
+		for opt in self.opts:
+			dex['opts'][opt] = self.opts[opt]
+		return dex
+
+
 class isVault:
 	"""
 	abstract class for all vaults.

lib/Bastion/Movers/sCURL.py

@@ -229,9 +229,13 @@ class SCURLer:
 	I am a client for executing single SFTP operations on a remote host via CURL.
 	I execute all of my SFTP operations by executing CURL in a subshell.
 	"""
-	def __init__(self, user, host, **kwargs):
+	def __init__(self, user, host, root = None, **kwargs):
 		self.host    = host
 		self.user    = user
+		self.root    = pathlib.PurePosixPath("/")
+
+		if root is not None:
+			self.root = pathlib.PurePosixPath(root)
 
 		#--------------------------------------------------------------
 		#-- Configure while looking for additional keyword arguments. |
@@ -259,18 +263,18 @@ def __init__(self, user, host, **kwargs):
 
 		self.basecom = tuple(basecom)
 
-	def URL(self, rpath = None):
+	def URL(self, reqpath = None):
 		"""
-		Given an absolute path on the remote host (rpath), I construct an SFTP URL to the remote path.
+		Given a path (relative to my root) on the remote host (rpath), I construct an SFTP URL to the remote path.
 		"""
-		if rpath is None:
-			return "sftp://{}@{}/".format(self.user, self.host)
+		if reqpath is not None:
+			reqpath = pathlib.PurePosixPath(reqpath)
+			if reqpath.is_absolute():
+				raise ValueError("rpath {} must be a relative path".format(str(reqpath)))
+			rpath = self.root / reqpath
 		else:
-			rpath = pathlib.PurePosixPath(rpath)
-			if rpath.is_absolute():
-				return "sftp://{}@{}{}".format(self.user, self.host, str(rpath))
-			else:
-				raise ValueError("rpath {} must be an absolute path".format(str(rpath)))
+			rpath = self.root
+		return "sftp://{}@{}{}".format(self.user, self.host, str(rpath))
 
 	def ls(self, rpath):
 		"""

lib/Bastion/Vaults/SFTP.py

@@ -28,6 +28,7 @@ def asPurePath(x):
 	return pathlib.PurePosixPath(x)
 
 
+
 class Vault(Bastion.Model.Vault):
 	PROTOCOL = 'SFTP'
 
@@ -37,7 +38,7 @@ def __init__(self, name, **kwargs):
 		self.scratch = pathlib.Path("/tmp")
 		self.host    = None
 		self.login   = getpass.getuser()
-		self.key     = pathlib.Path("~/.ssh/id")
+		self.keypath = pathlib.Path("~/.ssh/id")
 		self.bank    = None
 
 	def configured(self, conf):
@@ -51,17 +52,16 @@ def configured(self, conf):
 			self.scratch = local.get(asPath, "scratch", "/tmp")
 
 			#-- Configuration relevant to remote (bank) host.
-			self.host  = remote.get('host')
-			self.login = remote.get('login', getpass.getuser())
-			self.key   = remote.get(asPath, 'key', pathlib.Path("~/.ssh/id").expanduser())
-			self.root  = remote.get(asPurePath, 'root', "/")
+			self.host    = remote.get('host')
+			self.login   = remote.get('login', getpass.getuser())
+			self.keypath = remote.get(asPath, 'key', pathlib.Path("~/.ssh/id").expanduser())
+			self.root    = remote.get(asPurePath, 'root', "/")
 
 		return self
 
 	@property
 	def bank(self):
-		client = SCURLer(self.login, self.host, keyfile = self.key)
-		return (client / self.root)
+		client = SCURLer(self.login, self.host, self.root, keyfile = self.key)
 
 #---------------------------------------
 #-- BEGIN Bastion.Model.Vault PROTOCOL |
@@ -149,7 +149,7 @@ def pack(self, asset, basis = None, **kwargs):
 		Given a local asset, I package (.tar, .zip, etc) the asset into my scratch (spool) space.
 		Without a given basis, I package everything (i.e. a full backup).
 		When a basis is given as either a datetime or an anchor (BLONDE), I do a differential backup.
-		I answer the BLONDE for the package, relative to the local scratch (spool) space.
+		I answer an instance of Bastion.Model.PackingReceipt
 		"""
 		detail  = 'F'
 		whence  = None
@@ -180,16 +180,24 @@ def pack(self, asset, basis = None, **kwargs):
 
 		package  = "{}.tar".format(str(blonded))
 		tag      = pathlib.PurePosixPath(ark.site) / ark.zone / ark.asset / package
-		spool    = (self.scratch / tag).parent
+		spooled  = (self.scratch / tag)
+		spool    = spooled.parent
 
 		#-- assure that the scratch path exists and all of the subpaths that create the repo folder for this asset.
 		spool.mkdir(parents = True, exist_ok = True)
 
 		#-- use the built-in python tar archiver, using "pax" (POSIX.1-2001) extensions.
 		pax((self.scratch / tag), asset.halo, **opts)
 
-		#-- answer the BLONDE of the newly created package.
-		return (blonded, tag, spool, package)
+		#-- start a receipt for packing the asset.
+		receipt = Bastion.Model.PackingReceipt(asset, blonded, spooled)
+
+		#-- some optional information that might be of use...
+		receipt['size'] = spooled.stat().st_size
+		receipt['tag']  = str(tag)
+
+		#-- Answer the receipt.
+		return receipt
 
 
 	def push(self, asset, basis = None, **kwargs):
@@ -200,12 +208,9 @@ def push(self, asset, basis = None, **kwargs):
 		{asset} - an instance of Bastion.Model.isAsset
 		{basis} - can be a datetime or a BLONDE.
 		"""
-		blonde, tag, spool, package = self.pack(asset, basis, **kwargs)
+		packed = self.pack(asset, basis, **kwargs)
 
-		#-- assure that the bank exists.
-		(self.bank / tag).parent.mkdir(parents = True, exist_ok = True)
-
-		transferred, receipt = self.put(self.scratch / tag, tag)
+		xferrd = self.put(packed.spooled, packed.opts['tag'])
 
 		if transferred:
 			#-- clean up!
@@ -223,13 +228,13 @@ def put(self, halo, tag, **kwargs):
 		there = self.bank / tag
 		logger.debug("put source {} to {}".format(str(here), str(there)))
 		started = datetime.datetime.now()
-		shutil.copy(here, there)
+
 		completed = datetime.datetime.now()
 		receipt = {
-			'tag': str(tag),
-			'source': str(here),
-			'destination': str(there),
-			'started': started.isoformat(),
+			'tag':       str(tag),
+			'source':    str(here),
+			'URL':       str(there),
+			'started':   started.isoformat(),
 			'completed': completed.isoformat()
 		}
 		return (True, receipt)