adding SCURLer (sftp by libCURL) and new SFTP storage vault

lab/fortress_example.py

@@ -14,6 +14,7 @@
 BLONDE  = "3AQXEGFS024A03CMFZMT.tar"
 
 fortress = SCURLer(USER, 'sftp.fortress.rcac.purdue.edu', keyfile = KEYPATH, silent = False, verbose = True)
-home     = fortress['/home/{}'.format(USER)]
-bastion  = fortress['/home/{}/bastion'.format(USER)]
+home     = fortress / 'home' / '{}'.format(USER)
+bastion  = fortress / 'home' / '{}'.format(USER) / 'bastion'
+
 

lib/Bastion/Model.py

@@ -129,12 +129,43 @@ def provision(self, *args):
 		"""
 		raise NotImplementedError
 
+	def pack(self, asset, basis = None, **kwargs):
+		"""
+		Given a local asset, I package (.tar, .zip, etc) the asset into my scratch (spool) space.
+		Without a given basis, I package everything (i.e. a full backup).
+		When a basis is given as either a datetime or an anchor (BLONDE), I do a differential backup.
+		I answer a tuple like (blonde, tag, repo, packaged)
+		Where...
+		* blonde is the BLONDE of the newly created archive
+		* tag is the relative path to the archive file
+		* spool is the absolute path to the folder holding the new archive file.
+		* package is the name of the archive file in the spool folder
+		"""
+		raise NotImplementedError
+
+	def unpack(self, halo, root, **kwargs):
+		"""
+		Given a local packed archive object (e.g. .tar, .zip, etc.) at halo, 
+		I unpack the archive into the given (local) root path.
+		"""
+		raise NotImplementedError
+
+
 	def push(self, asset, basis = None, **kwargs):
 		"""
 		Given an asset, I push a backup of the asset to this vault.
 		push(asset) - creates a full backup in this vault, creating a new base for differentials
 		push(asset, basis) - creates a differential backup relative to the given basis.
 		{basis} - can be a datetime or a BLONDE.
+		A typical implementation of .push() ...
+		1. call .pack() method to create a local archive in my scratch (spool) space
+		2. use .put() to transfer the local archive to the vault space
+		3. perform a vault-specific transfer verification
+		4. remove the local, scratch (spool) archive file.
+		Answers a tuple of (transferred, blonde, receipt), where...
+		* transferred - is the True/False indication answered by .put()
+		* blonde - is the string reprsentation of the blonde for the archive of the asset,
+		* receipt - is detailed, structured answer given by the .put() operation.
 		"""
 		raise NotImplementedError
 
@@ -144,8 +175,9 @@ def pull(self, blonde, **kwargs):
 	def put(self, halo, tag, **kwargs):
 		"""
 		Given path to a local file (aka Host Asset LOcation), 
-		move the file from the local scope to this vault and store
-		the object at tag (the path relative to the root of this vault)
+		move the file from the local scope to this vault and store the object at tag (the path relative to the root of this vault)
+		Answers (True, receipt) or (False, receipt) where True indicates transfer success and False indicates transfer failure.
+		receipt is a (possibly nested) dictionary that is a more detailed (and specific to the actual transfer method) description of the transfer as executed.
 		"""
 		raise NotImplementedError
 

lib/Bastion/Movers/sCURL.py

@@ -79,6 +79,15 @@ def toJDN(self):
 	def URL(self):
 		return self.scurler.URL(self.rpath)
 
+	def __repr__(self):
+		return "[{}@{}:{}]".format(self.scurler.user, self.scurler.host, str(self.rpath))
+
+	def __truediv__(self, subpath):
+		subpath = pathlib.PurePosixPath(subpath)
+		if subpath.is_absolute():
+			raise Exception("sub paths must be relative")
+		return Alien(self.scurler, self.rpath / subpath)
+
 	def __lshift__(self, lpath):
 		"""
 		put/upload operation
@@ -100,7 +109,7 @@ def __iter__(self):
 		Iterates over the contents of the folder, 
 		each item being a pathlib.Path object.
 		"""
-		return iter(self.ls())
+		return iter(self.lsall())
 
 	def ls(self):
 		"""
@@ -112,6 +121,11 @@ def ls(self):
 	def lsall(self):
 		return self.scurler.lsall(self.rpath)
 
+	@property
+	def is_dir(self):
+		#-- alias for is_folder
+		return self.is_folder
+
 	@property
 	def is_folder(self):
 		#-- First look at the permission bits.
@@ -128,6 +142,45 @@ def is_folder(self):
 			folderq = False
 		return folderq
 
+	@property
+	def exists(self):
+		me = self.rpath.name
+		up = self.scurler / self.rpath.parent
+		try:
+			exists = any([(entry.rpath.name == me) for entry in up])
+		except NotADirectoryError:
+			exists = False
+		return exists
+
+	@property
+	def files(self):
+		"""
+		If I am a folder on the remote host, then I answer a list of the names of the files that I contain (without recursion).
+		If I am NOT a folder on the remote host, I answer an empty list.
+		"""
+		try:
+			entries = self.lsall()
+		except NotADirectoryError:
+			files = [ ]
+		else:
+			files = [entry.rpath.name for entry in entries if not entry.is_folder]
+		return files
+
+	@property
+	def folders(self):
+		"""
+		If I am a folder on the remote host, then I answer a list of names for my subfolders (without recursion).
+		If I am NOT a folder on the remote host, I answer an empty list.
+		"""
+		try:
+			entries = self.lsall()
+		except NotADirectoryError:
+			folders = [ ]
+		else:
+			folders = [entry.rpath.name for entry in entries if entry.is_folder]
+		return folders
+
+
 
 def parsel(entry, rpath = None):
 	"""
@@ -189,7 +242,6 @@ def __init__(self, user, host, **kwargs):
 		self.mkdirs  = kwargs.get('mkdirs', True)   #-- default to creating missing directories in upload paths.
 		self.CURL    = kwargs.get('curl', "/usr/bin/curl")
 
-
 		self.lastop  = None
 
 		#-------------------------------------------------------------------------
@@ -349,8 +401,6 @@ def mkdir(self, rpath):
 		else:
 			return False
 
-	def __getitem__(self, rpath):
-		"""
-		Using array index semantics, I answer an "Alien" interface to the given remote path (rpath).
-		"""
-		return Alien(self, rpath)
+	def __truediv__(self, rpath):
+		root = pathlib.PurePosixPath("/")
+		return Alien(self, root / rpath)

lib/Bastion/Vaults/BFD.py

@@ -128,13 +128,12 @@ def provision(self, *args):
 		else:
 			raise ValueError
 
-	def push(self, asset, basis = None, **kwargs):
+	def pack(self, asset, basis = None, **kwargs):
 		"""
-		Given an asset, I push a backup of the asset to this vault.
-		push(asset) - creates a full backup in this vault, creating a new base for differentials
-		push(asset, basis) - creates a differential backup relative to the given basis.
-		{asset} - an instance of Bastion.Model.isAsset
-		{basis} - can be a datetime or a BLONDE.
+		Given a local asset, I package (.tar, .zip, etc) the asset into my scratch (spool) space.
+		Without a given basis, I package everything (i.e. a full backup).
+		When a basis is given as either a datetime or an anchor (BLONDE), I do a differential backup.
+		I answer the BLONDE for the package, relative to the local scratch (spool) space.
 		"""
 		detail  = 'F'
 		whence  = None
@@ -148,46 +147,77 @@ def push(self, asset, basis = None, **kwargs):
 			detail = 'D'
 			if isinstance(basis, BLONDE):
 				#-- I was given a BLONDE (a reference to a full backup)
-				anchor = basis
-				whence = anchor.when.earliest
-				genus  = anchor.genus
+				anchor  = basis
+				whence  = anchor.when.earliest
+				genus   = anchor.genus
 				blonded = BLONDE.forDiffBackup(anchor)
 
 			if isinstance(basis, datetime.datetime):
-				whence = basis
-				genus  = "___"
+				whence  = basis
+				genus   = "___"
 				blonded = BLONDE(asset.ARK, detail, genus)
 
 			opts['since'] = whence
 
 		else:
 			blonded = BLONDE.forFullBackup(asset.ARK)
 
-		tarp = "{}.tar".format(str(blonded))
+		package  = "{}.tar".format(str(blonded))
+		tag      = pathlib.PurePosixPath(ark.site) / ark.zone / ark.asset / package
+		spool    = (self.scratch / tag).parent
+
+		#-- assure that the scratch path exists and all of the subpaths that create the repo folder for this asset.
+		spool.mkdir(parents = True, exist_ok = True)
+
+		#-- use the built-in python tar archiver, using "pax" (POSIX.1-2001) extensions.
+		pax((self.scratch / tag), asset.halo, **opts)
+
+		#-- answer the BLONDE of the newly created package.
+		return (blonded, tag, spool, package)
+
 
-		#-- assure that the scratch and bank paths exist.
-		(self.scratch / ark.site / ark.zone / ark.asset).mkdir(parents = True, exist_ok = True)
-		(self.bank / ark.site / ark.zone / ark.asset).mkdir(parents = True, exist_ok = True)
+	def push(self, asset, basis = None, **kwargs):
+		"""
+		Given an asset, I push a backup of the asset to this vault.
+		push(asset) - creates a full backup in this vault, creating a new base for differentials
+		push(asset, basis) - creates a differential backup relative to the given basis.
+		{asset} - an instance of Bastion.Model.isAsset
+		{basis} - can be a datetime or a BLONDE.
+		"""
+		blonde, tag, spool, package = self.pack(asset, basis, **kwargs)
 
-		tag = "{}/{}/{}/{}".format(ark.site, ark.zone, ark.asset, tarp)
+		#-- assure that the bank exists.
+		(self.bank / tag).parent.mkdir(parents = True, exist_ok = True)
 
-		pax(self.scratch / tag, asset.halo, **opts)
+		transferred, receipt = self.put(self.scratch / tag, tag)
 
-		self.put(self.scratch / tag, tag)
+		if transferred:
+			#-- clean up!
+			(self.scratch / tag).unlink()
 
-		#-- clean up!
-		(self.scratch / tag).unlink()
+		return (transferred, blonde, receipt)
 
-		return blonded
 
 	def pull(self, ark, **kwargs):
 		raise NotImplementedError
 
+
 	def put(self, halo, tag, **kwargs):
-		here = halo
+		here  = halo
 		there = self.bank / tag
 		logger.debug("put source {} to {}".format(str(here), str(there)))
+		started = datetime.datetime.now()
 		shutil.copy(here, there)
+		completed = datetime.datetime.now()
+		receipt = {
+			'tag': str(tag),
+			'source': str(here),
+			'destination': str(there),
+			'started': started.isoformat(),
+			'completed': completed.isoformat()
+		}
+		return (True, receipt)
+
 
 	def get(self, tag, halo, **kwargs):
 		shutil.copystat(self.bank / tag, halo)

lib/Bastion/Vaults/HPSS.py

@@ -321,7 +321,7 @@ def lsx(self, path = None):
 
 
 class Vault(Bastion.Model.Vault):
-	PROTOCOL = 'HPSS'
+	PROTOCOL = 'HTAR'
 
 	def __init__(self, name, **kwargs):
 		Bastion.Model.Vault.__init__(self, name, **kwargs)