Lots of debugging, feature build out with bank and amend verbs; more …

…debugging in report serialization.

bin/bastion.py

@@ -11,7 +11,7 @@
 from ruamel.yaml.scalarstring import PreservedScalarString
 
 logger = logging.getLogger()
-logging.basicConfig(level = logging.WARN)
+logging.basicConfig(level = logging.DEBUG)
 
 
 BIN_PATH = pathlib.Path(sys.argv[0]).absolute().parent
@@ -26,8 +26,8 @@
 from Bastion.Chronology import Quantim
 from Bastion.Site       import Site
 from Bastion.Condo      import Condex
-from Bastion.Model      import ARK
-from Bastion.CARP       import Request, isResult, isReport
+from Bastion.Model      import ARK, isAsset
+from Bastion.CARP       import Request, isResult, isReport, BagReceipt
 import Bastion.Vaults.HPSS
 import Bastion.Vaults.BFD
 import Bastion.Vaults.SFTP
@@ -109,7 +109,7 @@ def logroot(self):
 
 	def site(self, s):
 		if isinstance(s, ARK):
-			return Site(s.name).configure(self.conf)
+			return Site(s.site).configured(self.conf)
 		elif isinstance(s, str):
 			#-- Assume that s is the name of a site.
 			return Site(s).configured(self.conf)
@@ -119,18 +119,31 @@ def site(self, s):
 	def asset(self, ark):
 		return self.site(ark.site).asset(ark)
 
-	def vault(self, name, site = None):
+	def vault(self, which, site = None):
 		"""
 		I answer an instance of a storage vault given a name and optional site.
+		.vault(name:str)
+		.vault(asset:isAsset)
+		.vault(ark:ARK)
 		"""
-		if ((name[0] == '{') and (name[-1] == '}')):
-			name = name[1:-1]
-		if name in self.conf['vaults']:
-			protocol = self.conf['vaults'][name]['protocol']
-			cls = Bastion.Model.Vault.handling(protocol)
-			return cls(name).configured(self.conf)
-		else:
-			return None
+		if isinstance(which, str):
+			name = which
+			if ((name[0] == '{') and (name[-1] == '}')):
+				name = name[1:-1]
+			if name in self.conf['vaults']:
+				protocol = self.conf['vaults'][name]['protocol']
+				cls = Bastion.Model.Vault.handling(protocol)
+				return cls(name).configured(self.conf)
+
+		if isinstance(which, isAsset):
+			asset = which
+			return self.vault(asset.policy.vault)
+
+		if isinstance(which, ARK):
+			asset = self.asset(which)
+			return self.vault(asset.policy.vault)
+
+		return None
 
 	@property
 	def sites(self):
@@ -287,6 +300,7 @@ def emit_JSON(self, result, ostream):
 
 	def emit_PROSE(self, result, ostream):
 		request = result.request
+		ostream.write("ID: {}\n".format(request.ID))
 		ostream.write("request: {}\n".format(request.lede))
 		ostream.write("reply: {reply.code} {reply.gloss}\n".format(reply=result.status))
 		ostream.write("# {reply.lede}\n".format(reply=result))
@@ -385,7 +399,7 @@ def do_bank_zone(self, request):
 
 		site = self.site(ark)
 		zone = site.zone(ark)
-		arks = [asset.ark for asset in zone]
+		arks = [asset.ARK for asset in zone]
 		request['log.scope'] = "site/{}".format(site.name)
 		return self._bank_assets(request, arks)
 
@@ -401,18 +415,17 @@ def _bank_assets(self, request, arks):
 		"""
 		helper method to push (bank) many assets given a single (batched) request.
 		"""
-		results = [ ]
+		results = BagReceipt()
 		for ark in arks:
 			asset = self.asset(ark)
-			vault = self.vault(asset.policy.vault)
+			vault = self.vault(asset)
 			result = vault.push(asset, client = self.hostname)
 			results.append(result)
 
-		all_succeeded = all([result.indicates_success for result in results])
-		all_failed    = all([result.indicates_failure for result in results])
-
-		pushed = [result['blonde'] for result in results if result.indicates_success]
+		all_succeeded = all([result.succeeded for result in results])
+		all_failed    = all([result.failed for result in results])
 
+		pushed = [result.blonde for result in results if result.succeeded]
 		if all_succeeded:
 			return request.succeeded(results, report = "all {} assets successfully pushed".format(len(results)))
 		elif all_failed:
@@ -482,25 +495,25 @@ def do_amend_zone(self, request):
 
 		site = self.site(ark)
 		zone = site.zone(ark)
-		arks = [asset.ark for asset in zone]
+		arks = [asset.ARK for asset in zone]
 		request['log.scope'] = "site/{}".format(site.name)
 		return self._amend_assets(request, arks)
 
 	def _amend_assets(self, request, arks):
 		"""
 		helper method to amend (differential backup) many assets given a single (batched) request.
 		"""
-		results = [ ]
+		results = BagReceipt()
 		for ark in arks:
 			asset = self.asset(ark)
 			vault = self.vault(asset.policy.vault)
 			result = vault.amend(asset, client = self.hostname)
 			results.append(result)
 
-		all_succeeded = all([result.indicates_success for result in results])
-		all_failed    = all([result.indicates_failure for result in results])
-
-		amended = [result['blonde'] for result in results if result.indicates_success]
+		all_succeeded = all([result.succeeded for result in results])
+		all_failed    = all([result.failed for result in results])
+		
+		amended = [result.blonde for result in results if result.succeeded]
 
 		if all_succeeded:
 			return request.succeeded(result, report = "all {} assets successfully amended".format(len(results)))
@@ -686,9 +699,7 @@ def do_export_zones_provisioned(self, request):
 		export zones provisioned {vault} {site}
 		* lists all zones provisioned in the given vault for the named site
 		"""
-		print(request.args[0])
 		vault     = self.vault( request.args[0] )
-		print(vault.name)
 		site_name = request.args[1]
 
 		zone_names = vault.zones(site_name)
@@ -812,7 +823,7 @@ def do_refresh_keytab(self, request):
 			rusina      = app.site('rusina')
 			soundscapes = rusina.zone('soundscapes')
 			asset       = soundscapes['HackathonData']
-			vault       = app.vault(asset.policy.vault)
+			vault       = app.vault(asset)
 	else:
 		app.run( )
 

lib/Bastion/CARP.py

@@ -354,6 +354,36 @@ def toJDN(self, **kwargs):
 		raise NotImplementedError(".toJDN is subclass responsibility")
 
 
+class isBagReceipt:
+	"""
+	I am a thin wrapper around a sequence of results.
+	"""
+	def __init__(self):
+		self.results = [ ]
+
+	def append(self, result, *args):
+		if isinstance(result, isResult):
+			self.results.append(result)
+		else:
+			raise ValueError("isBagReceipt: cannot append a value of type {}".format(type(result)))
+
+	def __iter__(self):
+		return iter(self.results)
+
+
+	def __getitem__(self, i):
+		return self.results[i]
+
+	def __len__(self):
+		return len(self.results)
+
+	def toJDN(self, **kwargs):
+		return [toJDN(result) for result in self]
+
+
+BagReceipt = isBagReceipt
+
+
 class isWorkflowReceipt(isReceipt, isTagged):
 	"""
 	I am a thin wrapper around a sequence of results.

lib/Bastion/Clerks/SFTP.py

@@ -11,7 +11,7 @@
 
 class Clerk(isClerk):
 	def __init__(self, vault, **kwargs):
-		isClerk.__init__(self)
+		isClerk.__init__(self, vault)
 		self.vault   = vault
 		self.scurler = SCURLer(self.vault.sfURL, keyfile = self.vault.keypath)
 
@@ -73,10 +73,12 @@ def manifest(self, *args):
 		else:
 			raise ValueError
 
+		blondes = [ ]
 		cell = self.scurler / ark.site / ark.zone / ark.asset
-		if cell.exists():
-			blondes = [ ]
+		print("CELL:", cell)
+		if cell.exists:
 			for alien in cell.lsall():
+				print(alien)
 				if not alien.is_dir():
 					blondes.append( BLONDE.decode(alien.stem) )
 		manifest = Manifest(ark, blondes)

lib/Bastion/Curator.py

@@ -86,7 +86,7 @@ def forFullBackup(cls, ark, **kwargs):
 	@classmethod
 	def forDiffBackup(cls, anchor, **kwargs):
 		"""
-		Given a full backup reference (BLONDE), I generate a new BLONDE for a differential backup.
+		Given a full backup reference (anchor:BLONDE), I answer a new BLONDE for a differential backup.
 		"""
 		when  = kwargs.get('when', datetime.datetime.now())
 		return cls(anchor.badge, 'D', anchor.genus, when)

lib/Bastion/Model.py

@@ -232,9 +232,9 @@ def amend(self, asset, **kwargs):
 		Given an asset, I push a differential backup based on the most recent genus.
 		"""
 		#-- Get the current genus (aka full backup or anchor)
-		manifest = self.clerk.manifest(asset.ark)
+		manifest = self.clerk.manifest(asset.ARK)
 		#-- Call .push with the genus as the basis for the differential backup.
-		return self.push(asset, manifest.head.genus, **kwargs)
+		return self.push(asset, manifest.head.anchor, **kwargs)
 
 	def push(self, asset, basis = None, **kwargs):
 		"""

lib/Bastion/Movers/SFTP.py

@@ -26,7 +26,7 @@ def provision(self, *args):
 		provision(ark) - ensures that the site, zone, and asset folders exist.
 		provision(site, zone, asset_name) - an alias for provision(ark)
 		"""
-		raise NotImplementedError
+		pass
 
 	def put(self, halo, tag, **kwargs):
 		here  = halo

lib/Bastion/Packers/CARP.py

@@ -4,7 +4,7 @@
 
 from Bastion.Model    import isAsset
 from Bastion.CARP     import isRequest, isReceipt
-from Bastion.Curator  import BLONDE
+from Bastion.Curator  import BLONDE, Snap
 
 
 logger = logging.getLogger(__name__)
@@ -63,10 +63,14 @@ def __init__(self, subject, *args, **kwargs):
 					whence  = basis.when.earliest
 					genus   = basis.genus
 					blonded = BLONDE.forDiffBackup(basis)
-				if isinstance(basis, datetime.datetime):
+				elif isinstance(basis, datetime.datetime):
 					whence  = basis
 					genus   = "___"
 					blonded = BLONDE(asset.ARK, detail, genus)
+				elif isinstance(basis, Snap):
+					whence  = basis.earliest
+					genus   = basis.genus
+					blonded = BLONDE.forDiffBackup(basis.anchor)
 			else:
 				detail = 'F'
 				blonded = BLONDE.forFullBackup(asset.ARK)

lib/Bastion/Packers/TARs.py

@@ -10,7 +10,7 @@
 
 from Bastion.Chronology import quantim
 from Bastion.Common import asPath, prefer
-import Bastion.Model
+from Bastion.Model import isAsset, isPacker
 from Bastion.CARP import isRequest, Report
 from Bastion.Packers.CARP import PackRequest, PackingReceipt
 
@@ -61,21 +61,27 @@ def pax(tarp, folder, **kwargs):
 	"""
 	#-- If tar runs into any problems, it should throw an exception.
 	with tarfile.open(tarp, "w", format = tarfile.PAX_FORMAT) as tar:
+		opts = { }
 		if 'since' in kwargs:
 			when = kwargs['since']
-			tar.add(folder, filter = ifFileChanged(when))
-		else:
-			tar.add(folder)
+			if isinstance(when, datetime.datetime):
+				opts['filter'] = ifFileChanged( kwargs['since'] )
+			else:
+				raise ValueError("pax: since must be an instance of datetime")
+		if 'rebase' in kwargs:
+			opts['arcname'] = kwargs['rebase']
+
+		tar.add(folder, **opts)
 
 	#-- if no exceptions were generated during the tar construction,
 	#-- then we get here and we can return a happy True!
 	return True
 
 
 
-class Packer(Bastion.Model.isPacker):
+class Packer(isPacker):
 	def __init__(self, vault, **kwargs):
-		Bastion.Model.isPacker.__init__(self, vault)
+		isPacker.__init__(self, vault)
 		#-- find a scratch folder in order of lowest (default) to highest priority.
 		#-- 1. /tmp
 		#-- 2. vault's scratch folder (if any)
@@ -88,10 +94,8 @@ def pack(self, subject, *args, **kwargs):
 		Can be invoked as...
 		.pack(request)
 		.pack(asset)
-		.pack(asset, basis)
-		.pack(asset, since)
-		Without a given basis, I package everything (i.e. a full backup).
-		When a basis is given as either a datetime or an anchor (BLONDE), I do a differential backup.
+		.pack(asset, since:datetime)
+		Without a given "since", I package everything (i.e. a full backup).
 		On success, I answer a report that wraps an instance of Packers.CARP.Packed
 		On failure, I answer a ReportException
 		Where...
@@ -107,11 +111,13 @@ def pack(self, subject, *args, **kwargs):
 		elif isinstance(subject, isRequest):
 			#-- .pack(request:isRequest)
 			request = PackRequest(subject)
-		else:
+		elif isinstance(subject, isAsset):
 			#-- .pack(asset)
 			#-- .pack(asset, basis)
 			#-- .pack(asset, since)
 			request = PackRequest(subject, *args, **kwargs)
+		else:
+			raise ValueError
 
 		ark     = request.asset.ARK
 		package = "{}.tar".format(str(request.blonde))
@@ -124,7 +130,12 @@ def pack(self, subject, *args, **kwargs):
 
 		try:
 			#-- use the built-in python tar archiver, using "pax" (POSIX.1-2001) extensions.
-			pax(tarp, request.asset.halo, **kwargs)
+			opts = {
+				'rebase': pathlib.PurePosixPath(ark.zone) / ark.asset
+			}
+			if request.basis is not None:
+				opts['since'] = request.since
+			pax(tarp, request.asset.halo, **opts)
 		except Exception as err:
 			report = request.failed(err)
 		else:
@@ -156,5 +167,5 @@ def catalog(tarp):
 		for info in tark.getmembers():
 			sz = info.size
 			mq = quantim( datetime.datetime.fromtimestamp(info.mtime) )
-			cats.append( (sz, mq, info.name) )
+			cats.append( [sz, mq, info.name] )
 	return cats