refactoring and debugging, WIP

bin/bastion.py

@@ -27,9 +27,10 @@
 from Bastion.Site       import Site
 from Bastion.Condo      import Condex
 from Bastion.Model      import ARK
-from Bastion.CARP       import Request, isReceipt
+from Bastion.CARP       import Request, isReceipt, isReport
 import Bastion.Vaults.HPSS
 import Bastion.Vaults.BFD
+import Bastion.Vaults.SFTP
 
 """
 zone backup procedure...
@@ -240,7 +241,7 @@ def remember(self, answer):
 			halo = scope / "{}.yaml".format(session)
 			halo.parent.mkdir(parents = True, exist_ok = True)
 			with open(halo, 'wt') as fout:
-				self.emit_YAML(answer, fout)
+				self.emit_YAML(answer.toJDN(), fout)
 
 	def emit(self, answer, ostream = None):
 		ostream = ostream if (ostream is not None) else sys.stdout
@@ -256,7 +257,6 @@ def emit(self, answer, ostream = None):
 	def emit_YAML(self, answer, ostream):
 		yaml = YAML()
 		yaml.default_flow_style = False
-		answer.report = PreservedScalarString(answer.report)
 		yaml.dump(answer, ostream)
 
 	def emit_JSON(self, answer, ostream):
@@ -365,13 +365,13 @@ def do_bank_asset(self, request):
 		vault   = self.vault(asset.policy.vault)
 
 		request['log.scope'] = "site/{}".format(ark.site)
-		receipt = vault.push(asset, client = self.hostname)
+		result = vault.push(asset, client = self.hostname)
 
-		if receipt.indicates_success:
-			blonde = receipt.body['blonde']
-			return request.succeeded("pushed full backup of {} to {}".format(str(ark), str(blonde)), receipt)
+		if result.indicates_success:
+			blonde = result.record['blonde']
+			return request.succeeded(result, report = "pushed full backup of {} to {}".format(str(ark), str(blonde)))
 		else:
-			return request.failed("while pushing full backup of {}, something went wrong!".format(str(ark)), receipt)
+			return request.failed(result, report = "while pushing full backup of {}, something went wrong!".format(str(ark)))
 
 	#↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑
 	#-- END bank (backup) operations   |

bin/test.py

@@ -1,9 +1,11 @@
+
 #!/usr/bin/env python3
 
 import sys
 import os
 import pathlib
 import logging
+import socket
 
 logger = logging.getLogger()
 logging.basicConfig(level = logging.DEBUG)
@@ -16,93 +18,21 @@
 
 sys.path.insert(0, str(LIB_PATH))
 
-from Bastion.Common  import *
-from Bastion.Site    import Site
-from Bastion.Condo   import Condex
-import Bastion.HPSS
-from Bastion.Curator import BLOND
-
-"""
-zone backup procedure...
-1. read conf file(s)
-2. connect to the storage vault
-3. retrieve manifest for given zone
-4. get most recent anchor snap
-5. compute drift between current datetime and most recent anchor snap
-6. perform backup
-	a. differential if drift < policy
-	b. anchor (full) if drift >= policy
-"""
-
-
-class App:
-	CONF_SEARCH_ORDER = [
-		pathlib.Path('/etc/bastion'),
-		APP_PATH / 'etc',
-		pathlib.Path('~/.bastion').expanduser()
-	]
-
-	def info(self, msg):
-		logger.info(msg)
-
-	def debug(self, msg):
-		logger.debug(msg)
-
-	def warn(self, msg):
-		logger.warn(msg)
-
-	def error(self, msg):
-		logger.error(msg)
-
-	def critical(self, msg):
-		logger.critical(msg)
-
-	def __init__(self):
-		self.conf = Condex()
-
-	def configured(self):
-		for folder in App.CONF_SEARCH_ORDER:
-			folder = folder.expanduser()
-			for confile in folder.rglob("conf-*.yaml"):
-				self.info("reading conf from {}".format(str(folder / confile)))
-				self.conf.load(folder / confile)
-		return self
-
-
-
-
-
-
+from bastion import App
 
 if __name__ == '__main__':
 	app = App().configured()
-	#app.run( )
-
-	vault = Bastion.HPSS.Vault('fortress').configured(app.conf)
-	site = Site('idifhub').configured(app.conf)
-
-	RzLiDAR = site.zones[0]
-	assets  = site.assets(RzLiDAR)
-	asset   = assets[0]
-
 
+	host = os.environ.get('BASTION_SITE', socket.gethostname())
 
-#bastion site {site} backup
-#bastion zone {zone} backup
-#bastion manifest {zone}
-#bastion backups tidy idifhub
-#bastion backups update idifhub
-#bastion backups catalog idifhub
-#bastion keytab refresh fortress
-#bastion zone { } restore
-#bastion asset { } restore
-#bastion asset { } backup
-#bastion zone { } backup
+	if host == 'rusina':
+		rusina      = app.site('rusina')
+		soundscapes = rusina.zone('soundscapes')
+		asset       = soundscapes['HackathonData']
+		vault       = app.vault(asset.policy.vault)
+	elif host == 'scout':
+		scout = app.site('scout')
+		ADS   = scout.zone('ADS')
+		asset = ADS['Titanic3']
+		vault = app.vault(asset.policy.vault)
 
-#bastion restore asset { }
-#bastion restore zone { }
-#bastion backup asset { }
-#bastion backup zone { }
-#bastion export zone manifest { }
-#bastion export asset manifest { }
-#bastion export site list 

lib/Bastion/CARP.py

@@ -22,6 +22,7 @@
 import datetime
 import uuid
 import traceback
+import pprint
 
 import yaml
 
@@ -181,7 +182,8 @@ def crashed(self, record = None, *args, **kwargs):
 	def inconclusive(self, record = None, *args, **kwargs):
 		return Report.Inconclusive(self, record, *args, **kwargs)
 
-
+#-- legacy alias
+Request = isRequest
 
 class isResult:
 	def __init__(self, request, status, record, **kwargs):
@@ -220,19 +222,21 @@ def toJDN(self, **kwargs):
 			'status':    self.status.code,
 			'message':   self.status.gloss,
 			'lede':      self.lede,
+			'answers':   self.request.ID,
 			'answered':  self.when.isoformat( ),
 			'elapsed':   self.elapsed.total_seconds(),
 			'context':   { },
 			'record':    toJDN(self.record) if self.record else None
 		}
+
 		#-- Populate the result's context (if any)
 		for k, v in self.context.items( ):
 			result['context'][k] = toJDN(v)
 
 		#-- Construct the JDN for return
 		jdn = { }
 		jdn['request'] = toJDN(self.request)
-		jdn['request']['result'] = result
+		jdn['result']  = result
 
 		return jdn
 
@@ -257,20 +261,15 @@ def __init__(self, request, status, record = None, *args, **kwargs):
 	@property
 	def body(self):
 		if getattr(self, '_body', None) is None:
-			self._body = self.toMD()
+			if callable(getattr(self.record, 'toJDN', None)):
+				self._body = pprint.pformat(self.record.toJDN())
+			else:
+				self._body = pprint.pformat(self.record)
 		return self._body
 
-	def toMD(self):
-		"""
-		I answer a markdown representation of self.
-		Base class is naive and just the string representation of my record.
-		Override in subclasses for detailed output.
-		"""
-		return str(self.record)
-
 	def toJDN(self, **kwargs):
-		jdn = isReceipt.toJDN(self, **kwargs)
-		jdn['request']['result']['report'] = self.body
+		jdn = isResult.toJDN(self, **kwargs)
+		jdn['result']['report'] = self.body
 		return jdn
 
 	def __str__(self):
@@ -346,6 +345,9 @@ def __init__(self, results):
 			else:
 				raise ValueError("result must be an instance of Bastion.CARP.isResult")
 
+	def __getitem__(self, i):
+		return self.results[i]
+
 	def __iter__(self):
 		return iter(self.results)
 

lib/Bastion/Common.py

@@ -413,3 +413,9 @@ def asPurePath(x):
 	Accessor/wrapper for answering x already cast as a pathlib.PurePosixPath instance
 	"""
 	return pathlib.PurePosixPath(x)
+
+def asInteger(x):
+	return int(x)
+
+def asNumeric(x):
+	return float(x)

lib/Bastion/Curator.py

@@ -5,12 +5,12 @@
 """
 import datetime
 
-from Bastion.Common     import Thing, Boggle, canTextify
+from Bastion.Common     import Thing, Boggle, canStruct
 from Bastion.Model      import ARK, isAsset
 from Bastion.Chronology import Quantim
 
 
-class BLONDE(canTextify):
+class BLONDE:
 	"""
 	BLOb Name and Description Encoding
 	I am a structured name describing a point in time for a single ARK.
@@ -64,6 +64,9 @@ def encode(badge, when, detail, genus):
 	def __str__(self):
 		return self.encode(self.badge, self.when, self.detail, self.genus)
 
+	def toJDN(self, **kwargs):
+		return str(self)
+
 	@classmethod
 	def decode(cls, blonde):
 		ds = Thing(**{

lib/Bastion/Movers/SFTP.py

@@ -2,20 +2,13 @@
 import logging
 
 import Bastion.Model 
+from   Bastion.Common import asPath, asPurePath
 from   Bastion.NetOps.sCURL import SCURLer
+from   Bastion.Movers.CARP import PutRequest, PutReceipt
 
 logger = logging.getLogger(__name__)
 
 
-def asPath(x):
-	return pathlib.Path(x)
-
-def asPurePath(x):
-	return pathlib.PurePosixPath(x)
-
-
-
-
 class Mover(Bastion.Model.isMover):
 	def __init__(self, vault, **kwargs):
 		Bastion.Model.isMover.__init__(self, vault)
@@ -36,8 +29,26 @@ def provision(self, *args):
 		raise NotImplementedError
 
 	def put(self, halo, tag, **kwargs):
-		raise NotImplementedError
-
+		here  = halo
+		there = self.vault.sfURL / tag
+		logger.debug( "put source {} to {}".format(here.as_uri(), str(there)) )
+
+		#-- Create the put request.
+		request = PutRequest(halo, str(self.vault.sfURL), tag)
+
+		try:
+			#-- execute sCURL operation, here.
+			copied = self.scurler.put(halo, tag)
+		except Exception as err:
+			#-- shutil should throw an error if there was a problem copying.
+			report = request.failed(err)
+		else:
+			if copied:
+				report = request.succeeded( PutReceipt(halo, str(self.vault.sfURL), tag) )
+			else:
+				report = request.failed( PutReceipt(halo, str(self.vault.sfURL), tag), report = "curl upload operation failed" )
+
+		return report
 
 	def get(self, tag, halo, **kwargs):
 		raise NotImplementedError

lib/Bastion/NetOps/sCURL.py

@@ -31,13 +31,6 @@
 }
 
 
-#class isRemotePosixFileSystem:
-#	def ls(self, *args, **kwargs):
-#		raise NotImplementedError
-#
-#	def 
-
-
 class sfURL(tuple):
 	"""
 	secure file URL
@@ -319,14 +312,18 @@ def __init__(self, *args, **kwargs):
 		self.mkdirs  = kwargs.get('mkdirs', True)    #-- default to creating missing directories in upload paths.
 		self.CURL    = kwargs.get('curl', "/usr/bin/curl")
 
+		self.CURL    = pathlib.Path(self.CURL)
+
+		assert self.CURL.exists(), "local path to curl command ({}) does not exist".format(self.CURL)
+
 		self.lastop  = None
 
 		#-------------------------------------------------------------------------
 		#-- Construct the base command given my default configuration.           |
 		#-- All SFTP operations will be built by augmenting the base command ... |
 		#-- ... with additional flags and arguments.                             |
 		#-------------------------------------------------------------------------
-		basecom = [self.CURL]
+		basecom = [str(self.CURL)]
 		if self.keypath is not None:
 			basecom.extend(['--key', str(self.keypath)])
 		if self.silent:

lib/Bastion/Site.py

@@ -6,7 +6,7 @@
 import random
 import datetime
 
-from Bastion.Common import entity, Thing, canStruct, DAYS, RDN, asPath, asLogLevel
+from Bastion.Common import entity, Thing, canStruct, DAYS, RDN, asPath, asLogLevel, canConStruct
 from Bastion.Condo import CxNode
 from Bastion.Model import ARK, isAsset, isZone, isSite
 #from .Curator import Asset

lib/Bastion/Vaults/BFD.py

@@ -50,12 +50,6 @@ def clerk(self):
 			self._clerk = Bastion.Clerks.BFD.Clerk(self)
 		return self._clerk
 
-	@property
-	def packer(self):
-		if getattr(self, '_packer', None) is None:
-			self._packer = Bastion.Packers.TARs.Packer(self)
-		return self._packer
-
 	@property
 	def mover(self):
 		if getattr(self, '_mover', None) is None:

lib/Bastion/Vaults/CARP.py

@@ -55,7 +55,7 @@ def __init__(self, subject, *args, **kwargs):
 				'halo':   str(asset.halo)
 			}
 
-			isRequest.__init__(self, "PackRequest", asset, **xtras)
+			isRequest.__init__(self, "PushRequest", asset, **xtras)
 
 	@property
 	def asset(self):