BIG changes

bin/bastion.py

@@ -29,7 +29,8 @@
 from Bastion.Condo      import *
 from Bastion.Actions    import *
 from Bastion.Model      import ARK
-import Bastion.HPSS
+import Bastion.Vaults.HPSS
+import Bastion.Vaults.BFD
 
 """
 zone backup procedure...
@@ -137,6 +138,9 @@ def logroot(self):
 	def site(self, name):
 		return Site(name).configured(self.conf)
 
+	def asset(self, ark):
+		return self.site(ark.site).asset(ark)
+
 	def vault(self, name, site = None):
 		"""
 		I answer an instance of a storage vault given a name and optional site.
@@ -220,6 +224,8 @@ def run(self):
 		except Exception as err:
 			tb = traceback.format_exception(err)
 			answer = CRASHED( ''.join(tb), tb )
+			#-- always log crashes!
+			answer['context']['log.scope'] = '*'
 
 		proc['task.ended'] = datetime.datetime.now().isoformat()
 
@@ -260,7 +266,12 @@ def record(self, answer, opts):
 		if 'log.scope' in answer['context']:
 			#-- only write a log file if we have an explicit log.scope in the answer's context block.
 			session = answer['context']['task.session']
-			halo = self.logroot / answer['context']['log.scope'] / "{}.yaml".format(session)
+
+			scope = self.logroot
+			if answer['context']['log.scope'] != '*':
+				scope = scope / answer['context']['log.scope']
+
+			halo = scope / "{}.yaml".format(session)
 			halo.parent.mkdir(parents = True, exist_ok = True)
 			with open(halo, 'wt') as fout:
 				self.emit_YAML(answer, opts, fout)
@@ -354,11 +365,16 @@ def do_bank_asset(self, comargs, comdex, opts):
 		site  = self.site(ark.site)
 		asset = site.asset(ark)
 		vault = self.vault(asset.policy.vault)
-		flag, stdout, stderr = vault.push(asset, client = self.hostname)
-		if flag:
-			return SUCCESS(stdout, {'stdout': stdout, 'stderr': stderr})
+		blonde = vault.push(asset, client = self.hostname)
+
+		extras = {
+			'log.scope': "site/{}".format(ark.site)
+		}
+
+		if blonde:
+			return SUCCESS("pushed full backup {}".format(str(blonde)), context = extras)
 		else:
-			return FAILED(stdout, {'stdout': stdout, 'stderr': stderr})
+			return FAILED("something went wrong!", context = extras)
 
 	#↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑
 	#-- END bank (backup) operations   |
@@ -402,22 +418,37 @@ def do_update_site(self, comargs, comdex, opts):
 		"""
 		raise NotImplementedError
 
-	def do_update_zone(self, comargs, comdex):
+	def do_update_zone(self, comargs, comdex, opts):
 		"""
 		update zone {zone}
 		* creates a differential backup of each asset in {zone}
 		"""
 		raise NotImplementedError
 
-	def do_update_asset(self, comargs, comdex):
+	def do_update_asset(self, comargs, comdex, opts):
 		"""
 		update asset {ARK}
 		* asset given in ARK format
 		* determines backup level based on asset's policy and current manifest
 		* performs backup based on determined level
 		* typically used to "automatically" do updates in a scheduled (cron) job
 		"""
-		raise NotImplementedError
+		ark    = ARK(comdex[2])
+		asset  = self.asset(ark)
+		vault  = self.vault(asset.policy.vault)
+		banked = vault.manifest(ark)
+		if banked.anchors:
+			basis = banked.anchors[-1]
+			blonde = vault.push(asset, basis)
+		else:
+			blonde = vault.push(asset)
+
+		extras = {
+			'log.scope': "site/{}".format(ark.site)
+		}
+
+		return SUCCESS("pushed update {}".format(str(blonde)), context = extras)
+
 
 	#↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑
 	#-- END update (automatic) operations     |
@@ -564,6 +595,11 @@ def do_refresh_keytab(self, comargs, comdex, opts):
 	app = App().configured()
 	if sys.argv[1:] != ['shell']:
 		app.run( )
+	else:
+		rusina      = app.site('rusina')
+		soundscapes = rusina.zone('soundscapes')
+		asset       = soundscapes['HackathonData']
+		vault       = app.vault(asset.policy.vault)
 
 #-- does a full, level 0 backup
 #bastion backup site {site}

lib/Bastion/Chronology.py

@@ -144,6 +144,7 @@ def quaver(self):
 		sQ = Quantim.EN36[msq] + Quantim.EN36[lsq]
 		return self.separator.join([sY, sM, sD, sQ])
 
+	@property
 	def datetime(self):
 		"""
 		answers a python datetime.datetime object that is the midpoint of this quantum
@@ -155,15 +156,25 @@ def datetime(self):
 			elapsed_seconds = ((self.qM * Quantim.QUANTIM) + (Quantim.QUANTIM / 2)) * SECONDS
 			return (y + elapsed_seconds)
 
+	@property
 	def earliest(self):
 		"""
 		answers a python datetime.datetime object that is the earliest time within the range of this quantum
 		"""
 		y               = datetime.datetime(self.dY + 2000, self.dM, self.dD, 0, 0, 0)
-		elapsed_seconds = self.qM * Quantim.QUANTIM
+		elapsed_seconds = (self.qM * Quantim.QUANTIM) * SECONDS
+		return (y + elapsed_seconds)
+
+	def latest(self):
+		"""
+		answers a python datetime.datetime object that is the latest time possible within the range of this quantum.
+		"""
+		y               = datetime.datetime(self.dY + 2000, self.dM, self.dD, 0, 0, 0)
+		elapsed_seconds = ((self.qM * Quantim.QUANTIM) + (Quantim.QUANTIM - 1)) * SECONDS
 		return (y + elapsed_seconds)
 
 
+
 	@classmethod
 	def now(cls):
 		return cls(datetime.datetime.now())

lib/Bastion/Curator.py

@@ -6,7 +6,7 @@
 import pathlib
 
 from .Common     import *
-from .Model      import ARK
+from .Model      import ARK, isAsset
 from .Chronology import Quantim
 
 
@@ -18,20 +18,22 @@ class BLONDE(canTextify):
 	def __init__(self, asset, detail, basis = None, when = None):
 		if isinstance(asset, ARK):
 			self.badge = asset.badge
+		elif isinstance(asset, isAsset):
+			self.badge = asset.ARK.badge
 		elif isinstance(asset, str):
 			self.badge = asset
 		else:
 			raise ValueError
 
-		when = datetime.datetime.utcnow() if when is None else when
+		when = datetime.datetime.now() if when is None else when
 
 		self.when      = Quantim(when)
 		self.detail    = detail
 		self.basis     = basis
 		self.RDN       = str(self)
 
 		#-- Automatically create a basis reference for full backups.
-		if self.detail == 'F':
+		if (self.detail == 'F') and (self.basis is None):
 			self.basis = Boggle(3)
 
 	@property
@@ -64,7 +66,7 @@ def decode(cls, blonde):
 			'detail': blonde[16],
 			'basis':  blonde[17:20],
 		})
-		return cls(ds.badge, ds.when, ds.detail, ds.basis)
+		return cls(ds.badge, ds.detail, ds.basis, ds.when)
 
 	@classmethod
 	def forFullBackup(cls, ark, **kwargs):
@@ -140,9 +142,9 @@ def __init__(self, asset, items):
 		self._snaps   = None
 		self._anchors = None  #-- a map of anchor layer -> blob name
 
-		blondes        = [BLONDE(item) for item in items]
+		blondes       = list(items)
 		self._items   = tuple(sorted([blonde for blonde in blondes if (blonde.badge == self.badge)], key = lambda b: b.RDN))
-		self._anchors = dict([(item.layer, item) for item in self._items if item.isAnchor])
+		self._anchors = dict([(item.basis, item) for item in self._items if item.isAnchor])
 
 	def __iter__(self):
 		return iter(self._items)
@@ -203,7 +205,7 @@ def anchors(self):
 		"""
 		I answer a chronologically (earliest -> latest) tuple of BLONDEs for my anchor items.
 		"""
-		return tuple(sorted(self._anchors.values, key = lambda item: item.RDN))
+		return tuple(sorted(self._anchors.values(), key = lambda item: item.RDN))
 
 	#------------------------
 	#-- canTextify protocol |

lib/Bastion/Model.py

@@ -123,26 +123,27 @@ def provision(self, *args):
 		"""
 		raise NotImplementedError
 
-	def push(self, asset, **kwargs):
+	def push(self, asset, basis = None, **kwargs):
 		"""
 		Given an asset, I push a backup of the asset to this vault.
-		push(asset, detail = 'FULL')
-		push(asset, detail = 'DIFF')
+		push(asset) - creates a full backup in this vault, creating a new base for differentials
+		push(asset, basis) - creates a differential backup relative to the given basis.
+		{basis} - can be a datetime or a BLONDE.
 		"""
 		raise NotImplementedError
 
 	def pull(self, blonde, **kwargs):
 		raise NotImplementedError
 
-	def put(self, halo, tag):
+	def put(self, halo, tag, **kwargs):
 		"""
 		Given path to a local file (aka Host Asset LOcation), 
 		move the file from the local scope to this vault and store
 		the object at tag (the path relative to the root of this vault)
 		"""
 		raise NotImplementedError
 
-	def get(self, tag, halo):
+	def get(self, tag, halo, **kwargs):
 		"""
 		Given a tag (the path relative to the root of this vault), 
 		download the object and store it in the local file designated by halo.
@@ -205,7 +206,7 @@ def CURIE(self):
 
 	@property
 	def ARK(self):
-		return ARK(self.zone.site.name, RDN(self.zone), self.name)
+		return ARK(RDN(self.zone.site), RDN(self.zone), self.name)
 
 	@property
 	def badge(self):

lib/Bastion/Site.py

@@ -300,11 +300,11 @@ def configured(self, aconf):
 
 	@property
 	def created(self):
-		return datetime.datetime.fromtimestamp( self.path.stat().st_stime )
+		return datetime.datetime.fromtimestamp( self.halo.stat().st_ctime )
 
 	@property
 	def modified(self):
-		return datetime.datetime.fromtimestamp( self.path.stat().st_mtime )
+		return datetime.datetime.fromtimestamp( self.halo.stat().st_mtime )
 
 
 

lib/Bastion/Utils.py

@@ -0,0 +1,69 @@
+"""
+Bastion.Utils
+
+I mostly contain "helper" functions that are agnostic to the vault protocol, etc.
+"""
+import datetime
+import pathlib
+import tarfile
+import logging
+
+import Bastion.Model
+
+logger = logging.getLogger(__name__)
+
+class ifFileChanged:
+	"""
+	A class that can create callable instances to check
+	tarinfo metadata blocks for file modifications since a given data.
+	this is a filter for use with the python tarfile module and follows
+	the filter convention used where the filter modifies in-place the
+	given tarinfo object, or returns None to signal that the file should
+	be skipped.
+	e.g.
+	filter = ifChangedSince( datetime.datetime(2024,1,1) )
+	if filter(tarinfo):
+		#-- add to tar file
+	else:
+		#-- return None
+	"""
+	def __init__(self, when):
+		self.whence = when
+
+	def __call__(self, tarinfo):
+		if tarinfo.type not in (tarfile.REGTYPE, tarfile.AREGTYPE):
+			return tarinfo
+
+		then = datetime.datetime.fromtimestamp(tarinfo.mtime)
+		logger.debug("comparing file {} mod'd at {} to change limit at {}".format(tarinfo.name, then.isoformat(), self.whence.isoformat()))
+		if then > self.whence:
+			return tarinfo
+		else:
+			#-- returning None is rejecting this file for inclusion
+			#-- in the accumulating tar file.
+			return None
+
+def pax(tarp, asset, **kwargs):
+	"""
+	pax uses the python tarfile module to create a tar file using the
+	extended POSIX.1-2001 (aka "PAX") format.
+	gnutar (as of the date of writing, 2024-09-04) can read PAX format, but still defaults to creating archives using its own modified USTAR format.
+	{tarp} is the halo (path) to the local file where the tar will be built.
+	{asset} can be a file path or an instance of Bastion.Model.isAsset (e.g. Bastion.Site.Asset)
+	differential backups can be done by supplying the "since" keyword set the datetime which is the earliest allowed modification time for files to be admitted into the tar.
+	"""
+	if isinstance(asset, Bastion.Model.isAsset):
+		src = asset.halo
+	else:
+		src = pathlib.Path(asset)
+
+	with tarfile.open(tarp, "w", format = tarfile.PAX_FORMAT) as tar:
+		if 'since' in kwargs:
+			when = kwargs['since']
+			tar.add(src, filter = ifFileChanged(when))
+		else:
+			tar.add(src)
+
+	#-- if no exceptions were generated during the tar construction,
+	#-- then we get here and we can return a happy True!
+	return True