Major refactoring of the underlying model and thus code. New model is…

… based on "snaps" in time that are archived.

bin/bastion.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import pathlib
+import logging
+
+logger = logging.getLogger()
+logging.basicConfig(level = logging.DEBUG)
+
+
+BIN_PATH = pathlib.Path(sys.argv[0]).absolute().parent
+APP_PATH = BIN_PATH.parent
+LIB_PATH = APP_PATH / 'lib'
+LBX_PATH = APP_PATH / 'lib-exec'
+
+sys.path.insert(0, str(LIB_PATH))
+
+
+from Bastion.Common import *
+from Bastion.Site import Site
+from Bastion.Condo import *
+
+
+
+CONF_SEARCH_ORDER = [
+	pathlib.Path('/etc/bastion'),
+	APP_PATH / 'etc',
+	pathlib.Path('~/.bastion').expanduser()
+]
+
+if __name__ == '__main__':
+	comargs = dict(enumerate(sys.argv[1:]))
+	subject = comargs.get(0, 'help')
+	request = comargs.get(1, 'help')
+
+	conf = Condex()
+	for folder in CONF_SEARCH_ORDER:
+		for confile in folder.rglob("conf-*.yaml"):
+			print(confile)
+			conf.load(folder / confile)
+
+	if subject == 'keytab':
+		if request == 'refresh':
+			raise NotImplementedError
+		else:
+			Keytab(conf).help()
+
+	if subject == 'backups':
+		sname = comargs[2]
+		site = Site(sname).configured(conf)
+
+
+#bastion backups tidy idifhub
+#bastion backups update idifhub
+#bastion backups catalog idifhub
+#bastion keytab refresh fortress

bin/fossil.py

@@ -21,6 +21,8 @@
 #-----------------------------------------------------------------------------------------------------------------------
 museum  = Fossils(login = ndenny, conf = "idif.xlsx")  #-- the default login can also be part of the conf workbook.
 
+museum.vault("idifhub.ecn.purdue.edu", "LiDAR")
+
 #--------------------------------------------------------------------------------------------------------------
 #-- This is the "do everything" magic method that uses a lot of lower level methods to accomplish it's goal.  |
 #--------------------------------------------------------------------------------------------------------------
@@ -36,7 +38,7 @@
 #-- slug is the base32 encoding of the shake128/5 hash of the dataset's name.                                 |
 #-- backups are also annotated with provenance in more readable English text.                                 |
 #--------------------------------------------------------------------------------------------------------------
-report = museum.backup("QL2_3DEP_LiDAR_IN_2011_2013_l2")
+report = museum.fossilize("QL2_3DEP_LiDAR_IN_2011_2013_l2")
 
 #-- This forces a differential backup of the given dataset RIGHT NOW.
 #-- Without an optional override, default behavior is to use the most recent full backup as the basis for ths
@@ -62,3 +64,16 @@ def backup_full(folder, **kwargs):
 def backup_differential(folder):
 	return backup(folder, 1, **kwargs)
 
+
+SITE = sys.argv[1]
+
+site     = Site(SITE)
+manifest = Manifest(SITE)
+musuem   = HPSS.Museum(site)
+
+for zone in manifest.zones:
+	for asset in manifest[zone].assets:
+		curator = museum.curator(zone, asset)
+		if curator.branches.most_recent.deposited > asset.longevity:
+			curator.curate(asset)
+

etc/conf-idifhub.yaml

@@ -0,0 +1,69 @@
+sites:
+  idifhub:
+    host: idifhub.ecn.purdue.edu
+    logging:
+      level: DEBUG
+      path: /var/log/bastion
+      persist:
+        git:
+          repo: git@github.purdue.edu/ndenny/bastion-idif
+          key: /home/ndenny/.ssh/ndenny@rcac
+
+    #-- policy defined with the site is the default policy for all RZs within this site.
+    policy:
+      history: 60
+      drift:   30
+      vault:   fortress
+
+    #-- declare any (resource) zones within the scope of this site.
+    zones:
+      #-- this is the long form for declaring a resource zone (Rz)
+      LiDAR:
+        root: /home/jinha/www/lidar
+        #-- each resource zone can declare its own policy
+        policy:
+          history: 60
+          drift: 30
+          vault: fortress
+      #-- this next example is the short form for declaring a reource zone
+      #-- this resource zone inherits all of its policy, etc. from the site in which it resides.
+      hyperspectral: /home/jinha/www/hsi     
+
+assets:
+  idifhub:
+    #-- Top level keys are the names of (resource) zones for this site.
+    LiDAR:
+      #-- short form of an asset is just the folder or file that is immediately subordinate to the RZ's root
+      #-- assets declared using the short form will inherit their policy, etc. from superior contexts.
+      - QL2_3DEP-LiDAR_IN_2011_2013_l2
+      #-- long form of an asset allows for policy override, etc.
+      - name: QL2_3DEP_LiDAR_IN_2017_2019
+        policy:
+          history: 90
+          drift:   30
+      - QL2_3DEP_LiDAR_IN_2017_2019_l2
+      - QL2_3DEP_LiDAR_IN_2017_2019_laz
+      - name: QLX_3DEP-LiDAR_US_South
+        about: low resolution LiDAR coverage of US South (census) Region
+        RDN: SOUTHLRZ
+    #-- this is a special case that includes all files and folders in the RZ's root folder as distinct assets.
+    hyperspectral: "*"
+
+vaults:
+  fortress:
+    protocol: HPSS
+    host: fortress.rcac.purdue.edu
+    login: ndenny
+    key:
+      path: /home/ndenny/.private/hpss.unix.keytab
+      refresh:
+        period: 60
+        ssh:
+          host: data.rcac.purdue.edu
+          user: ndenny
+          key:  /home/ndenny/.ssh/ndenny@rcac
+
+  BFD:
+    protocol: copy
+    zone: /mnt/BFD/bastion
+

lab/bootstrap.py

@@ -0,0 +1,32 @@
+import sys
+import pathlib
+import logging
+import datetime
+
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level = logging.DEBUG)
+
+RUN_PATH = pathlib.Path(sys.argv[0]).absolute().parent
+APP_PATH = RUN_PATH.parent
+LIB_PATH = APP_PATH / 'lib'
+LBX_PATH = APP_PATH / 'lib-exec'
+
+sys.path.insert(0, str(LIB_PATH))
+
+from Bastion.Common import *
+from Bastion.Site import Site
+from Bastion.Condo import *
+from Bastion.Curator import Snap
+
+CONF_SEARCH_ORDER = [
+	pathlib.Path('/etc/bastion'),
+	APP_PATH / 'etc',
+	pathlib.Path('~/.bastion').expanduser()
+]
+
+conf = Condex().load(APP_PATH / 'etc' / 'conf-idifhub.yaml')
+site = Site('idifhub').configured( conf )
+
+slug = site.zone('LiDAR').assets.any.RDN
+latest = Snap.dub(slug, datetime.datetime.now(), "D", "XXV")