roll up

bin/bastion.py

@@ -4,6 +4,9 @@
 import os
 import pathlib
 import logging
+import json
+
+import yaml
 
 logger = logging.getLogger()
 logging.basicConfig(level = logging.DEBUG)
@@ -17,41 +20,215 @@
 sys.path.insert(0, str(LIB_PATH))
 
 
-from Bastion.Common import *
-from Bastion.Site import Site
-from Bastion.Condo import *
-
+from Bastion.Common  import *
+from Bastion.Site    import Site
+from Bastion.Condo   import *
+from Bastion.Actions import *
+from Bastion.Model   import ARK
+import Bastion.HPSS
+
+"""
+zone backup procedure...
+1. read conf file(s)
+2. connect to the storage vault
+3. retrieve manifest for given zone
+4. get most recent anchor snap
+5. compute drift between current datetime and most recent anchor snap
+6. perform backup
+	a. differential if drift < policy
+	b. anchor (full) if drift >= policy
+"""
+
+def SUCCESS(obj):
+	return {
+		'reply': {
+			'status':  '200',
+			'message': 'Ok'
+		},
+		'body': obj
+	}
+
+
+def FAILED(obj):
+	return {
+		'reply': {
+			'status':  '400',
+			'message': 'Bad Request'
+		},
+		'body': obj
+	}
+
+
+def CRASHED(obj):
+	return {
+		'reply': {
+			'status':  '500',
+			'message': 'Internal Application Error (crash)'
+		},
+		'body': obj
+	}
+
+
+
+
+class App:
+	CONF_SEARCH_ORDER = [
+		pathlib.Path('/etc/bastion'),
+		APP_PATH / 'etc',
+		pathlib.Path('~/.bastion').expanduser()
+	]
+
+	def info(self, msg):
+		logger.info(msg)
+
+	def debug(self, msg):
+		logger.debug(msg)
+
+	def warn(self, msg):
+		logger.warn(msg)
+
+	def error(self, msg):
+		logger.error(msg)
+
+	def critical(self, msg):
+		logger.critical(msg)
+
+	def __init__(self):
+		self.conf = Condex()
+
+	def configured(self):
+		for folder in App.CONF_SEARCH_ORDER:
+			folder = folder.expanduser()
+			for confile in folder.rglob("conf-*.yaml"):
+				self.info("reading conf from {}".format(str(folder / confile)))
+				self.conf.load(folder / confile)
+		return self
+
+	def site(self, name):
+		return Site(name).configured(self.conf)
+
+	def vault(self, name):
+		if ((name[0] == '{') and (name[-1] == '}')):
+			name = name[1:-1]
+		if name in self.conf['vaults']:
+			protocol = self.conf['vaults'][name]['protocol']
+			if protocol == 'HPSS':
+				return Bastion.HPSS.Vault(name).configured(self.conf)
+			else:
+				raise NotImplementedError
+		else:
+			return None
+
+	def run(self):
+		comargs = sys.argv[1:]
+		comdex  = dict(enumerate(sys.argv[1:]))
+		#verb    = comargs.get(0, 'help')
+
+		menu = [
+			("help",                  self.do_help),
+			("export zone assets",    self.do_export_zone_assets),
+			("export site list",      self.do_export_site_list),
+			("export asset manifest", self.do_export_asset_manifest),
+			("backup asset",          self.do_backup_asset)
+		]
+
+		action = self.do_help  #-- default is to show the help.
+		for request, method in menu:
+			tokens = request.split()
+			if tokens == comargs[:len(tokens)]:
+				action = method
+		try:
+			answer = action(comargs, comdex)
+		except Exception as err:
+			answer = CRASHED(str(err))
+
+		sys.stdout.write( yaml.dump(answer, default_flow_style = False) )
+		if answer['reply']['status'][0] in ('1','2','3'):
+			sys.exit(0)
+		else:
+			sys.exit(1)
+
+	#----------------------
+	#-- basic operations  |
+	#----------------------
+	def do_help(self, comargs, comdex):
+		raise NotImplementedError
+
+	#----------------------
+	#-- backup operations |
+	#----------------------
+	def do_backup_asset(self, comargs, comdex):
+		ark   = ARK(comdex[2])
+		print(ark)
+		site  = self.site(ark.site)
+		vault = self.vault('fortress')
+		vault.provision(ark)
+		asset = site.asset(ark)
+		flag, stdout, stderr = vault.htar(asset)
+		if flag:
+			return SUCCESS({'stdout': stdout, 'stderr': stderr})
+		else:
+			return FAILED({'stdout': stdout, 'stderr': stderr})
+
+	#----------------------
+	#-- export operations |
+	#----------------------
+	def do_export_site_list(self, comargs, comdex):
+		name   = comdex[3]
+		vault  = self.vault(name)
+		sitels = list(vault.sites)
+		report = {
+			'vault': name,
+			'sites': list(vault.sites)
+		}
+		return SUCCESS(report)
+
+	def do_export_zone_assets(self, comargs, comdex):
+		spec   = comdex[3]
+		vname  = comdex[4]
+		ark    = ARK("[{}]".format(spec))
+		vault  = self.vault(vname)
+		zone   = ark.zone
+		assets = vault.assets(ark.site, ark.zone)
+		report = {
+			'vault': vname,
+			'site': ark.site,
+			'zone': ark.zone,
+			'assets': list(assets)
+		}
+		return SUCCESS(report)
+
+	def do_export_asset_manifest(self, comargs, comdex):
+		raise NotImplementedError
 
 
-CONF_SEARCH_ORDER = [
-	pathlib.Path('/etc/bastion'),
-	APP_PATH / 'etc',
-	pathlib.Path('~/.bastion').expanduser()
-]
 
 if __name__ == '__main__':
-	comargs = dict(enumerate(sys.argv[1:]))
-	subject = comargs.get(0, 'help')
-	request = comargs.get(1, 'help')
-
-	conf = Condex()
-	for folder in CONF_SEARCH_ORDER:
-		for confile in folder.rglob("conf-*.yaml"):
-			print(confile)
-			conf.load(folder / confile)
-
-	if subject == 'keytab':
-		if request == 'refresh':
-			raise NotImplementedError
-		else:
-			Keytab(conf).help()
-
-	if subject == 'backups':
-		sname = comargs[2]
-		site = Site(sname).configured(conf)
+	app = App().configured()
+	app.run( )
 
 
+#bastion site {site} backup
+#bastion zone {zone} backup
+#bastion manifest {zone}
 #bastion backups tidy idifhub
 #bastion backups update idifhub
 #bastion backups catalog idifhub
 #bastion keytab refresh fortress
+#bastion zone { } restore
+#bastion asset { } restore
+#bastion asset { } backup
+#bastion zone { } backup
+
+#bastion restore asset { }
+#bastion restore zone { }
+#bastion backup asset { }
+#bastion backup zone { }
+#bastion export zone assets { }
+#bastion export asset manifest { }
+#bastion export site list
+
+#bastion export asset manifest @idifhub:LiDAR/QL2023_IN_18 {fortress}
+#bastion export site list {fortress}
+#bastion export zone assets @idifhub:LiDAR {fortress}
+#bastion backup asset [@idifhub:LiDAR/QL2023_IN_18]

bin/test.py

@@ -0,0 +1,108 @@
+#!/usr/bin/env python3
+
+import sys
+import os
+import pathlib
+import logging
+
+logger = logging.getLogger()
+logging.basicConfig(level = logging.DEBUG)
+
+
+BIN_PATH = pathlib.Path(sys.argv[0]).absolute().parent
+APP_PATH = BIN_PATH.parent
+LIB_PATH = APP_PATH / 'lib'
+LBX_PATH = APP_PATH / 'lib-exec'
+
+sys.path.insert(0, str(LIB_PATH))
+
+from Bastion.Common  import *
+from Bastion.Site    import Site
+from Bastion.Condo   import Condex
+import Bastion.HPSS
+from Bastion.Curator import BLOND
+
+"""
+zone backup procedure...
+1. read conf file(s)
+2. connect to the storage vault
+3. retrieve manifest for given zone
+4. get most recent anchor snap
+5. compute drift between current datetime and most recent anchor snap
+6. perform backup
+	a. differential if drift < policy
+	b. anchor (full) if drift >= policy
+"""
+
+
+class App:
+	CONF_SEARCH_ORDER = [
+		pathlib.Path('/etc/bastion'),
+		APP_PATH / 'etc',
+		pathlib.Path('~/.bastion').expanduser()
+	]
+
+	def info(self, msg):
+		logger.info(msg)
+
+	def debug(self, msg):
+		logger.debug(msg)
+
+	def warn(self, msg):
+		logger.warn(msg)
+
+	def error(self, msg):
+		logger.error(msg)
+
+	def critical(self, msg):
+		logger.critical(msg)
+
+	def __init__(self):
+		self.conf = Condex()
+
+	def configured(self):
+		for folder in App.CONF_SEARCH_ORDER:
+			folder = folder.expanduser()
+			for confile in folder.rglob("conf-*.yaml"):
+				self.info("reading conf from {}".format(str(folder / confile)))
+				self.conf.load(folder / confile)
+		return self
+
+
+
+
+
+
+
+if __name__ == '__main__':
+	app = App().configured()
+	#app.run( )
+
+	vault = Bastion.HPSS.Vault('fortress').configured(app.conf)
+	site = Site('idifhub').configured(app.conf)
+
+	RzLiDAR = site.zones[0]
+	assets  = site.assets(RzLiDAR)
+	asset   = assets[0]
+
+
+
+#bastion site {site} backup
+#bastion zone {zone} backup
+#bastion manifest {zone}
+#bastion backups tidy idifhub
+#bastion backups update idifhub
+#bastion backups catalog idifhub
+#bastion keytab refresh fortress
+#bastion zone { } restore
+#bastion asset { } restore
+#bastion asset { } backup
+#bastion zone { } backup
+
+#bastion restore asset { }
+#bastion restore zone { }
+#bastion backup asset { }
+#bastion backup zone { }
+#bastion export zone manifest { }
+#bastion export asset manifest { }
+#bastion export site list 

etc/conf-idifhub.yaml

@@ -54,6 +54,7 @@ vaults:
     protocol: HPSS
     host: fortress.rcac.purdue.edu
     login: ndenny
+    root: /home/ndenny
     key:
       path: /home/ndenny/.private/hpss.unix.keytab
       refresh:

lib/Bastion/Actions.py

@@ -0,0 +1,5 @@
+"""
+Bastion.Actions
+
+I am the functions that execute the various actions of the bastion app.
+"""