Merge branch 'master' of github.itap.purdue.edu:ndenny/bastion

bin/bastion.py

@@ -6,6 +6,8 @@
 import logging
 import socket
 import json
+import datetime
+import time
 
 from ruamel.yaml import YAML
 from ruamel.yaml.scalarstring import PreservedScalarString
@@ -20,11 +22,12 @@
 APP_PATH = BIN_PATH.parent
 LIB_PATH = APP_PATH / 'lib'
 LBX_PATH = APP_PATH / 'lib-exec'
+IONICE   = pathlib.Path("/usr/bin/ionice")
 
 sys.path.insert(0, str(LIB_PATH))
 
 
-from Bastion.Common     import Boggle, asPath, toJDN, ALWAYS, RDN
+from Bastion.Common     import Boggle, asPath, toJDN, ALWAYS, RDN, HOURS
 from Bastion.Chronology import Quantim
 from Bastion.Site       import Site
 from Bastion.Condo      import Condex
@@ -228,8 +231,10 @@ def run(self):
 			("refresh keytab",              self.do_refresh_keytab),
 			("enroll assets",               self.do_enroll_assets),
 
-			("queue assets",				self.do_queue_asset_updates),
-			("bank queued assets",			self.do_bank_queued_assets)
+			("queue assets",                self.do_queue_asset_updates),
+			("bank queued assets",          self.do_bank_queued_assets),
+
+			("become conductor",            self.do_become_conductor)
 		]
 
 		#-- Look for an explicitly declared session ID in opts;
@@ -380,7 +385,7 @@ def do_bank_asset(self, request):
 		request['log.scope'] = "site/{}".format(ark.site)
 		result = vault.push(asset, client = self.hostname)
 
-		if result.indicates_success:
+		if result.succeeded:
 			return request.succeeded(result, report = "pushed full backup of {} to {}".format(str(ark), str(result.blonde)))
 		else:
 			return request.failed(result, report = "while pushing full backup of {}, something went wrong!".format(str(ark)))
@@ -960,16 +965,64 @@ def do_bank_queued_assets(self, request):
 			else:
 				logger.info("performing update for {}".format(slug))
 				ark = ARK(slug.decode('utf-8'))
-				command = "{}/bastion.py bank asset {}".format(str(BIN_PATH), str(ark))
-				#os.system(command)
-				print(command)
+				command = '{}/bastion.py bank asset "{}"'.format(str(BIN_PATH), str(ark))
+				logger.debug("executing... {}".format(command))
+				os.system(command)
 				updates.append(str(ark))
 
 		if updates:
 			return request.succeeded(updates, report = "performed {} updates ({} expected)".format(len(updates), len(zone.assets)))
 		else:
 			return request.succeeded(updates, report = "no updates to perform")
 
+	def do_become_conductor(self, request):
+		"""
+		I enter into a forever loop, napping for 15 second intervals, then waking to check the time and do any necessary housekeeping and backup operations.
+		"""
+		simulated = ('simulate!' in request.args.values()) or ('simulated!' in request.args.values())
+		lastCheckIn = datetime.datetime(1, 1, 1, 0, 0, 0)  #-- 1 JAN 1 CE
+		while ALWAYS:
+			now = datetime.datetime.now()
+			logger.debug("tick-tock {}".format(now.isoformat()))
+			if (now - lastCheckIn) < (24*HOURS):
+				time.sleep(15.0) #-- Sleep 15 seconds.
+			else:
+				#-- A brand new day!
+				for site in self.sites:
+					#-- Update the asset list for all of the sites that I'm managing...
+					for zone in site.zones:
+						if zone.dynamic:
+							ecommand = '{}/bastion.py enroll assets "{}"'.format(str(BIN_PATH), str(zone.ARK))
+							if not simulated:
+								logger.debug("executing... {}".format(str(ecommand)))
+								os.system(ecommand)
+							else:
+								logger.debug("simulating... {}".format(str(ecommand)))
+				for site in self.sites:
+					#-- Queue assets for updates for all of the sites that I'm managing...
+					for zone in site.zones:
+						qcommand = '{}/bastion.py queue assets "{}"'.format(str(BIN_PATH), str(zone.ARK))
+						logger.debug("executing... {}".format(str(qcommand)))
+						if not simulated:
+							logger.debug("simulating... {}".format(str(qcommand)))
+							os.system(qcommand)
+						else:
+							logger.debug("simulating... {}".format(str(qcommand)))
+				for site in self.sites:
+					#-- Spawn a subprocess to pop all of the objects in the queue for each siteXzone combo....
+					for zone in site.zones:
+						bcommand = '{}/bastion.py bank queued assets "{}"'.format(str(BIN_PATH), str(zone.ARK))
+						if IONICE:
+							#-- if IONICE is defined, then run the bank operations with IO niced to "Idle",
+							#-- i.e. disk access for banking is only done when no other processes need the disk.
+							bcommand = "{} -c 3 -t {}".format(str(IONICE), bcommand)
+						if not simulated:
+							logger.debug("executing... {}".format(str(bcommand)))
+							os.system(bcommand)
+						else:
+							logger.debug("simulating... {}".format(str(bcommand)))
+				lastCheckIn = datetime.datetime.now()
+
 
 if __name__ == '__main__':
 	app = App().configured()
@@ -1038,3 +1091,4 @@ def do_bank_queued_assets(self, request):
 
 #bastion perform request filed
 #bastion perform request queued
+#bastion become conductor

bin/go.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/bash
+
+#-- Launch redis server in daemon mode.
+/usr/bin/redis-server --daemonize yes
+
+#-- redis should launch and be ready almost instantly;
+#-- however, just to be sure, we'll put a slight delay here.
+sleep 3
+
+#-- Launch bastion as the "conductor" process.
+#-- Under normal circumstances, this process doesn't end.
+/opt/bastion/bin/bastion.py become conductor
+

lib/Bastion/Common.py

@@ -271,7 +271,10 @@ def path(self):
 		"""
 		A Pure POSIX path object for the path relative to my zone.
 		"""
-		return pathlib.PurePosixPath(self[1])
+		if self[1] is not None:
+			return pathlib.PurePosixPath(self[1])
+		else:
+			return None
 
 	def __str__(self):
 		if self.path:

lib/Bastion/Model.py

@@ -44,8 +44,8 @@ def __new__(cls, *args):
 		if len(args) == 3:
 			site, zone, asset = args
 			s  = RDN(site)
-			z  = RDN(zone)
-			a  = asset.name if isinstance(asset, isAsset) else str(asset)
+			z  = RDN(zone) if zone is not None else zone
+			a  = asset.name if isinstance(asset, isAsset) else asset
 			st = s if (s[0] == '@') else "@{}".format(s)
 			return tuple.__new__(cls, [st, z, a])
 
@@ -82,7 +82,10 @@ def zolo(self):
 
 	@property
 	def CURIE(self):
-		return CURIE("@{}".format(self.site), self.zolo)
+		if self.zolo:
+			return CURIE("@{}".format(self.site), str(self.zolo))
+		else:
+			return CURIE("@{}".format(self.site), None)
 
 	@property
 	def badge(self):

lib/Bastion/Site.py

@@ -440,6 +440,7 @@ def __init__(self, site, name, root = None):
 		isZone.__init__(self, site, str(name), root)
 		self.site    = Site(site)
 		self.policy  = self.site.policy #--inherit my site's default policy
+		self.dynamic = False  #-- when True, I should automatically enroll subfolders as assets.
 
 	@property
 	def assets(self):
@@ -449,12 +450,19 @@ def configured(self, conf):
 		if conf:
 			if entity(conf).isString:
 				#-- short form
-				self.root = pathlib.Path(conf)
+				p = pathlib.Path(conf)
+				if p.name == '*':
+					self.root = pathlib.Path(p.parent)
+					self.dynamic = True
+				else:
+					self.root = p
 			else:
 				#-- long form
 				self.root = pathlib.Path(conf['root'])
 				if 'policy' in conf:
 					self.policy = RetentionPolicy(conf['policy'])
+				if 'dynamic' in conf:
+					self.dynamic = conf['dynamic']
 		return self
 
 	def __div__(self, name):