diff --git a/bin/bastion.py b/bin/bastion.py index 305c575..3264037 100755 --- a/bin/bastion.py +++ b/bin/bastion.py @@ -6,6 +6,8 @@ import logging import socket import json +import datetime +import time from ruamel.yaml import YAML from ruamel.yaml.scalarstring import PreservedScalarString @@ -20,11 +22,12 @@ APP_PATH = BIN_PATH.parent LIB_PATH = APP_PATH / 'lib' LBX_PATH = APP_PATH / 'lib-exec' +IONICE = pathlib.Path("/usr/bin/ionice") sys.path.insert(0, str(LIB_PATH)) -from Bastion.Common import Boggle, asPath, toJDN, ALWAYS, RDN +from Bastion.Common import Boggle, asPath, toJDN, ALWAYS, RDN, HOURS from Bastion.Chronology import Quantim from Bastion.Site import Site from Bastion.Condo import Condex @@ -228,8 +231,10 @@ def run(self): ("refresh keytab", self.do_refresh_keytab), ("enroll assets", self.do_enroll_assets), - ("queue assets", self.do_queue_asset_updates), - ("bank queued assets", self.do_bank_queued_assets) + ("queue assets", self.do_queue_asset_updates), + ("bank queued assets", self.do_bank_queued_assets), + + ("become conductor", self.do_become_conductor) ] #-- Look for an explicitly declared session ID in opts; @@ -380,7 +385,7 @@ def do_bank_asset(self, request): request['log.scope'] = "site/{}".format(ark.site) result = vault.push(asset, client = self.hostname) - if result.indicates_success: + if result.succeeded: return request.succeeded(result, report = "pushed full backup of {} to {}".format(str(ark), str(result.blonde))) else: return request.failed(result, report = "while pushing full backup of {}, something went wrong!".format(str(ark))) @@ -960,9 +965,9 @@ def do_bank_queued_assets(self, request): else: logger.info("performing update for {}".format(slug)) ark = ARK(slug.decode('utf-8')) - command = "{}/bastion.py bank asset {}".format(str(BIN_PATH), str(ark)) - #os.system(command) - print(command) + command = '{}/bastion.py bank asset "{}"'.format(str(BIN_PATH), str(ark)) + logger.debug("executing... {}".format(command)) + os.system(command) updates.append(str(ark)) if updates: @@ -970,6 +975,54 @@ def do_bank_queued_assets(self, request): else: return request.succeeded(updates, report = "no updates to perform") + def do_become_conductor(self, request): + """ + I enter into a forever loop, napping for 15 second intervals, then waking to check the time and do any necessary housekeeping and backup operations. + """ + simulated = ('simulate!' in request.args.values()) or ('simulated!' in request.args.values()) + lastCheckIn = datetime.datetime(1, 1, 1, 0, 0, 0) #-- 1 JAN 1 CE + while ALWAYS: + now = datetime.datetime.now() + logger.debug("tick-tock {}".format(now.isoformat())) + if (now - lastCheckIn) < (24*HOURS): + time.sleep(15.0) #-- Sleep 15 seconds. + else: + #-- A brand new day! + for site in self.sites: + #-- Update the asset list for all of the sites that I'm managing... + for zone in site.zones: + if zone.dynamic: + ecommand = '{}/bastion.py enroll assets "{}"'.format(str(BIN_PATH), str(zone.ARK)) + if not simulated: + logger.debug("executing... {}".format(str(ecommand))) + os.system(ecommand) + else: + logger.debug("simulating... {}".format(str(ecommand))) + for site in self.sites: + #-- Queue assets for updates for all of the sites that I'm managing... + for zone in site.zones: + qcommand = '{}/bastion.py queue assets "{}"'.format(str(BIN_PATH), str(zone.ARK)) + logger.debug("executing... {}".format(str(qcommand))) + if not simulated: + logger.debug("simulating... {}".format(str(qcommand))) + os.system(qcommand) + else: + logger.debug("simulating... {}".format(str(qcommand))) + for site in self.sites: + #-- Spawn a subprocess to pop all of the objects in the queue for each siteXzone combo.... + for zone in site.zones: + bcommand = '{}/bastion.py bank queued assets "{}"'.format(str(BIN_PATH), str(zone.ARK)) + if IONICE: + #-- if IONICE is defined, then run the bank operations with IO niced to "Idle", + #-- i.e. disk access for banking is only done when no other processes need the disk. + bcommand = "{} -c 3 -t {}".format(str(IONICE), bcommand) + if not simulated: + logger.debug("executing... {}".format(str(bcommand))) + os.system(qcommand) + else: + logger.debug("simulating... {}".format(str(bcommand))) + lastCheckIn = datetime.datetime.now() + if __name__ == '__main__': app = App().configured() @@ -1038,3 +1091,4 @@ def do_bank_queued_assets(self, request): #bastion perform request filed #bastion perform request queued +#bastion become conductor \ No newline at end of file diff --git a/lib/Bastion/Common.py b/lib/Bastion/Common.py index 7da7b2f..c5764ba 100644 --- a/lib/Bastion/Common.py +++ b/lib/Bastion/Common.py @@ -271,7 +271,10 @@ def path(self): """ A Pure POSIX path object for the path relative to my zone. """ - return pathlib.PurePosixPath(self[1]) + if self[1] is not None: + return pathlib.PurePosixPath(self[1]) + else: + return None def __str__(self): if self.path: diff --git a/lib/Bastion/Model.py b/lib/Bastion/Model.py index 3c42392..560e348 100644 --- a/lib/Bastion/Model.py +++ b/lib/Bastion/Model.py @@ -44,8 +44,8 @@ def __new__(cls, *args): if len(args) == 3: site, zone, asset = args s = RDN(site) - z = RDN(zone) - a = asset.name if isinstance(asset, isAsset) else str(asset) + z = RDN(zone) if zone is not None else zone + a = asset.name if isinstance(asset, isAsset) else asset st = s if (s[0] == '@') else "@{}".format(s) return tuple.__new__(cls, [st, z, a]) @@ -82,7 +82,10 @@ def zolo(self): @property def CURIE(self): - return CURIE("@{}".format(self.site), self.zolo) + if self.zolo: + return CURIE("@{}".format(self.site), str(self.zolo)) + else: + return CURIE("@{}".format(self.site), None) @property def badge(self): diff --git a/lib/Bastion/Site.py b/lib/Bastion/Site.py index cc4cee8..2cbf723 100644 --- a/lib/Bastion/Site.py +++ b/lib/Bastion/Site.py @@ -440,6 +440,7 @@ def __init__(self, site, name, root = None): isZone.__init__(self, site, str(name), root) self.site = Site(site) self.policy = self.site.policy #--inherit my site's default policy + self.dynamic = False #-- when True, I should automatically enroll subfolders as assets. @property def assets(self): @@ -449,12 +450,19 @@ def configured(self, conf): if conf: if entity(conf).isString: #-- short form - self.root = pathlib.Path(conf) + p = pathlib.Path(conf) + if p.name == '*': + self.root = pathlib.Path(p.parent) + self.dynamic = True + else: + self.root = p else: #-- long form self.root = pathlib.Path(conf['root']) if 'policy' in conf: self.policy = RetentionPolicy(conf['policy']) + if 'dynamic' in conf: + self.dynamic = conf['dynamic'] return self def __div__(self, name):