diff --git a/bin/bastion.py b/bin/bastion.py index 836f81e..b815e92 100755 --- a/bin/bastion.py +++ b/bin/bastion.py @@ -27,9 +27,10 @@ from Bastion.Site import Site from Bastion.Condo import Condex from Bastion.Model import ARK -from Bastion.CARP import Request, isReceipt +from Bastion.CARP import Request, isReceipt, isReport import Bastion.Vaults.HPSS import Bastion.Vaults.BFD +import Bastion.Vaults.SFTP """ zone backup procedure... @@ -240,7 +241,7 @@ def remember(self, answer): halo = scope / "{}.yaml".format(session) halo.parent.mkdir(parents = True, exist_ok = True) with open(halo, 'wt') as fout: - self.emit_YAML(answer, fout) + self.emit_YAML(answer.toJDN(), fout) def emit(self, answer, ostream = None): ostream = ostream if (ostream is not None) else sys.stdout @@ -256,7 +257,6 @@ def emit(self, answer, ostream = None): def emit_YAML(self, answer, ostream): yaml = YAML() yaml.default_flow_style = False - answer.report = PreservedScalarString(answer.report) yaml.dump(answer, ostream) def emit_JSON(self, answer, ostream): @@ -365,13 +365,13 @@ def do_bank_asset(self, request): vault = self.vault(asset.policy.vault) request['log.scope'] = "site/{}".format(ark.site) - receipt = vault.push(asset, client = self.hostname) + result = vault.push(asset, client = self.hostname) - if receipt.indicates_success: - blonde = receipt.body['blonde'] - return request.succeeded("pushed full backup of {} to {}".format(str(ark), str(blonde)), receipt) + if result.indicates_success: + blonde = result.record['blonde'] + return request.succeeded(result, report = "pushed full backup of {} to {}".format(str(ark), str(blonde))) else: - return request.failed("while pushing full backup of {}, something went wrong!".format(str(ark)), receipt) + return request.failed(result, report = "while pushing full backup of {}, something went wrong!".format(str(ark))) #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ #-- END bank (backup) operations | diff --git a/bin/test.py b/bin/test.py index a2d8616..42df2ac 100755 --- a/bin/test.py +++ b/bin/test.py @@ -1,9 +1,11 @@ + #!/usr/bin/env python3 import sys import os import pathlib import logging +import socket logger = logging.getLogger() logging.basicConfig(level = logging.DEBUG) @@ -16,93 +18,21 @@ sys.path.insert(0, str(LIB_PATH)) -from Bastion.Common import * -from Bastion.Site import Site -from Bastion.Condo import Condex -import Bastion.HPSS -from Bastion.Curator import BLOND - -""" -zone backup procedure... -1. read conf file(s) -2. connect to the storage vault -3. retrieve manifest for given zone -4. get most recent anchor snap -5. compute drift between current datetime and most recent anchor snap -6. perform backup - a. differential if drift < policy - b. anchor (full) if drift >= policy -""" - - -class App: - CONF_SEARCH_ORDER = [ - pathlib.Path('/etc/bastion'), - APP_PATH / 'etc', - pathlib.Path('~/.bastion').expanduser() - ] - - def info(self, msg): - logger.info(msg) - - def debug(self, msg): - logger.debug(msg) - - def warn(self, msg): - logger.warn(msg) - - def error(self, msg): - logger.error(msg) - - def critical(self, msg): - logger.critical(msg) - - def __init__(self): - self.conf = Condex() - - def configured(self): - for folder in App.CONF_SEARCH_ORDER: - folder = folder.expanduser() - for confile in folder.rglob("conf-*.yaml"): - self.info("reading conf from {}".format(str(folder / confile))) - self.conf.load(folder / confile) - return self - - - - - - +from bastion import App if __name__ == '__main__': app = App().configured() - #app.run( ) - - vault = Bastion.HPSS.Vault('fortress').configured(app.conf) - site = Site('idifhub').configured(app.conf) - - RzLiDAR = site.zones[0] - assets = site.assets(RzLiDAR) - asset = assets[0] - + host = os.environ.get('BASTION_SITE', socket.gethostname()) -#bastion site {site} backup -#bastion zone {zone} backup -#bastion manifest {zone} -#bastion backups tidy idifhub -#bastion backups update idifhub -#bastion backups catalog idifhub -#bastion keytab refresh fortress -#bastion zone { } restore -#bastion asset { } restore -#bastion asset { } backup -#bastion zone { } backup + if host == 'rusina': + rusina = app.site('rusina') + soundscapes = rusina.zone('soundscapes') + asset = soundscapes['HackathonData'] + vault = app.vault(asset.policy.vault) + elif host == 'scout': + scout = app.site('scout') + ADS = scout.zone('ADS') + asset = ADS['Titanic3'] + vault = app.vault(asset.policy.vault) -#bastion restore asset { } -#bastion restore zone { } -#bastion backup asset { } -#bastion backup zone { } -#bastion export zone manifest { } -#bastion export asset manifest { } -#bastion export site list diff --git a/lib/Bastion/CARP.py b/lib/Bastion/CARP.py index 7e5796e..845ae46 100644 --- a/lib/Bastion/CARP.py +++ b/lib/Bastion/CARP.py @@ -22,6 +22,7 @@ import datetime import uuid import traceback +import pprint import yaml @@ -181,7 +182,8 @@ def crashed(self, record = None, *args, **kwargs): def inconclusive(self, record = None, *args, **kwargs): return Report.Inconclusive(self, record, *args, **kwargs) - +#-- legacy alias +Request = isRequest class isResult: def __init__(self, request, status, record, **kwargs): @@ -220,11 +222,13 @@ def toJDN(self, **kwargs): 'status': self.status.code, 'message': self.status.gloss, 'lede': self.lede, + 'answers': self.request.ID, 'answered': self.when.isoformat( ), 'elapsed': self.elapsed.total_seconds(), 'context': { }, 'record': toJDN(self.record) if self.record else None } + #-- Populate the result's context (if any) for k, v in self.context.items( ): result['context'][k] = toJDN(v) @@ -232,7 +236,7 @@ def toJDN(self, **kwargs): #-- Construct the JDN for return jdn = { } jdn['request'] = toJDN(self.request) - jdn['request']['result'] = result + jdn['result'] = result return jdn @@ -257,20 +261,15 @@ def __init__(self, request, status, record = None, *args, **kwargs): @property def body(self): if getattr(self, '_body', None) is None: - self._body = self.toMD() + if callable(getattr(self.record, 'toJDN', None)): + self._body = pprint.pformat(self.record.toJDN()) + else: + self._body = pprint.pformat(self.record) return self._body - def toMD(self): - """ - I answer a markdown representation of self. - Base class is naive and just the string representation of my record. - Override in subclasses for detailed output. - """ - return str(self.record) - def toJDN(self, **kwargs): - jdn = isReceipt.toJDN(self, **kwargs) - jdn['request']['result']['report'] = self.body + jdn = isResult.toJDN(self, **kwargs) + jdn['result']['report'] = self.body return jdn def __str__(self): @@ -346,6 +345,9 @@ def __init__(self, results): else: raise ValueError("result must be an instance of Bastion.CARP.isResult") + def __getitem__(self, i): + return self.results[i] + def __iter__(self): return iter(self.results) diff --git a/lib/Bastion/Common.py b/lib/Bastion/Common.py index 1a64030..8cd01af 100644 --- a/lib/Bastion/Common.py +++ b/lib/Bastion/Common.py @@ -413,3 +413,9 @@ def asPurePath(x): Accessor/wrapper for answering x already cast as a pathlib.PurePosixPath instance """ return pathlib.PurePosixPath(x) + +def asInteger(x): + return int(x) + +def asNumeric(x): + return float(x) diff --git a/lib/Bastion/Curator.py b/lib/Bastion/Curator.py index dbdd2eb..ec51ced 100644 --- a/lib/Bastion/Curator.py +++ b/lib/Bastion/Curator.py @@ -5,12 +5,12 @@ """ import datetime -from Bastion.Common import Thing, Boggle, canTextify +from Bastion.Common import Thing, Boggle, canStruct from Bastion.Model import ARK, isAsset from Bastion.Chronology import Quantim -class BLONDE(canTextify): +class BLONDE: """ BLOb Name and Description Encoding I am a structured name describing a point in time for a single ARK. @@ -64,6 +64,9 @@ def encode(badge, when, detail, genus): def __str__(self): return self.encode(self.badge, self.when, self.detail, self.genus) + def toJDN(self, **kwargs): + return str(self) + @classmethod def decode(cls, blonde): ds = Thing(**{ diff --git a/lib/Bastion/Movers/SFTP.py b/lib/Bastion/Movers/SFTP.py index e0f0aa7..7a288f2 100644 --- a/lib/Bastion/Movers/SFTP.py +++ b/lib/Bastion/Movers/SFTP.py @@ -2,20 +2,13 @@ import logging import Bastion.Model +from Bastion.Common import asPath, asPurePath from Bastion.NetOps.sCURL import SCURLer +from Bastion.Movers.CARP import PutRequest, PutReceipt logger = logging.getLogger(__name__) -def asPath(x): - return pathlib.Path(x) - -def asPurePath(x): - return pathlib.PurePosixPath(x) - - - - class Mover(Bastion.Model.isMover): def __init__(self, vault, **kwargs): Bastion.Model.isMover.__init__(self, vault) @@ -36,8 +29,26 @@ def provision(self, *args): raise NotImplementedError def put(self, halo, tag, **kwargs): - raise NotImplementedError - + here = halo + there = self.vault.sfURL / tag + logger.debug( "put source {} to {}".format(here.as_uri(), str(there)) ) + + #-- Create the put request. + request = PutRequest(halo, str(self.vault.sfURL), tag) + + try: + #-- execute sCURL operation, here. + copied = self.scurler.put(halo, tag) + except Exception as err: + #-- shutil should throw an error if there was a problem copying. + report = request.failed(err) + else: + if copied: + report = request.succeeded( PutReceipt(halo, str(self.vault.sfURL), tag) ) + else: + report = request.failed( PutReceipt(halo, str(self.vault.sfURL), tag), report = "curl upload operation failed" ) + + return report def get(self, tag, halo, **kwargs): raise NotImplementedError diff --git a/lib/Bastion/NetOps/sCURL.py b/lib/Bastion/NetOps/sCURL.py index a5a9a51..af1df55 100644 --- a/lib/Bastion/NetOps/sCURL.py +++ b/lib/Bastion/NetOps/sCURL.py @@ -31,13 +31,6 @@ } -#class isRemotePosixFileSystem: -# def ls(self, *args, **kwargs): -# raise NotImplementedError -# -# def - - class sfURL(tuple): """ secure file URL @@ -319,6 +312,10 @@ def __init__(self, *args, **kwargs): self.mkdirs = kwargs.get('mkdirs', True) #-- default to creating missing directories in upload paths. self.CURL = kwargs.get('curl', "/usr/bin/curl") + self.CURL = pathlib.Path(self.CURL) + + assert self.CURL.exists(), "local path to curl command ({}) does not exist".format(self.CURL) + self.lastop = None #------------------------------------------------------------------------- @@ -326,7 +323,7 @@ def __init__(self, *args, **kwargs): #-- All SFTP operations will be built by augmenting the base command ... | #-- ... with additional flags and arguments. | #------------------------------------------------------------------------- - basecom = [self.CURL] + basecom = [str(self.CURL)] if self.keypath is not None: basecom.extend(['--key', str(self.keypath)]) if self.silent: diff --git a/lib/Bastion/Site.py b/lib/Bastion/Site.py index 94b5b2a..581465f 100644 --- a/lib/Bastion/Site.py +++ b/lib/Bastion/Site.py @@ -6,7 +6,7 @@ import random import datetime -from Bastion.Common import entity, Thing, canStruct, DAYS, RDN, asPath, asLogLevel +from Bastion.Common import entity, Thing, canStruct, DAYS, RDN, asPath, asLogLevel, canConStruct from Bastion.Condo import CxNode from Bastion.Model import ARK, isAsset, isZone, isSite #from .Curator import Asset diff --git a/lib/Bastion/Vaults/BFD.py b/lib/Bastion/Vaults/BFD.py index 08f22a6..1e1d109 100644 --- a/lib/Bastion/Vaults/BFD.py +++ b/lib/Bastion/Vaults/BFD.py @@ -50,12 +50,6 @@ def clerk(self): self._clerk = Bastion.Clerks.BFD.Clerk(self) return self._clerk - @property - def packer(self): - if getattr(self, '_packer', None) is None: - self._packer = Bastion.Packers.TARs.Packer(self) - return self._packer - @property def mover(self): if getattr(self, '_mover', None) is None: diff --git a/lib/Bastion/Vaults/CARP.py b/lib/Bastion/Vaults/CARP.py index 6e42f89..f822b01 100644 --- a/lib/Bastion/Vaults/CARP.py +++ b/lib/Bastion/Vaults/CARP.py @@ -55,7 +55,7 @@ def __init__(self, subject, *args, **kwargs): 'halo': str(asset.halo) } - isRequest.__init__(self, "PackRequest", asset, **xtras) + isRequest.__init__(self, "PushRequest", asset, **xtras) @property def asset(self): diff --git a/lib/Bastion/Vaults/Common.py b/lib/Bastion/Vaults/Common.py index 28aa9fc..e1647e4 100644 --- a/lib/Bastion/Vaults/Common.py +++ b/lib/Bastion/Vaults/Common.py @@ -19,6 +19,12 @@ class isSpoolingVault(Bastion.Model.isVault): def __init__(self, name, **kwargs): Bastion.Model.Vault.__init__(self, name, **kwargs) + @property + def packer(self): + if getattr(self, '_packer', None) is None: + self._packer = Bastion.Packers.TARs.Packer(self) + return self._packer + def push(self, asset, basis = None, **kwargs): """ Given an asset, I push a backup of the asset to this vault. @@ -39,8 +45,8 @@ def push(self, asset, basis = None, **kwargs): return request.failed( PushReceipt([packing, movement]) ) #-- clean up! - if packing.record.tarp.exists(): - packing.record.tarp.unlink() + #if packing.record.tarp.exists(): + # packing.record.tarp.unlink() return request.succeeded( PushReceipt([packing, movement]) ) diff --git a/lib/Bastion/Vaults/SFTP.py b/lib/Bastion/Vaults/SFTP.py index 706728c..b4e68f4 100644 --- a/lib/Bastion/Vaults/SFTP.py +++ b/lib/Bastion/Vaults/SFTP.py @@ -2,7 +2,7 @@ import logging import getpass -from Bastion.Common import asPath, asPurePath +from Bastion.Common import asPath, asPurePath, asInteger import Bastion.Model import Bastion.Movers.SFTP import Bastion.Clerks.SFTP @@ -15,14 +15,14 @@ class Vault(Bastion.Vaults.Common.isSpoolingVault): PROTOCOL = 'SFTP' def __init__(self, name, **kwargs): - Bastion.Model.Vault.__init__(self, name, **kwargs) + Bastion.Vaults.Common.isSpoolingVault.__init__(self, name, **kwargs) self.condex = None self.scratch = pathlib.Path("/tmp") self.host = None + self.port = 22 self.login = getpass.getuser() self.keypath = pathlib.Path("~/.ssh/id") - self.bank = None self.root = pathlib.PurePosixPath("/") #-- the default remote root folder self._packer = None #-- cached instance of Packer @@ -40,12 +40,20 @@ def configured(self, conf): #-- Configuration relevant to remote (bank) host. self.host = section.get('bank.host') + self.port = section.get(asInteger, 'bank.port', 22) self.login = section.get('bank.login', getpass.getuser()) - self.keypath = section.get(asPath, 'bank.key', pathlib.Path("~/.ssh/id").expanduser()) + self.keypath = section.get(asPath, 'bank.key', pathlib.Path("~/.ssh/ID").expanduser()) self.root = section.get(asPurePath, 'bank.root', "/") return self + @property + def sfURL(self): + """ + I am an instance of Bastion.NetOps.sCURL.sfURL (secure file URL) that point to the root of my bank. + """ + return Bastion.NetOps.sCURL.sfURL(self.host, self.root, user = self.login, port = self.port) + @property def mover(self): if getattr(self, '_mover', None) is None: