From d41dc7cd828a70905b02f6cf5e4552de42655942 Mon Sep 17 00:00:00 2001 From: Nathan Denny Date: Sat, 4 Jan 2025 06:35:46 -0500 Subject: [PATCH] lots of updates, still trying to stabilize the semantics of the CARP action-result-report --- lib/Bastion/CARP.py | 90 ++++++++++++---- lib/Bastion/Chronology.py | 11 +- lib/Bastion/Clerks/BFD.py | 40 ++----- lib/Bastion/Clerks/SFTP.py | 23 ++-- lib/Bastion/Common.py | 35 +++--- lib/Bastion/Condo.py | 92 ++++++++-------- lib/Bastion/Curator.py | 8 +- lib/Bastion/Model.py | 97 +++++++---------- lib/Bastion/Movers/BFD.py | 130 ++++------------------ lib/Bastion/Movers/CARP.py | 155 ++++++++++++-------------- lib/Bastion/Movers/SFTP.py | 62 +---------- lib/Bastion/NetOps/sCURL.py | 3 +- lib/Bastion/Packers/CARP.py | 28 ++--- lib/Bastion/Packers/TARs.py | 14 +-- lib/Bastion/Site.py | 13 +-- lib/Bastion/Vaults/BFD.py | 204 +++++------------------------------ lib/Bastion/Vaults/CARP.py | 82 ++++++++++++++ lib/Bastion/Vaults/Common.py | 95 ++++++---------- lib/Bastion/Vaults/SFTP.py | 95 ++++++---------- 19 files changed, 502 insertions(+), 775 deletions(-) create mode 100644 lib/Bastion/Vaults/CARP.py diff --git a/lib/Bastion/CARP.py b/lib/Bastion/CARP.py index 2b3d167..189cf21 100644 --- a/lib/Bastion/CARP.py +++ b/lib/Bastion/CARP.py @@ -2,8 +2,24 @@ Bastion.CARP Common Action-Result/Report Protocol + +Basic concepts... +requests (instances of isRequest or subclasses) encapsulates details of the action requested. +receipts contain details of the action's execution +results encapsulate the request and the receipt +reports are results with some extra features for output for human readability +... +Of these concepts, the two most important are request and result. + +Basic plan for using CARP in applications... +Within an action method... +1. generate request +2. create an object with details of the action's success or failure (a receipt) +3. use one of the request reporting methods to generate a response report (e.g. report = request.failed(log) ) +4. return the report. """ import operator +import datetime import uuid import traceback @@ -153,6 +169,19 @@ def toJDN(self, **kwargs): return jdn + def succeeded(self, record = None, *args, **kwargs): + return Report.Success(self, record, *args, **kwargs) + + def failed(self, record = None, *args, **kwargs): + return Report.Failed(self, record, *args, **kwargs) + + def crashed(self, record = None, *args, **kwargs): + return Report.Crashed(self, record, *args, **kwargs) + + def inconclusive(self, record = None, *args, **kwargs): + return Report.Inconclusive(self, record, *args, **kwargs) + + class isResult: def __init__(self, request, status, record, **kwargs): @@ -176,6 +205,7 @@ def __init__(self, request, status, record, **kwargs): self.context = { } self.when = kwargs.get('when', datetime.datetime.now()) + context = kwargs.get('context', None) if context is not None: self.context = context.copy() @@ -219,9 +249,9 @@ def inconclusive(self): return self.status.is_inconclusive -class Report(isResult): +class isReport(isResult): def __init__(self, request, status, record = None, *args, **kwargs): - isReceipt.__init__(self, request, status, record, *args, **kwargs) + isResult.__init__(self, request, status, record, *args, **kwargs) self._body = kwargs.get('report', None) @property @@ -233,10 +263,10 @@ def body(self): def toMD(self): """ I answer a markdown representation of self. - Base class is naive and just returns an empty string. + Base class is naive and just the string representation of my record. Override in subclasses for detailed output. """ - return "" + return str(self.record) def toJDN(self, **kwargs): jdn = isReceipt.toJDN(self, **kwargs) @@ -246,18 +276,24 @@ def toJDN(self, **kwargs): def __str__(self): return yaml.dump(self.toJDN(), default_flow_style = False, indent = 3, sort_keys = True) + + +class Report(isReport): @classmethod def Success(cls, request, record = None, *args, **kwargs): return cls(request. ResultStatus.Ok, record, *args, **kwargs) @classmethod def Failed(cls, request, record = None, *args, **kwargs): - return cls(request, ResultStatus.Failed, record, *args, **kwargs) + if isinstance(record, Exception): + return ExceptionReport(request, ResultStatus.Failed, record, *args, **kwargs) + else: + return cls(request, ResultStatus.Failed, record, *args, **kwargs) @classmethod def Crashed(cls, request, record = None, *args, **kwargs): if isinstance(record, Exception): - return ReportException(request, ResultStatus.Crashed, record, *args, **kwargs) + return ExceptionReport(request, ResultStatus.Crashed, record, *args, **kwargs) else: return cls(request, ResultStatus.Crashed, record, *args, **kwargs) @@ -266,7 +302,7 @@ def Inconclusive(cls, request, record = None, *args, **kwargs): return cls(request, ResultStatus.Inconclusive, record, *args, **kwargs) -class ReportException(Report): +class ExceptionReport(isReport): def __init__(self, request, *args, **kwargs): #-- Can be called as... #-- ReportException(request, errobj), or... @@ -275,7 +311,7 @@ def __init__(self, request, *args, **kwargs): status = args[0] errobj = args[1] else: - status = ReplyStatus.Failed + status = ReplyStatus.Fault errobj = args[0] assert status.indicates_failure, "ReportException must be created with a reply status indicating failure" @@ -287,15 +323,31 @@ def __init__(self, request, *args, **kwargs): -class Request(isRequest): - def succeeded(self, record = None, *args, **kwargs): - return Report.Success(self, record, *args, **kwargs) - - def failed(self, record = None, *args, **kwargs): - return Report.Failed(self, record, *args, **kwargs) - - def crashed(self, record = None, *args, **kwargs): - return Report.Crashed(self, record, *args, **kwargs) +class isReceipt: + """ + I am an abstract base class for receipts. + Mostly, I'm used for run-time type checking. + """ + def toJDN(self, **kwargs): + raise NotImplementedError("is subclass responsibility") + + +class isReceiptOfResults(isReceipt): + """ + I am a thin wrapper around a sequence of results. + I am mostly used to build a single "record" from a sequence of operations, + with each operation having its own report. + """ + def __init__(self, results): + self.results = [ ] + for result in results: + if isinstance(result, isResult): + self.results.append(result) + else: + raise ValueError("result must be an instance of Bastion.CARP.isResult") + + def __iter__(self): + return iter(self.results) - def inconclusive(self, record = None, *args, **kwargs): - return Report.Inconclusive(self, record, *args, **kwargs) + def toJDN(self, **kwargs): + return [result.toJDN(**kwargs) for result in self.results] diff --git a/lib/Bastion/Chronology.py b/lib/Bastion/Chronology.py index c4d8522..e632ea3 100644 --- a/lib/Bastion/Chronology.py +++ b/lib/Bastion/Chronology.py @@ -4,7 +4,7 @@ import string import datetime import logging -from .Common import * +from Bastion.Common import SECONDS logger = logging.getLogger(__name__) @@ -87,10 +87,9 @@ def __init__(self, whence, separator = None): elif isinstance(whence, str): if self.separator: words = whence.split(self.separator) - if len(words) == 3: - yW = words[0] - dW = words[1] - qW = words[2] + if len(words) == 4: + wY = words[0] + wDMQ = ''.join(words[1:]) else: raise Exception("Quantim:__init__ parse error for '{}'".format(whence)) else: @@ -118,7 +117,6 @@ def __init__(self, whence, separator = None): def __str__(self): lsq = self.qM % 36 msq = self.qM // 36 - xmap = list(string.digits + string.ascii_uppercase) sY = "{:03d}".format(self.dY) sM = "{:X}".format(self.dM) sD = "{:02d}".format(self.dD) @@ -137,7 +135,6 @@ def quaver(self): lsq = self.qM % 36 msq = self.qM // 36 - xmap = list(string.digits + string.ascii_uppercase) sY = "{:03d}".format(self.dY) sM = "{:X}".format(self.dM) sD = "{:02d}".format(self.dD) diff --git a/lib/Bastion/Clerks/BFD.py b/lib/Bastion/Clerks/BFD.py index 0e2819e..11a1de4 100644 --- a/lib/Bastion/Clerks/BFD.py +++ b/lib/Bastion/Clerks/BFD.py @@ -1,40 +1,22 @@ -import os import pathlib -import subprocess -import operator -import datetime -import json -import socket -import logging -import getpass -import shutil -import tarfile import logging -from Bastion.Common import Thing, Unknown -from Bastion.Model import isAsset, isClerk -from Bastion.Curator import Manifest, BLONDE, Snap -from Bastion.Utils import pax +from Bastion.Common import RDN +from Bastion.Model import ARK, isVault, isAsset, isClerk +from Bastion.Curator import Manifest, BLONDE logger = logging.getLogger(__name__) class Clerk(isClerk): - def __init__(self, vault, **kwargs): + def __init__(self, vault, root, **kwargs): isClerk.__init__(self) - self.vault = vault - - @property - def bfd(self): - return self.vault.bfd - @property - def bank(self): - return self.vault.bank + assert isinstance(vault, isVault), "vault must be an instance of Bastion.Model.isVault" + assert isinstance(root, pathlib.PosixPath), "root must be an instance of PosixPath" - @property - def scratch(self): - return self.vault.scratch + self.vault = vault + self.root = root #----------------------------------------- #-- BEGIN Bastion.Model.isClerk PROTOCOL | @@ -49,7 +31,6 @@ def sites(self): for entry in self.root.iterdir(): if entry.is_dir(): sites.append(entry.name) - return tuple(sorted(sites)) def zones(self, site): @@ -84,11 +65,12 @@ def manifest(self, *args): ark = None if len(args) == 1: arg = args[0] - if isinstance(arg, Bastion.Model.isAsset): + if isinstance(arg, isAsset): ark = arg.ARK else: ark = arg elif len(args) == 3: + site, zone, asset = args ark = ARK(site, zone, asset) else: raise ValueError @@ -99,7 +81,7 @@ def manifest(self, *args): for item in cell.iterdir(): if not item.is_dir(): blondes.append( BLONDE.decode(item.stem) ) - manifest = Bastion.Curator.Manifest(ark, blondes) + manifest = Manifest(ark, blondes) return manifest #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ diff --git a/lib/Bastion/Clerks/SFTP.py b/lib/Bastion/Clerks/SFTP.py index 8879fa6..9a6edd2 100644 --- a/lib/Bastion/Clerks/SFTP.py +++ b/lib/Bastion/Clerks/SFTP.py @@ -1,20 +1,8 @@ -import os -import pathlib -import subprocess -import operator -import datetime -import json -import socket -import logging -import getpass -import shutil -import tarfile import logging -from Bastion.Common import Thing, Unknown -from Bastion.Model import isAsset, isClerk -from Bastion.Curator import Manifest, BLONDE, Snap -from Bastion.Packers.TARs import pax +from Bastion.Common import RDN +from Bastion.Model import ARK, isAsset, isClerk +from Bastion.Curator import Manifest, BLONDE from Bastion.NetOps.sCURL import SCURLer @@ -75,11 +63,12 @@ def manifest(self, *args): ark = None if len(args) == 1: arg = args[0] - if isinstance(arg, Bastion.Model.isAsset): + if isinstance(arg, isAsset): ark = arg.ARK else: ark = arg elif len(args) == 3: + site, zone, asset = args ark = ARK(site, zone, asset) else: raise ValueError @@ -90,7 +79,7 @@ def manifest(self, *args): for alien in cell.lsall(): if not alien.is_dir(): blondes.append( BLONDE.decode(alien.stem) ) - manifest = Bastion.Curator.Manifest(ark, blondes) + manifest = Manifest(ark, blondes) return manifest #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ diff --git a/lib/Bastion/Common.py b/lib/Bastion/Common.py index 89b616e..caabbed 100644 --- a/lib/Bastion/Common.py +++ b/lib/Bastion/Common.py @@ -11,6 +11,7 @@ import random import pathlib import logging +import urllib import yaml @@ -58,25 +59,25 @@ class Alchemist(tuple): I am an extensible translator. """ ALCHEMISTS = { } - def __new__(cls, form, magic = None): + def __new__(cls, form, reflex = None): if form not in Alchemist.ALCHEMISTS: - if not magic: - magic = 'to{}'.format(form) - Alchemist.ALCHEMISTS[form] = tuple.__new__(cls, (form, magic)) + if not reflex: + reflex = 'to{}'.format(form) + Alchemist.ALCHEMISTS[form] = tuple.__new__(cls, (form, reflex)) return Alchemist.ALCHEMISTS[form] - def __init__(self, form, magic = None): - if getattr(self, 'spells', None) is None: - self.spells = [ ] + def __init__(self, form, reflex = None): + if getattr(self, 'magic', None) is None: + self.magic = [ ] - form = property(operator.itemgetter(0)) - magic = property(operator.itemgetter(1)) + form = property(operator.itemgetter(0)) + reflex = property(operator.itemgetter(1)) def __call__(self, subject, **kwargs): #-- first, we look to see if the given subject already has an embedded map (spell) #-- e.g. if we're trying to cast the subject into a JSON serializable dictionary (JDN), #-- then we'd look to see if subject as a .toJDN() method (or some such) - spell = getattr(subject, self.magic, None) + spell = getattr(subject, self.reflex, None) if spell: if callable(spell): return spell(**kwargs) @@ -109,9 +110,9 @@ def _toJDN_Exception(x, **kwargs): def _toJDN_path(x, **kwargs): return str(x) -toJDN.spells.append( (datetime.date, _toJDN_date) ) -toJDN.spells.append( (Exception, _toJDN_Exception) ) -toJDN.spells.append( (pathlib.PurePath, _toJDN_path) ) +toJDN.magic.append( (datetime.date, _toJDN_date) ) +toJDN.magic.append( (Exception, _toJDN_Exception) ) +toJDN.magic.append( (pathlib.PurePath, _toJDN_path) ) class entity: @@ -387,7 +388,7 @@ def prefer(options, **kwargs): if n == 1: return choices[0] - return choiecs[:n] + return choices[:n] @@ -409,3 +410,9 @@ def asPath(x): Accessor/wrapper for answering x already cast as a pathlib.Path instance """ return pathlib.Path(x) + +def asPurePath(x): + """ + Accessor/wrapper for answering x already cast as a pathlib.PurePosixPath instance + """ + return pathlib.PurePosixPath(x) diff --git a/lib/Bastion/Condo.py b/lib/Bastion/Condo.py index 4959187..9ac6fa0 100644 --- a/lib/Bastion/Condo.py +++ b/lib/Bastion/Condo.py @@ -17,10 +17,8 @@ print(conf['some']['other']['key']) """ import sys -import os import collections.abc as pycollections import json -import datetime import fnmatch import logging import pathlib @@ -31,10 +29,10 @@ except: pass -try: - import openpyxl -except: - pass +#try: +# import openpyxl +#except: +# pass @@ -205,6 +203,15 @@ def flattened(self): """ return dict([(k, self[k]) for k in self.keys.deep]) + def copy(self): + """ + Answers a new nested dictionary by shallow (objects are referenced, not copied) copy. + """ + clone = CxNode() + for k, v in self.flattened: + clone[k] = v + return clone + def __iter__(self): for k in sorted(self.children.keys()): yield CxItem(k, self.children[k]) @@ -414,41 +421,42 @@ def loadCSV(self, fname, **kwargs): return self.node def loadXLSX(self, srcpath, sheet = None, **kwargs): - xlspath = pathlib.Path(srcpath) - if 'openpyxl' in sys.modules: - keyCol = kwargs.get('keyCol', 'A') - valCol = kwargs.get('valCol', 'B') - bgnRow = kwargs.get('bgnRow', 2) - - wb = openpyxl.load_workbook(str(xlspath)) - ws = None - if sheet: - if isinstance(sheet, str): - ws = wb[sheet] - elif isinstance(sheet, int): - #-- Get sheet by index - dex = dict(enumerate(wb.sheetnames)) - ws = dex[sheet] - - if not ws: - msg = "Condo/loadXLSX: .xlsx workbook sheets must be by name or index" - logger.error(msg) - raise ValueError(msg) - - for i in range(2, ws.max_row+1): - k = ws["{}{}".format(keyCol,i)].value - v = ws["{}{}".format(valCol,i)].value - if k is not None: - k = k.strip() - if k: - self.node[k] = v - - else: - msg = "CxNode/loadXLSX: openpyxl module not available" - logger.error(msg) - raise Exception(msg) - - return self.node + raise NotImplementedError +# xlspath = pathlib.Path(srcpath) +# if 'openpyxl' in sys.modules: +# keyCol = kwargs.get('keyCol', 'A') +# valCol = kwargs.get('valCol', 'B') +# bgnRow = kwargs.get('bgnRow', 2) +# +# wb = openpyxl.load_workbook(str(xlspath)) +# ws = None +# if sheet: +# if isinstance(sheet, str): +# ws = wb[sheet] +# elif isinstance(sheet, int): +# #-- Get sheet by index +# dex = dict(enumerate(wb.sheetnames)) +# ws = dex[sheet] +# +# if not ws: +# msg = "Condo/loadXLSX: .xlsx workbook sheets must be by name or index" +# logger.error(msg) +# raise ValueError(msg) +# +# for i in range(2, ws.max_row+1): +# k = ws["{}{}".format(keyCol,i)].value +# v = ws["{}{}".format(valCol,i)].value +# if k is not None: +# k = k.strip() +# if k: +# self.node[k] = v +# +# else: +# msg = "CxNode/loadXLSX: openpyxl module not available" +# logger.error(msg) +# raise Exception(msg) +# +# return self.node @@ -481,7 +489,7 @@ def sed(self, tokens): if token[0] == '^': #-- load the file path = token[1:] - node.load(path) + self.node.load(path) elif token[-1] == ':': key = token[:-1] diff --git a/lib/Bastion/Curator.py b/lib/Bastion/Curator.py index 672f33b..507aef7 100644 --- a/lib/Bastion/Curator.py +++ b/lib/Bastion/Curator.py @@ -3,11 +3,11 @@ I provide mostly data structures for working with archives and backups. """ -import pathlib +import datetime -from .Common import * -from .Model import ARK, isAsset -from .Chronology import Quantim +from Bastion.Common import Thing, Boggle, canTextify +from Bastion.Model import ARK, isAsset +from Bastion.Chronology import Quantim class BLONDE(canTextify): diff --git a/lib/Bastion/Model.py b/lib/Bastion/Model.py index 55973c6..fc08c50 100644 --- a/lib/Bastion/Model.py +++ b/lib/Bastion/Model.py @@ -3,7 +3,7 @@ """ import pathlib -from .Common import RDN, CURIE, Slug40, entity, toJDN +from Bastion.Common import RDN, CURIE, Slug40, entity, toJDN class ARK(tuple): @@ -99,15 +99,9 @@ def perform(self, request): class isVault: - """ - abstract class for all vaults. - """ - pass - - -class Vault(isVault): """ I am the base class for all storage vaults. + Users of Vaults primarily interact with the vault through the .push and .pull methods. """ PROTOCOLS = { } @@ -131,9 +125,12 @@ def mover(self): def packer(self): #-- Answers a packer (if needed) for this vault. raise NotImplementedError + #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ + #-- END ACTOR ACCESSORS | + #---------------------------------------------------------------------------- #---------------------------------------------------------------------------- - #-- BEGIN CLERK DELEGATION | + #-- BEGIN CLERK DELEGATES | #-- These methods are basically pass-throughs to the clerk's implementation | #↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ @property @@ -172,7 +169,7 @@ def manifest(self, ark): #-- BEGIN MOVER DELEGATES | #-- These methods are pass throughs to the attached mover. | #↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ - def provision(self, *args): + def provision(self, *args, **kwargs): """ provision(ark) - ensures that the site, zone, and asset folders exist. provision(site, zone, asset_name) - an alias for provision(ark) @@ -180,27 +177,6 @@ def provision(self, *args): """ return self.mover.provision(*args, **kwargs) - def push(self, asset, basis = None, **kwargs): - """ - Given an asset, I push a backup of the asset to this vault. - push(asset) - creates a full backup in this vault, creating a new base for differentials - push(asset, basis) - creates a differential backup relative to the given basis. - {basis} - can be a datetime or a BLONDE. - A typical implementation of .push() ... - 1. call .pack() method to create a local archive in my scratch (spool) space - 2. use .put() to transfer the local archive to the vault space - 3. perform a vault-specific transfer verification - 4. remove the local, scratch (spool) archive file. - Answers a tuple of (transferred, blonde, receipt), where... - * transferred - is the True/False indication answered by .put() - * blonde - is the string reprsentation of the blonde for the archive of the asset, - * receipt - is detailed, structured answer given by the .put() operation. - """ - return self.mover.push(asset, basis, **kwargs) - - def pull(self, blonde, **kwargs): - return self.mover.pull(blonde, **kwargs) - def put(self, halo, tag, **kwargs): """ Given path to a local file (aka Host Asset LOcation), @@ -247,20 +223,48 @@ def unpack(self, halo, root, **kwargs): #-- END PACKER DELEGATES | #---------------------------------------------------------------------------- + #----------------------------------------------------------- + #-- BEGIN SUBCLASS RESPONSIBILITY | + #-- Subclasses must implement these methods. | + #↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ + def push(self, asset, basis = None, **kwargs): + """ + Given an asset, I push a backup of the asset to this vault. + push(asset) - creates a full backup in this vault, creating a new base for differentials + push(asset, basis) - creates a differential backup relative to the given basis. + {basis} - can be a datetime or a BLONDE. + A typical implementation of .push() ... + 1. call .pack() method to create a local archive in my scratch (spool) space + 2. use .put() to transfer the local archive to the vault space + 3. perform a vault-specific transfer verification + 4. remove the local, scratch (spool) archive file. + Answers a tuple of (transferred, blonde, receipt), where... + * transferred - is the True/False indication answered by .put() + * blonde - is the string reprsentation of the blonde for the archive of the asset, + * receipt - is detailed, structured answer given by the .put() operation. + """ + raise NotImplementedError + + def pull(self, blonde, **kwargs): + raise NotImplementedError + def configured(self, conf): raise NotImplementedError + #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ + #-- END SUBCLASS RESPONSIBILITY | + #---------------------------------------------------------------------------- @classmethod def register(cls): - Vault.PROTOCOLS[cls.PROTOCOL] = cls + isVault.PROTOCOLS[cls.PROTOCOL] = cls @staticmethod def for_protocol(protocol): - return Vault.PROTOCOLS[protocol] + return isVault.PROTOCOLS[protocol] @staticmethod def handling(protocol): - return Vault.PROTOCOLS[protocol] + return isVault.PROTOCOLS[protocol] @@ -440,27 +444,6 @@ def provision(self, *args): """ raise NotImplementedError - def push(self, asset, basis = None, **kwargs): - """ - Given an asset, I push a backup of the asset to this vault. - push(asset) - creates a full backup in this vault, creating a new base for differentials - push(asset, basis) - creates a differential backup relative to the given basis. - {basis} - can be a datetime or a BLONDE. - A typical implementation of .push() ... - 1. call .pack() method to create a local archive in my scratch (spool) space - 2. use .put() to transfer the local archive to the vault space - 3. perform a vault-specific transfer verification - 4. remove the local, scratch (spool) archive file. - Answers a tuple of (transferred, blonde, receipt), where... - * transferred - is the True/False indication answered by .put() - * blonde - is the string reprsentation of the blonde for the archive of the asset, - * receipt - is detailed, structured answer given by the .put() operation. - """ - raise NotImplementedError - - def pull(self, blonde, **kwargs): - raise NotImplementedError - def put(self, halo, tag, **kwargs): """ Given path to a local file (aka Host Asset LOcation), @@ -495,7 +478,7 @@ def entity_isZone(self): return isinstance(self.subject, isZone) def entity_isARK(self): - return isinstance(self.subject, isARK) + return isinstance(self.subject, ARK) def entity_isVault(self): return isinstance(self.subject, isVault) @@ -514,4 +497,4 @@ def entity_isMover(self): def _toJDN_isAsset(asset): return str(asset.CURIE) -toJDN.spells.append( (isAsset, _toJDN_isAsset) ) +toJDN.magic.append( (isAsset, _toJDN_isAsset) ) diff --git a/lib/Bastion/Movers/BFD.py b/lib/Bastion/Movers/BFD.py index 6f8ab1b..a3bdafe 100644 --- a/lib/Bastion/Movers/BFD.py +++ b/lib/Bastion/Movers/BFD.py @@ -1,37 +1,22 @@ -import os import pathlib -import operator -import datetime import logging -import getpass import shutil -import logging -from Bastion.Common import Thing, Unknown -from Bastion.Utils import pax -from Bastion.Model import isMover -from Bastion.Curator import Manifest, BLONDE, Snap +from Bastion.Model import ARK, isMover, isVault +from Bastion.Movers.CARP import PutRequest, PutReceipt logger = logging.getLogger(__name__) -class Mover(Bastion.Model.isMover): +class Mover(isMover): def __init__(self, vault, **kwargs): isMover.__init__(self) self.vault = vault + self.root = kwargs.get('root', vault.root) - @property - def bfd(self): - return self.vault.bfd - - @property - def bank(self): - return self.vault.bank - - @property - def scratch(self): - return self.vault.scratch + assert isinstance(self.vault, isVault), "vault must be an instance of Bastion.Model.isVault" + assert isinstance(self.root, pathlib.Path), "root must be an instance of Path" #----------------------------------------- #-- BEGIN Bastion.Model.isMover PROTOCOL | @@ -49,102 +34,31 @@ def provision(self, *args): else: raise ValueError - repo = self.bank / ark.site / ark.zone / ark.asset + repo = self.root / ark.site / ark.zone / ark.asset return repo.mkdir(parents = True, exist_ok = True) + def put(self, halo, tag, **kwargs): + here = halo + there = self.root / tag + logger.debug( "put source {} to {}".format(here.as_uri(), there.as_uri()) ) - def pack(self, asset, basis = None, **kwargs): - """ - Given a local asset, I package (.tar, .zip, etc) the asset into my scratch (spool) space. - Without a given basis, I package everything (i.e. a full backup). - When a basis is given as either a datetime or an anchor (BLONDE), I do a differential backup. - I answer the BLONDE for the package, relative to the local scratch (spool) space. - """ - detail = 'F' - whence = None - basis = None - blonded = None - tarp = None - ark = asset.ARK - opts = { } - - if basis: - detail = 'D' - if isinstance(basis, BLONDE): - #-- I was given a BLONDE (a reference to a full backup) - anchor = basis - whence = anchor.when.earliest - genus = anchor.genus - blonded = BLONDE.forDiffBackup(anchor) - - if isinstance(basis, datetime.datetime): - whence = basis - genus = "___" - blonded = BLONDE(asset.ARK, detail, genus) - - opts['since'] = whence + #-- Create the put request. + request = PutRequest(halo, self.root.as_uri(), tag) + try: + shutil.copy(here, there) + except Exception as err: + #-- shutil should throw an error if there was a problem copying. + report = request.failed(err) else: - blonded = BLONDE.forFullBackup(asset.ARK) - - package = "{}.tar".format(str(blonded)) - tag = pathlib.PurePosixPath(ark.site) / ark.zone / ark.asset / package - spool = (self.scratch / tag).parent - - #-- assure that the scratch path exists and all of the subpaths that create the repo folder for this asset. - spool.mkdir(parents = True, exist_ok = True) - - #-- use the built-in python tar archiver, using "pax" (POSIX.1-2001) extensions. - pax((self.scratch / tag), asset.halo, **opts) - - #-- answer the BLONDE of the newly created package. - return (blonded, tag, spool, package) + report = request.succeeded( PutReceipt(halo, self.root.as_uri(), tag) ) + return report - def push(self, asset, basis = None, **kwargs): - """ - Given an asset, I push a backup of the asset to this vault. - push(asset) - creates a full backup in this vault, creating a new base for differentials - push(asset, basis) - creates a differential backup relative to the given basis. - {asset} - an instance of Bastion.Model.isAsset - {basis} - can be a datetime or a BLONDE. - """ - blonde, tag, spool, package = self.pack(asset, basis, **kwargs) - - #-- assure that the bank exists. - (self.bank / tag).parent.mkdir(parents = True, exist_ok = True) - - transferred, receipt = self.put(self.scratch / tag, tag) - - if transferred: - #-- clean up! - (self.scratch / tag).unlink() - - return (transferred, blonde, receipt) - - - def pull(self, ark, **kwargs): + def get(self, tag, halo, **kwargs): + #shutil.copystat(self.bank / tag, halo) raise NotImplementedError - - def put(self, halo, tag, **kwargs): - here = halo - there = self.bank / tag - logger.debug("put source {} to {}".format(str(here), str(there))) - started = datetime.datetime.now() - shutil.copy(here, there) - completed = datetime.datetime.now() - receipt = { - 'tag': str(tag), - 'source': str(here), - 'destination': str(there), - 'started': started.isoformat(), - 'completed': completed.isoformat() - } - return (True, receipt) - - def get(self, tag, halo, **kwargs): - shutil.copystat(self.bank / tag, halo) #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ #-- END Bastion.Model.isMover PROTOCOL | #--------------------------------------- diff --git a/lib/Bastion/Movers/CARP.py b/lib/Bastion/Movers/CARP.py index a79dab2..af0e0ea 100644 --- a/lib/Bastion/Movers/CARP.py +++ b/lib/Bastion/Movers/CARP.py @@ -7,30 +7,57 @@ import logging from Bastion.Common import Thing, Unknown -from Bastion.Model import Request, isAsset, isResult +from Bastion.Model import isRequest, isAsset, isResult from Bastion.Curator import Manifest, BLONDE, Snap -from Bastion.CARP import * +from Bastion.CARP import isReceipt, isRequest logger = logging.getLogger(__name__) -class PushReceipt: - def __init__(self, packed, moved): - #-- packed - the report structure for the pack operation. - #-- moved - the report structure for the move operation. - assert isinstance(packed, isResult), "packed must be an instance of Bastion.CARP.isResult" - assert isinstance(moved, isResult), "moved must be an instance of Bastion.CARP.isResult" - self.pack = packed - self.put = moved +class PutReceipt(isReceipt): + def __init__(self, halo, rendpoint, tag): + assert isinstance(halo, pathlib.Path), "halo must be an instance of Path" + assert isinstance(tag, (pathlib.PurePosixPath, str)), "tag must be an instance of str or PurePosixPath" + self.local = halo + self.target = Thing(endpoint = rendpoint, tag = tag) + + def toJDN(self, **kwargs): + jdn = { + 'source': { + 'path': str(self.local) + }, + 'target': { + 'endpoint': toJDN(self.target.endpoint), + 'tag': toJDN(self.target.tag) + } + } + return jdn + + +class GetReceipt(isReceipt): + def __init__(self, rendpoint, tag, halo): + assert isinstance(halo, pathlib.Path), "halo must be an instance of Path" + assert isinstance(tag, (pathlib.PurePosixPath, str)), "tag must be an instance of str or PurePosixPath" + self.local = halo + self.target = Thing(endpoint = rendpoint, tag = tag) def toJDN(self, **kwargs): - return [ toJDN(self.pack), toJDN(self.put) ] + jdn = { + 'source': { + 'endpoint': toJDN(self.target.endpoint), + 'tag': toJDN(self.target.tag) + }, + 'target': { + 'path': str(self.local) + } + } + return jdn -class PutRequest(Request): +class PutRequest(isRequest): """ - Constructs a request for putting a given local file. + A request for putting a given file into a target endpoint. """ def __init__(self, halo, rendpoint, tag): """ @@ -38,87 +65,41 @@ def __init__(self, halo, rendpoint, tag): rendpoint - is a string (or object) that describes the remote endpoint. tag - the tag that will identify this file on the remote system, typically a string, or pathlib.Path """ + assert isinstance(halo, pathlib.Path), "halo must be an instance of Path" + assert isinstance(tag, (pathlib.PurePosixPath, str)), "tag must be an instance of str or PurePosixPath" + self.local = halo - self.remote = Thing(endpoint = endpoint, tag = tag) + self.bank = Thing(endpoint = rendpoint, tag = tag) - def toJDN(self, **kwargs): - jdn = { - 'local': str(self.local), - 'remote' = { - 'endpoint': toJDN(self.remote.endpoint), - 'tag': toJDN(self.remote.tag) - } + xtras = { + "local.path": str(self.local) + "bank.endpoint": toJDN(self.bank.endpoint) + "bank.tag": toJDN(self.bank.tag) } - return jdn + isRequest.__init__(self, "PutRequest", **xtras) -class PushRequest(Request): + +class GetRequest(isRequest): """ - Constructs a request for pushing a given asset. - Can be invoked as... - PushRequest(request) - casts a generic request into an instance of PackRequest - PushRequest(asset) - implies a full backup of the asset - PushRequest(asset, since) - implies a differential backup of the asset relative to the given datetime (since) - PushRequest(asset, basis) - implies a differential backup of the asset relative to the given basis (BLONDE) + A request for retrieving a given object from an endpoint and storing it to a local file path. """ - def __init__(self, subject, *args, **kwargs): - if isinstance(subject, Request): - Request.__init__("PushRequest", ID = subject.ID, opened = subject.when, context = subject.context) - - else: - assert isinstance(subject, isAsset), "PushRequest should be given an instance of isAsset" - asset = subject - basis = args[0] if args else None - whence = None - genus = None - - if basis: - detail = 'D' - if isinstance(basis, BLONDE): - #-- I was given a BLONDE (a reference to a full backup) - whence = basis.when.earliest - genus = basis.genus - blonded = BLONDE.forDiffBackup(basis) - if isinstance(basis, datetime.datetime): - whence = basis - genus = "___" - blonded = BLONDE(asset.ARK, detail, genus) - else: - detail = 'F' - blonded = BLONDE.forFullBackup(asset.ARK) - - xtras = { - 'blonde': blonded, - 'detail': detail, - 'since': whence, - 'basis': basis, - 'genus': genus, - 'asset': asset, - 'halo': str(asset.halo) - } - - Request.__init__("PushRequest", asset, **xtras) - - @property - def asset(self): - return self['asset'] - - @property - def basis(self): - return self['basis'] - - @property - def detail(self): - return self['detail'] + def __init__(self, rendpoint, tag, halo): + """ + rendpoint - is a string (or object) that describes the remote endpoint. + tag - the tag that will identify this file on the remote system, typically a string, or pathlib.Path + halo - is a path to a local file. + """ + assert isinstance(halo, pathlib.Path), "halo must be an instance of Path" + assert isinstance(tag, (pathlib.PurePosixPath, str)), "tag must be an instance of str or PurePosixPath" - @property - def blonde(self): - return self['blonde'] + self.local = halo + self.bank = Thing(endpoint = rendpoint, tag = tag) - @property - def since(self): - return self['since'] + xtras = { + "local.path": str(self.local) + "bank.endpoint": toJDN(self.bank.endpoint) + "bank.tag": toJDN(self.bank.tag) + } - @property - def genus(self): - return self['genus'] + isRequest.__init__(self, "GetRequest", **xtras) diff --git a/lib/Bastion/Movers/SFTP.py b/lib/Bastion/Movers/SFTP.py index d25eaef..e0f0aa7 100644 --- a/lib/Bastion/Movers/SFTP.py +++ b/lib/Bastion/Movers/SFTP.py @@ -1,21 +1,8 @@ -import os import pathlib -import subprocess -import operator -import datetime -import json -import socket -import logging -import getpass -import shutil -import tarfile import logging -from Bastion.Common import Thing, Unknown import Bastion.Model -from Bastion.Curator import Manifest, BLONDE, Snap -from Bastion.CARP import Request, Receipt - +from Bastion.NetOps.sCURL import SCURLer logger = logging.getLogger(__name__) @@ -36,7 +23,7 @@ def __init__(self, vault, **kwargs): @property def bank(self): - client = SCURLer(self.login, self.host, self.root, keyfile = self.key) + return self.scurler #----------------------------------------- #-- BEGIN Bastion.Model.isMover PROTOCOL | @@ -46,53 +33,14 @@ def provision(self, *args): provision(ark) - ensures that the site, zone, and asset folders exist. provision(site, zone, asset_name) - an alias for provision(ark) """ - ark = ARK(*args) - repo = self.scurler / ark.site / ark.zone / ark.asset - return repo.mkdir(parents = True, exist_ok = True) - - def push(self, asset, basis = None, **kwargs): - """ - Given an asset, I push a backup of the asset to this vault. - push(asset) - creates a full backup in this vault, creating a new base for differentials - push(asset, basis) - creates a differential backup relative to the given basis. - {asset} - an instance of Bastion.Model.isAsset - {basis} - can be a datetime or a BLONDE. - """ - packing = self.vault.pack(asset, basis, **kwargs) - - #-- if the pack operation failed, we return the failure report. - if packing.failed: - return packing - - transferred = self.put(packing.spooled, packed.opts['tag']) - - if transferred: - #-- clean up! - (self.scratch / tag).unlink() - - return (transferred, blonde, receipt) - - def pull(self, ark, **kwargs): raise NotImplementedError def put(self, halo, tag, **kwargs): - here = halo - there = self.bank / tag - logger.debug("put source {} to {}".format(str(here), str(there))) - started = datetime.datetime.now() - - completed = datetime.datetime.now() - receipt = { - 'tag': str(tag), - 'source': str(here), - 'URL': str(there), - 'started': started.isoformat(), - 'completed': completed.isoformat() - } - return (True, receipt) + raise NotImplementedError + def get(self, tag, halo, **kwargs): - shutil.copystat(self.bank / tag, halo) + raise NotImplementedError #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ diff --git a/lib/Bastion/NetOps/sCURL.py b/lib/Bastion/NetOps/sCURL.py index 5b6fdae..a5a9a51 100644 --- a/lib/Bastion/NetOps/sCURL.py +++ b/lib/Bastion/NetOps/sCURL.py @@ -6,7 +6,6 @@ import subprocess import pathlib import datetime -import sys import getpass import operator @@ -74,7 +73,7 @@ def __str__(self): def __truediv__(self, subpath): path = self.path / subpath - return sfURL(self.host, path, user = user, port = port) + return sfURL(self.host, path, user = self.user, port = self.port) class Alien: diff --git a/lib/Bastion/Packers/CARP.py b/lib/Bastion/Packers/CARP.py index fac1c14..8ce1d62 100644 --- a/lib/Bastion/Packers/CARP.py +++ b/lib/Bastion/Packers/CARP.py @@ -1,9 +1,10 @@ +import pathlib +import datetime import logging -from Bastion.Common import Thing, Unknown -import Bastion.Model -from Bastion.Curator import Manifest, BLONDE, Snap -from Bastion.CARP import * +from Bastion.Model import isAsset +from Bastion.CARP import isRequest +from Bastion.Curator import BLONDE logger = logging.getLogger(__name__) @@ -19,11 +20,11 @@ def __init__(self, asset, tag, packaged, tarp): assert isinstance(tag, pathlib.PurePosixPath), "tag must be pathlib.PurePosixPath" assert isinstance(tarp, pathlib.PosixPath), "tarp must be pathlib.PosixPath" - self.asset = asset - self.tag = tag - self.packaged = packaged - self.tarp = tarp - self.blonde = BLONDE.decode(tag.name.stem) + self.asset = asset #-- the asset that was packed + self.tag = tag #-- the full name of the packed object + self.packaged = packaged #-- an inventory of what was packed + self.tarp = tarp #-- the path to the local package (TARfile Path → "tarp") + self.blonde = BLONDE.decode(tag.name.stem) #-- the BLONDE for the packed object def toJDN(self): jdn = { @@ -36,7 +37,7 @@ def toJDN(self): return jdn -class PackRequest(Request): +class PackRequest(isRequest): """ Constructs a request for packing a given asset. Can be invoked as... @@ -45,10 +46,9 @@ class PackRequest(Request): PackRequest(asset, since) - implies a differential backup of the asset relative to the given datetime (since) PackRequest(asset, basis) - implies a differential backup of the asset relative to the given basis (BLONDE) """ -# def __init__(self, asset, basis = None, **kwargs): def __init__(self, subject, *args, **kwargs): - if isinstance(subject, Request): - Request.__init__("PackRequest", ID = subject.ID, opened = subject.when, context = subject.context) + if isinstance(subject, isRequest): + isRequest.__init__("PackRequest", ID = subject.ID, opened = subject.when, context = subject.context) else: asset = subject @@ -81,7 +81,7 @@ def __init__(self, subject, *args, **kwargs): 'halo': str(asset.halo) } - Request.__init__("PackRequest", asset, **xtras) + isRequest.__init__("PackRequest", asset, **xtras) @property def asset(self): diff --git a/lib/Bastion/Packers/TARs.py b/lib/Bastion/Packers/TARs.py index e82101d..d30e18c 100644 --- a/lib/Bastion/Packers/TARs.py +++ b/lib/Bastion/Packers/TARs.py @@ -6,13 +6,14 @@ import datetime import pathlib import tarfile -import tempfile import logging -from Bastion.Common import * -from Bastion.CARP import Report +from Bastion.Chronology import quantim +from Bastion.Common import asPath, prefer +import Bastion.Model +from Bastion.CARP import isRequest, Report from Bastion.Packers.CARP import PackRequest, PackingReceipt -from Bastion.Chronology import + logger = logging.getLogger(__name__) @@ -103,7 +104,7 @@ def pack(self, subject, *args, **kwargs): if isinstance(subject, PackRequest): #-- .pack(request:PackRequest) request = subject - elif isinstance(subject, Request): + elif isinstance(subject, isRequest): #-- .pack(request:isRequest) request = PackRequest(subject) else: @@ -123,7 +124,7 @@ def pack(self, subject, *args, **kwargs): try: #-- use the built-in python tar archiver, using "pax" (POSIX.1-2001) extensions. - pax(tarp, asset.halo, **opts) + pax(tarp, request.asset.halo, **kwargs) except Exception as err: report = Report.Failed(request, err) else: @@ -155,6 +156,5 @@ def catalog(tarp): for info in tark.getmembers(): sz = info.size mq = quantim( datetime.datetime.fromtimestamp(info.mtime) ) - #pt = pathlib.PurePosixPath(info.name) cats.append( (sz, mq, info.name) ) return cats diff --git a/lib/Bastion/Site.py b/lib/Bastion/Site.py index a5e0045..e195b64 100644 --- a/lib/Bastion/Site.py +++ b/lib/Bastion/Site.py @@ -4,10 +4,11 @@ import logging import pathlib import random +import datetime -from .Common import * -from .Condo import CxNode -from .Model import ARK, isAsset, isZone, isSite +from Bastion.Common import entity, Thing, canTextify, DAYS, RDN, asPath, asLogLevel +from Bastion.Condo import CxNode +from Bastion.Model import ARK, isAsset, isZone, isSite #from .Curator import Asset logger = logging.getLogger(__name__) @@ -131,10 +132,10 @@ def configured(self, conf): return self def resources(self, zone): - zname = zone.name if isinstance(zone, ResourceZone) else zone + zname = zone.name if isinstance(zone, Zone) else zone if zname not in self._catalogs: - self._catalogs[zname] = ResourceCatalog(self, zname) + self._catalogs[zname] = AssetCatalog(self, zname) return self._catalogs[zname] @@ -212,7 +213,7 @@ def toJDN(self, **kwargs): def fromJDN(cls, jdn): depth = jdn['history'] * DAYS radius = jdn['drift'] * DAYS - return csl(history = depth, drift = radius) + return cls(history = depth, drift = radius) diff --git a/lib/Bastion/Vaults/BFD.py b/lib/Bastion/Vaults/BFD.py index 1ae393f..7544e70 100644 --- a/lib/Bastion/Vaults/BFD.py +++ b/lib/Bastion/Vaults/BFD.py @@ -1,197 +1,54 @@ -import os import pathlib -import subprocess -import operator -import datetime -import json -import socket -import logging -import getpass -import shutil -import tarfile import logging -from Bastion.Common import Thing, Unknown import Bastion.Model -from Bastion.Curator import Manifest, BLONDE, Snap -from Bastion.Clerks.BFD import Clerk -from Bastion.Movers.BFD import Mover +import Bastion.Vaults.Common +import Bastion.Clerks.BFD +import Bastion.Movers.BFD import Bastion.Packers.TARs logger = logging.getLogger(__name__) -class Vault(Bastion.Model.Vault): +class Vault(Bastion.Vaults.Common.isSpoolingVault): PROTOCOL = 'BFD' def __init__(self, name, **kwargs): Bastion.Model.Vault.__init__(self, name, **kwargs) - self.bfd = pathlib.Path("/") - self.bank = self.bfd / "bank" - self.scratch = self.bfd / "scratch" + self.bfd = pathlib.Path("/") + self.bank = self.bfd / "bank" + self._scratch = self.bfd / "scratch" + + self._clerk = None + self._packer = None + self._mover = None def configured(self, conf): confkey = "vaults.{}".format(self.name) if confkey in conf: - section = conf[confkey] - self.bfd = pathlib.Path( section['root'] ) - self.bank = self.bfd / "bank" - self.scratch = self.bfd / "scratch" + section = conf[confkey] + self.condex = section.copy( ) + self.bfd = pathlib.Path( section['root'] ) + self.bank = self.bfd / "bank" + self._scratch = self.bfd / "scratch" if 'bank.path' in section: self.bank = pathlib.Path( section['bank.path'] ) if 'scratch.path' in section: - self.scratch = pathlib.Path( section['scratch.path'] ) + self._scratch = pathlib.Path( section['scratch.path'] ) return self - def changed(self, *args, **kwargs): - #-- React to internal events that can invalidate cached items, etc. - for aspect in args: - if aspect in ('clerk', 'mover', 'packer'): - setattr(self, '_{}'.format(aspect), None) - return self + @property + def scratch(self): + return self._scratch @property - def sites(self): - """ - I answer a sorted collection of the names of all sites known to this vault. - """ - #-- sites are top level elements relative to the root of the vault. - sites = [ ] - for entry in self.root.iterdir(): - if entry.is_dir(): - sites.append(entry.name) - - return tuple(sorted(sites)) - - def zones(self, site): - """ - Given a site name, I answer a sorted collection of zone names that are known to this vault. - """ - #-- a zone will be a subdirectory (subfolder) of the given site. - #-- look for all of the subfolders of root / site - zones = [ ] - sroot = self.root / RDN(site) - if sroot.exists( ): - for entry in sroot.iterdir( ): - if entry.is_dir(): - zones.append(entry.name) - return tuple(sorted(zones)) - - def assets(self, site, zone): - #-- assets will be subdirectories (subfolders) of a given site, zone. - assets = [ ] - zroot = self.root / site / zone - if zroot.exists( ): - for entry in zroot.is_dir(): - assets.append(entry.name) - return tuple(sorted(assets)) - - def manifest(self, *args): - """ - I answer a manifest of the named asset... - .manifest(ark) - .manifest(asset) - .manifest(site, zone, asset) - """ - if len(args) == 1: - arg = args[0] - if isinstance(arg, Bastion.Model.ARK): - ark = arg - repo = self.bank / ark.site / ark.zone / ark.asset - blondes = [entry.name for entry in repo.iterdir() if entry.is_file()] - return Bastion.Curator.Manifest(ark, blondes) - if isinstance(arg, Bastion.Model.isAsset): - return self.manifest(arg.ARK) - elif len(args) == 3: - site, zone, asset = args - ark = ARK(site, zone, asset) - return self.manifest(ark) - raise ValueError - - def provision(self, *args): - """ - .provision(ark) - ensures that the site, zone, and asset folders exist. - .provision(asset) - .provision(site, zone, asset_name) - an alias for provision(ark) - """ - if len(args) == 1: - arg = args[0] - - if isinstance(arg, ARK): - ark = arg - repo = self.bank / ark.site / ark.zone / ark.assets - repo.mkdir(0o770, parents = True, exist_ok = True) - request = Request("ProvisionRequest") - request["ARK"] = str(ark.CURIE) - request["repo"] = str(repo) - return Report.Success(request) - if isinstance(arg, isAsset): - return self.provision(arg.ark) - - elif len(args) == 3: - site, zone, asset = args - ark = ARK(site, zone, asset) - return self.provision(ark) - - raise ValueError - - def push(self, asset, basis = None, **kwargs): - """ - Given an asset, I push a backup of the asset to this vault. - .push(asset) - creates a full backup in this vault, creating a new base for differentials - .push(asset, basis) - creates a differential backup relative to the given basis. - .push(asset, since) - creates a differential backup relative to the given datetime. - {asset} - an instance of Bastion.Model.isAsset - {basis} - can be a datetime or a BLONDE. - """ - #-- Create the request... - request = Request("PushRequest") - request["asset"] = str(asset.CURIE) - if basis: - if isinstance(basis, datetime.datetime): - request['since'] = basis - else: - request['basis'] = basis - - packing = self.pack(asset, basis, **kwargs) - - #-- if the pack operation failed, we return the failure report. - if packing.failed: - return packing - - transfer = self.put(packing.spooled, packed.opts['tag']) - - if transfer.succeeded: - #-- clean up! - (self.scratch / tag).unlink() - - return (transfer, blonde, receipt) - - - def pull(self, ark, **kwargs): - raise NotImplementedError - - def put(self, halo, tag, **kwargs): - here = halo - there = self.bank / tag - logger.debug("put source {} to {}".format(str(here), str(there))) - started = datetime.datetime.now() - shutil.copy(here, there) - completed = datetime.datetime.now() - receipt = { - 'tag': str(tag), - 'source': str(here), - 'destination': str(there), - 'started': started.isoformat(), - 'completed': completed.isoformat() - } - return (True, receipt) - - def get(self, tag, halo, **kwargs): - shutil.copystat(self.bank / tag, halo) + def clerk(self): + if getattr(self, '_clerk', None) is None: + self._clerk = Bastion.Clerks.BFD.Clerk(self, self.bank) + return self._clerk @property def packer(self): @@ -199,19 +56,10 @@ def packer(self): self._packer = Bastion.Packers.TARs.Packer(self) return self._packer - @property - def clerk(self): - """ - I am my own clerk. - """ - return self - @property def mover(self): - """ - I am my own mover. - """ - return self + if getattr(self, '_mover', None) is None: + self._mover = Bastion.Movers.BFD.Mover(self) #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ #-- END Bastion.Model.Vault PROTOCOL | diff --git a/lib/Bastion/Vaults/CARP.py b/lib/Bastion/Vaults/CARP.py new file mode 100644 index 0000000..f6dad62 --- /dev/null +++ b/lib/Bastion/Vaults/CARP.py @@ -0,0 +1,82 @@ +import logging +import datetime + +from Bastion.CARP import isRequest, isReceiptOfResults +from Bastion.Curator import BLONDE + +logger = logging.getLogger(__name__) + + +class PushReceipt(isReceiptOfResults): + pass + + +class PushRequest(isRequest): + """ + Constructs a request for packing a given asset. + Can be invoked as... + PushRequest(request) - casts a generic request into an instance of PushRequest + PushRequest(asset) - implies a full backup of the asset + PushRequest(asset, since) - implies a differential backup of the asset relative to the given datetime (since) + PushRequest(asset, basis) - implies a differential backup of the asset relative to the given basis (BLONDE) + """ + def __init__(self, subject, *args, **kwargs): + if isinstance(subject, isRequest): + isRequest.__init__("PushRequest", ID = subject.ID, opened = subject.when, context = subject.context) + + else: + asset = subject + basis = args[0] if args else None + whence = None + genus = None + + if basis: + detail = 'D' + if isinstance(basis, BLONDE): + #-- I was given a BLONDE (a reference to a full backup) + whence = basis.when.earliest + genus = basis.genus + blonded = BLONDE.forDiffBackup(basis) + if isinstance(basis, datetime.datetime): + whence = basis + genus = "___" + blonded = BLONDE(asset.ARK, detail, genus) + else: + detail = 'F' + blonded = BLONDE.forFullBackup(asset.ARK) + + xtras = { + 'blonde': blonded, + 'detail': detail, + 'since': whence, + 'basis': basis, + 'genus': genus, + 'asset': asset, + 'halo': str(asset.halo) + } + + isRequest.__init__("PackRequest", asset, **xtras) + + @property + def asset(self): + return self['asset'] + + @property + def basis(self): + return self['basis'] + + @property + def detail(self): + return self['detail'] + + @property + def blonde(self): + return self['blonde'] + + @property + def since(self): + return self['since'] + + @property + def genus(self): + return self['genus'] diff --git a/lib/Bastion/Vaults/Common.py b/lib/Bastion/Vaults/Common.py index 6f0f633..3048917 100644 --- a/lib/Bastion/Vaults/Common.py +++ b/lib/Bastion/Vaults/Common.py @@ -1,77 +1,48 @@ """ Bastion.Vault.Common """ -from Curator import Archive, Branch, Snap +import logging -class isClerk: - """ - I am an abstract type for "clerk" objects that do data management in the context of a vault. - """ - @property - def sites(self): - raise NotImplementedError - - @property - def zones(self): - raise NotImplementedError - - @property - def snaps(self): - raise NotImplementedError - - @property - def branches(self): - raise NotImplementedError - - @property - def archives(self): - raise NotImplementedError +import Bastion.Model +from Bastion.Vaults.CARP import PushRequest, PushReceipt +import Bastion.Packers.TARs -class isSiteClerk: - def __init__(self, site): - self.site = site +logger = logging.getLogger(__name__) - def zones(self): - raise NotImplementedError +class isSpoolingVault(Bastion.Model.Vault): + """ + I am a vault that uses a local scratch (spool) volume as a buffer + to pack archives before transferring them to final storage location. + """ + def __init__(self, name, **kwargs): + Bastion.Model.Vault.__init__(self, name, **kwargs) -class isZoneClerk: - def __init__(self, site, zone): - self.site = site - self.zone = zone + def push(self, asset, basis = None, **kwargs): + """ + Given an asset, I push a backup of the asset to this vault. + push(asset) - creates a full backup in this vault, creating a new base for differentials + push(asset, basis) - creates a differential backup relative to the given basis. + {asset} - an instance of Bastion.Model.isAsset + {basis} - can be a datetime or a BLONDE. + """ + #-- Build the request object. + request = PushRequest(asset, basis, **kwargs) + packing = self.pack(asset, basis, **kwargs) + if packing.failed: + return request.failed( PushReceipt([packing]) ) -class isArchiveClerk: - def __init__(self, site, zone, archive): - self.site = site - self.zone = zone - self.resource = archive + movement = self.put(packing.tarp, packing.tag) + if movement.failed: + return request.failed( PushReceipt([packing, movement]) ) + #-- clean up! + if packing.tarp.exists(): + packing.tarp.unlink() -class isBranchClerk: - def __init__(self, site, zone, archive, branch): - self.site = site - self.zone = zone - self.resource = archive - self.branch = branch + return request.succeeded( PushReceipt([packing, movement]) ) - @property - def snaps(self): + def pull(self, ark, **kwargs): raise NotImplementedError - - -class isVault: - """ - I am an abstract base type for specialized Vault classes. - """ - def __getitem__(self, asset): - raise NotImplementedError - - @property - def assets(self): - raise NotImplementedError - - def put(self, asset, latest = None): - pass - diff --git a/lib/Bastion/Vaults/SFTP.py b/lib/Bastion/Vaults/SFTP.py index 82f7123..a513045 100644 --- a/lib/Bastion/Vaults/SFTP.py +++ b/lib/Bastion/Vaults/SFTP.py @@ -11,65 +11,55 @@ import tarfile import logging -from Bastion.Common import Thing, Unknown +from Bastion.Common import Thing, Unknown, asPath, asPurePath import Bastion.Model from Bastion.Curator import Manifest, BLONDE, Snap -from Bastion.Movers.sCURL import Mover -from Bastion.Clerks.sCURL import Clerk +import Bastion.Movers.SFTP +import Bastion.Clerks.SFTP logger = logging.getLogger(__name__) -def asPath(x): - return pathlib.Path(x) - -def asPurePath(x): - return pathlib.PurePosixPath(x) - - -class Vault(Bastion.Model.Vault): +class Vault(Bastion.Vaults.Common.isSpoolingVault): PROTOCOL = 'SFTP' def __init__(self, name, **kwargs): Bastion.Model.Vault.__init__(self, name, **kwargs) + self.condex = None self.scratch = pathlib.Path("/tmp") self.host = None self.login = getpass.getuser() self.keypath = pathlib.Path("~/.ssh/id") self.bank = None + self.root = pathlib.PurePosixPath("/") #-- the default remote root folder + + self._packer = None #-- cached instance of Packer self._mover = None #-- cached instance of Mover self._clerk = None #-- cached instance of Clerk def configured(self, conf): confkey = "vaults.{}".format(self.name) if confkey in conf: - section = conf[confkey] - remote = section['remote'] + section = conf[confkey] + self.condex = section.copy( ) #-- configuration relevant to local host. - local = section['local'] - self.scratch = local.get(asPath, "scratch", "/tmp") + self.scratch = section.get(asPath, "scratch.path", "/tmp") #-- Configuration relevant to remote (bank) host. - self.host = remote.get('host') - self.login = remote.get('login', getpass.getuser()) - self.keypath = remote.get(asPath, 'key', pathlib.Path("~/.ssh/id").expanduser()) - self.root = remote.get(asPurePath, 'root', "/") + self.host = section.get('remote.host') + self.login = section.get('remote.login', getpass.getuser()) + self.keypath = section.get(asPath, 'remote.key', pathlib.Path("~/.ssh/id").expanduser()) + self.root = section.get(asPurePath, 'remote.root', "/") return self - def changed(self, *args, **kwargs): - for aspect in args: - if aspect in ('mover', 'clerk'): - setattr(self, '_{}'.format(aspect), None) - return self - @property def mover(self): if getattr(self, '_mover', None) is None: - self._mover = Mover(self, self.host, self.login, self.keypath) + self._mover = Bastion.Movers.SFTP.Mover(self, self.host, self.login, self.keypath, root = self.root) return self._mover @property @@ -141,48 +131,23 @@ def pack(self, asset, basis = None, **kwargs): return receipt - def push(self, asset, basis = None, **kwargs): + def put(self, obj, *args, **kwargs): """ - Given an asset, I push a backup of the asset to this vault. - push(asset) - creates a full backup in this vault, creating a new base for differentials - push(asset, basis) - creates a differential backup relative to the given basis. - {asset} - an instance of Bastion.Model.isAsset - {basis} - can be a datetime or a BLONDE. + performs a transfer of the file at host area local (halo) path to the tag at the destination. + put(receipt:PackingReceipt) + put(halo:PosixPath, tag:PurePosixPath) """ - packed = self.pack(asset, basis, **kwargs) - - xferrd = self.put(packed.spooled, packed.opts['tag']) - - if transferred: - #-- clean up! - (self.scratch / tag).unlink() - - return (transferred, blonde, receipt) - - - def pull(self, ark, **kwargs): - raise NotImplementedError - - - def put(self, halo, tag, **kwargs): - here = halo - there = self.bank / tag - logger.debug("put source {} to {}".format(str(here), str(there))) - started = datetime.datetime.now() - - completed = datetime.datetime.now() - receipt = { - 'tag': str(tag), - 'source': str(here), - 'URL': str(there), - 'started': started.isoformat(), - 'completed': completed.isoformat() - } - return (True, receipt) - + if isinstance(obj, PackingReceipt): + here = obj.tarp + there = obj.tag + elif isinstance(obj, pathlib.PosixPath): + here = obj + there = args[0] + assert isinstance(there, pathlib.PurePosixPath), "tag must be an instance of PurePosixPath" + else: + raise ValueError("obj must be an instance of either PackingReceipt or PosixPath") - def get(self, tag, halo, **kwargs): - shutil.copystat(self.bank / tag, halo) + return self.mover.put(here, there) #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑