diff --git a/bin/bastion.py b/bin/bastion.py index 2f91aee..b505e32 100755 --- a/bin/bastion.py +++ b/bin/bastion.py @@ -202,14 +202,13 @@ def run(self): if not isinstance(answer, isReceipt): raise ValueError("actions must respond with an instance of CARP.isReceipt") except Exception as err: - tb = traceback.format_exception(err) - answer = request.crashed( ''.join(tb), tb ) + answer = request.crashed(err) #-- always log crashes! - answer.context['log.scope'] = '*' + answer['log.scope'] = '*' #-- check for override on the logging scope. if "log.scope" in opts: - answer.context['log.scope'] = opts['log.scope'] + answer['log.scope'] = opts['log.scope'] #-- write the answer to the log, if there is a given log.scope self.remember(answer) @@ -345,8 +344,8 @@ def do_bank_zone(self, request): receipt = vault.push(asset) receipts.append(receipt) - all_succeeded = all(receipt.indicates_success for receipt in receipts]) - all_failed = all(receipt.indicates_failure for receipt in receipts]) + all_succeeded = all([receipt.indicates_success for receipt in receipts]) + all_failed = all([receipt.indicates_failure for receipt in receipts]) pushed = [receipt.body['blonde'] for receipt in receipts if receipt.indicates_success] @@ -367,8 +366,8 @@ def do_bank_asset(self, request): asset = site.asset(ark) vault = self.vault(asset.policy.vault) + request['log.scope'] = "site/{}".format(ark.site) receipt = vault.push(asset, client = self.hostname) - request.context['log.scope'] = 'log.scope': "site/{}".format(ark.site) if receipt.indicates_success: blonde = receipt.body['blonde'] diff --git a/lib/Bastion/CARP.py b/lib/Bastion/CARP.py index 4a6aafc..2b3d167 100644 --- a/lib/Bastion/CARP.py +++ b/lib/Bastion/CARP.py @@ -3,9 +3,12 @@ Common Action-Result/Report Protocol """ +import operator import uuid import traceback +import yaml + from Bastion.Common import toJDN class ReplyStatus(tuple): @@ -70,7 +73,7 @@ def __repr__(self): return "{}: {}".format(self.code, self.gloss) @property - def title(self): + def lede(self): return self.DEFAULT_CATEGORY_GLOSS[self.category] @property @@ -115,14 +118,24 @@ class isRequest: def __init__(self, action, *args, **kwargs): self.ID = kwargs.get('ID', str(uuid.uuid4())) self.action = action - self.when = kwargs.get('when', datetime.datetime.now()) + self.when = kwargs.get('opened', datetime.datetime.now()) self.args = dict(enumerate(args)) self.context = { } + excluded = {'ID', 'opened', 'context'} + for k, v in kwargs.items(): + if k not in excluded: + self.context[k] = v if 'context' in kwargs: for k, v in kwargs['context'].items(): self.context[k] = v + def __getitem__(self, k): + return self.context[k] + + def __setitem__(self, k, v): + self.context[k] = v + def toJDN(self, **kwargs): jdn = { 'ID': self.ID, @@ -141,12 +154,12 @@ def toJDN(self, **kwargs): return jdn -class isReceipt: - def __init__(self, request, status, obj, **kwargs): +class isResult: + def __init__(self, request, status, record, **kwargs): #-- POSITIONAL ARGS... #-- request is a REQUIRED 2-tuple of (action, [arg1, ...]) #-- status is a REQUIRED instance of ReplyStatus - #-- obj is a REQUIRED JSON serializable object, the outcome of the action implied by the request. + #-- record is a REQUIRED JSON serializable object, the outcome of the action implied by the request. #-- KEYWORD ARGS... #-- context: an OPTIONAL dict of extra key-value associations. #-- ID: an OPTIONAL given ID string, defaults to a randomly generated UUID @@ -158,8 +171,8 @@ def __init__(self, request, status, obj, **kwargs): self.ID = kwargs.get('ID', str(uuid.uuid4())) self.request = request self.status = status - self.lede = kwargs.get('lede', status.category) - self.record = obj + self.lede = kwargs.get('lede', status.lede) + self.record = record self.context = { } self.when = kwargs.get('when', datetime.datetime.now()) @@ -180,7 +193,7 @@ def toJDN(self, **kwargs): 'answered': self.when.isoformat( ), 'elapsed': self.elapsed.total_seconds(), 'context': { }, - 'record': toJDN(self.record) + 'record': toJDN(self.record) if self.record else None } #-- Populate the result's context (if any) for k, v in self.context.items( ): @@ -190,6 +203,7 @@ def toJDN(self, **kwargs): jdn = { } jdn['request'] = toJDN(self.request) jdn['request']['result'] = result + return jdn @property @@ -205,37 +219,51 @@ def inconclusive(self): return self.status.is_inconclusive -class Receipt(isReceipt): - pass - - -class Report(isReceipt): - def __init__(self, request, status, obj = None, *args, **kwargs): - isReceipt.__init__(self, request, status, *args, **kwargs) +class Report(isResult): + def __init__(self, request, status, record = None, *args, **kwargs): + isReceipt.__init__(self, request, status, record, *args, **kwargs) self._body = kwargs.get('report', None) - def changed(self, *args): - if 'body' in args: - self._body = None + @property + def body(self): + if getattr(self, '_body', None) is None: + self._body = self.toMD() + return self._body + + def toMD(self): + """ + I answer a markdown representation of self. + Base class is naive and just returns an empty string. + Override in subclasses for detailed output. + """ + return "" def toJDN(self, **kwargs): jdn = isReceipt.toJDN(self, **kwargs) jdn['request']['result']['report'] = self.body return jdn - @property - def body(self): - if getattr(self, '_body', None) is None: - #-- encode my object, if I have one, to a YAML string. - if self.record: - self._body = toJDN(self.record) - else: - self._body = "" - return self._body - def __str__(self): - return yaml.dump(self.toJDN(), default_flow_style = False, sort_keys = True) + return yaml.dump(self.toJDN(), default_flow_style = False, indent = 3, sort_keys = True) + + @classmethod + def Success(cls, request, record = None, *args, **kwargs): + return cls(request. ResultStatus.Ok, record, *args, **kwargs) + @classmethod + def Failed(cls, request, record = None, *args, **kwargs): + return cls(request, ResultStatus.Failed, record, *args, **kwargs) + + @classmethod + def Crashed(cls, request, record = None, *args, **kwargs): + if isinstance(record, Exception): + return ReportException(request, ResultStatus.Crashed, record, *args, **kwargs) + else: + return cls(request, ResultStatus.Crashed, record, *args, **kwargs) + + @classmethod + def Inconclusive(cls, request, record = None, *args, **kwargs): + return cls(request, ResultStatus.Inconclusive, record, *args, **kwargs) class ReportException(Report): @@ -253,32 +281,21 @@ def __init__(self, request, *args, **kwargs): assert status.indicates_failure, "ReportException must be created with a reply status indicating failure" assert isinstance(errobj, Exception), "error object must be an instance of Exception" - Report.__init__(self, request, status, errobj) - - self.xobj = x - self.xname = type(xobj).__name__ - self.xmsg = str(xobj) + tracer = ''.join( traceback.format_exception(errobj) ) - def - - def __str__(self): - return ''.join( traceback.format_exception(self.xobj) ) + Report.__init__(self, request, status, errobj, body = tracer) class Request(isRequest): - #-- Subclasses can set their own REPORT. - #-- This is used mostly in the convenience methods: .succeeded, .failed, etc. - REPORTS = { } #-- An optional mapping of ReplyStatus code to report class or function. - - def succeeded(self, *args, **kwargs): - return Report(self, ResultStatus.Ok, *args, **kwargs) + def succeeded(self, record = None, *args, **kwargs): + return Report.Success(self, record, *args, **kwargs) - def failed(self, *args, **kwargs): - return Report(self, ResultStatus.Failed, *args, **kwargs) + def failed(self, record = None, *args, **kwargs): + return Report.Failed(self, record, *args, **kwargs) - def crashed(self, *args, **kwargs): - return Report(self, ResultStatus.Crashed, *args, **kwargs) + def crashed(self, record = None, *args, **kwargs): + return Report.Crashed(self, record, *args, **kwargs) - def inconclusive(self, *args, **kwargs) - return Report(self, ResultStatus.Inconclusive, *args, **kwargs) + def inconclusive(self, record = None, *args, **kwargs): + return Report.Inconclusive(self, record, *args, **kwargs) diff --git a/lib/Bastion/Common.py b/lib/Bastion/Common.py index 346c9d3..89b616e 100644 --- a/lib/Bastion/Common.py +++ b/lib/Bastion/Common.py @@ -30,6 +30,13 @@ class Thing: def __init__(self, **kwargs): for k, v in kwargs.items(): setattr(self, k, v) + self._varnames = list(kwargs.keys()) + + def toJDN(self, **kwargs): + jdn = { } + for v in self._varnames: + jdn[v] = getattr(self, v) + return jdn def RDN(x): @@ -93,7 +100,18 @@ def __call__(self, subject, **kwargs): def _toJDN_date(x, **kwargs): #-- x is an instance of datetime.date return x.isoformat() + +def _toJDN_Exception(x, **kwargs): + #-- x is an instance of Exception + fqn = "{}.{}".format(type(x).__module__, type(x).__name__) + return {"exception": fqn, "message": str(x)} + +def _toJDN_path(x, **kwargs): + return str(x) + toJDN.spells.append( (datetime.date, _toJDN_date) ) +toJDN.spells.append( (Exception, _toJDN_Exception) ) +toJDN.spells.append( (pathlib.PurePath, _toJDN_path) ) class entity: @@ -203,6 +221,9 @@ def __str__(self): def __repr__(self): return str(self) + def toJDN(self, **kwargs): + return str(self) + @property def ref(self): """ diff --git a/lib/Bastion/Model.py b/lib/Bastion/Model.py index 07ea99f..55973c6 100644 --- a/lib/Bastion/Model.py +++ b/lib/Bastion/Model.py @@ -3,7 +3,7 @@ """ import pathlib -from .Common import RDN, CURIE, Slug40, entity +from .Common import RDN, CURIE, Slug40, entity, toJDN class ARK(tuple): @@ -87,6 +87,17 @@ def badge(self): return Slug40(str(self.CURIE)) +class isActor: + """ + Abstract class for actors. + Actors are the "do-ers" in the model. + All action methods should return an instance of Bastion.CARP.Report + """ + def perform(self, request): + #-- given a request object, I read the request then delegate to one of my methods. + raise NotImplementedError + + class isVault: """ abstract class for all vaults. @@ -103,6 +114,24 @@ class Vault(isVault): def __init__(self, name, **kwargs): self.name = name + #---------------------------------------------------------------------------- + #-- BEGIN ACTOR ACCESSORS | + #↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ + @property + def clerk(self): + #-- Answers the clerk for this vault. + raise NotImplementedError + + @property + def mover(self): + #-- Answers a mover for this vault. + raise NotImplementedError + + @property + def packer(self): + #-- Answers a packer (if needed) for this vault. + raise NotImplementedError + #---------------------------------------------------------------------------- #-- BEGIN CLERK DELEGATION | #-- These methods are basically pass-throughs to the clerk's implementation | @@ -135,7 +164,6 @@ def manifest(self, ark): Given an ARK, I answer the manifest of held objects (aka BLONDEs). """ return self.clerk.manifest(ark) - #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ #-- END CLERK DELEGATES | #---------------------------------------------------------------------------- @@ -152,27 +180,6 @@ def provision(self, *args): """ return self.mover.provision(*args, **kwargs) - def pack(self, asset, basis = None, **kwargs): - """ - Given a local asset, I package (.tar, .zip, etc) the asset into my scratch (spool) space. - Without a given basis, I package everything (i.e. a full backup). - When a basis is given as either a datetime or an anchor (BLONDE), I do a differential backup. - I answer a tuple like (blonde, tag, repo, packaged) - Where... - * blonde is the BLONDE of the newly created archive - * tag is the relative path to the archive file - * spool is the absolute path to the folder holding the new archive file. - * package is the name of the archive file in the spool folder - """ - self.packer.pack(asset, basis, **kwargs) - - def unpack(self, halo, root, **kwargs): - """ - Given a local packed archive object (e.g. .tar, .zip, etc.) at halo, - I unpack the archive into the given (local) root path. - """ - return self.packer.unpack(halo, root, **kwargs) - def push(self, asset, basis = None, **kwargs): """ Given an asset, I push a backup of the asset to this vault. @@ -209,9 +216,35 @@ def get(self, tag, halo, **kwargs): download the object and store it in the local file designated by halo. """ return self.mover.get(tag, halo, **kwargs) + #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ + #-- END MOVER DELEGATES | + #---------------------------------------------------------------------------- + + #----------------------------------------------------------- + #-- BEGIN PACKER DELEGATES | + #-- These methods are pass throughs to the attached packer.| + #↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓ + def pack(self, asset, basis = None, **kwargs): + """ + Given a local asset, I package (.tar, .zip, etc) the asset into my scratch (spool) space. + Can be invoked as... + .pack(request) + .pack(asset) - constructs an archive file for a full backup + .pack(asset, basis) - constructs an archive file for a differential backup relative to the given basis (BLONDE) + .pack(asset, since) - constructs an archive file for a differential backup relative to since (datetime) + On success, I answer a report that wraps an instance of Packers.CARP.Packed + On failure, I answer a Report.Failed + """ + self.packer.pack(asset, basis, **kwargs) + def unpack(self, halo, root, **kwargs): + """ + Given a local packed archive object (e.g. .tar, .zip, etc.) at halo, + I unpack the archive into the given (local) root path. + """ + return self.packer.unpack(halo, root, **kwargs) #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ - #-- END CLERK DELEGATES | + #-- END PACKER DELEGATES | #---------------------------------------------------------------------------- def configured(self, conf): @@ -316,7 +349,7 @@ class isSite: pass -class isClerk: +class isClerk(isActor): """ abstract class for metadata and file management specific to the capabilities of a given vault type. """ @@ -362,7 +395,7 @@ def manifest(self, ark): raise NotImplementedError -class isPacker: +class isPacker(isActor): def __init__(self, vault): assert isinstance(vault, isVault), "vault must be an instance of Bastion.Model.isVault" @@ -391,7 +424,7 @@ def unpack(self, halo, root, **kwargs): -class isMover: +class isMover(isActor): """ abstract class for file movement in to and out of a specific vault type. """ @@ -477,3 +510,8 @@ def entity_isMover(self): entity.isZone = property(entity_isZone) entity.isSite = property(entity_isSite) entity.isVault = property(entity_isVault) + +def _toJDN_isAsset(asset): + return str(asset.CURIE) + +toJDN.spells.append( (isAsset, _toJDN_isAsset) ) diff --git a/lib/Bastion/Movers/CARP.py b/lib/Bastion/Movers/CARP.py index d540884..a79dab2 100644 --- a/lib/Bastion/Movers/CARP.py +++ b/lib/Bastion/Movers/CARP.py @@ -1,25 +1,124 @@ -import os -import pathlib -import subprocess -import operator -import datetime -import json -import socket -import logging -import getpass -import shutil -import tarfile +""" +Bastion.Movers.CARP + +Common Action-Result Protocol +""" + import logging from Bastion.Common import Thing, Unknown -import Bastion.Model +from Bastion.Model import Request, isAsset, isResult from Bastion.Curator import Manifest, BLONDE, Snap -from Bastion.CARP import Request, Receipt, ResultStatus +from Bastion.CARP import * + + +logger = logging.getLogger(__name__) + + +class PushReceipt: + def __init__(self, packed, moved): + #-- packed - the report structure for the pack operation. + #-- moved - the report structure for the move operation. + assert isinstance(packed, isResult), "packed must be an instance of Bastion.CARP.isResult" + assert isinstance(moved, isResult), "moved must be an instance of Bastion.CARP.isResult" + self.pack = packed + self.put = moved + + def toJDN(self, **kwargs): + return [ toJDN(self.pack), toJDN(self.put) ] class PutRequest(Request): - pass + """ + Constructs a request for putting a given local file. + """ + def __init__(self, halo, rendpoint, tag): + """ + halo - is a path to a local file. + rendpoint - is a string (or object) that describes the remote endpoint. + tag - the tag that will identify this file on the remote system, typically a string, or pathlib.Path + """ + self.local = halo + self.remote = Thing(endpoint = endpoint, tag = tag) + + def toJDN(self, **kwargs): + jdn = { + 'local': str(self.local), + 'remote' = { + 'endpoint': toJDN(self.remote.endpoint), + 'tag': toJDN(self.remote.tag) + } + } + return jdn + + +class PushRequest(Request): + """ + Constructs a request for pushing a given asset. + Can be invoked as... + PushRequest(request) - casts a generic request into an instance of PackRequest + PushRequest(asset) - implies a full backup of the asset + PushRequest(asset, since) - implies a differential backup of the asset relative to the given datetime (since) + PushRequest(asset, basis) - implies a differential backup of the asset relative to the given basis (BLONDE) + """ + def __init__(self, subject, *args, **kwargs): + if isinstance(subject, Request): + Request.__init__("PushRequest", ID = subject.ID, opened = subject.when, context = subject.context) + + else: + assert isinstance(subject, isAsset), "PushRequest should be given an instance of isAsset" + asset = subject + basis = args[0] if args else None + whence = None + genus = None + + if basis: + detail = 'D' + if isinstance(basis, BLONDE): + #-- I was given a BLONDE (a reference to a full backup) + whence = basis.when.earliest + genus = basis.genus + blonded = BLONDE.forDiffBackup(basis) + if isinstance(basis, datetime.datetime): + whence = basis + genus = "___" + blonded = BLONDE(asset.ARK, detail, genus) + else: + detail = 'F' + blonded = BLONDE.forFullBackup(asset.ARK) + + xtras = { + 'blonde': blonded, + 'detail': detail, + 'since': whence, + 'basis': basis, + 'genus': genus, + 'asset': asset, + 'halo': str(asset.halo) + } + + Request.__init__("PushRequest", asset, **xtras) + + @property + def asset(self): + return self['asset'] + + @property + def basis(self): + return self['basis'] + + @property + def detail(self): + return self['detail'] + + @property + def blonde(self): + return self['blonde'] + @property + def since(self): + return self['since'] -class PutReceipt(Receipt): - pass + @property + def genus(self): + return self['genus'] diff --git a/lib/Bastion/Movers/SFTP.py b/lib/Bastion/Movers/SFTP.py index 47f91cc..d25eaef 100644 --- a/lib/Bastion/Movers/SFTP.py +++ b/lib/Bastion/Movers/SFTP.py @@ -50,55 +50,6 @@ def provision(self, *args): repo = self.scurler / ark.site / ark.zone / ark.asset return repo.mkdir(parents = True, exist_ok = True) - def pack(self, asset, basis = None, **kwargs): - """ - Given a local asset, I package (.tar, .zip, etc) the asset into my scratch (spool) space. - Without a given basis, I package everything (i.e. a full backup). - When a basis is given as either a datetime or an anchor (BLONDE), I do a differential backup. - I answer an instance of Bastion.Model.PackingReceipt - """ - detail = 'F' - whence = None - basis = None - blonded = None - tarp = None - ark = asset.ARK - opts = { } - - if basis: - detail = 'D' - if isinstance(basis, BLONDE): - #-- I was given a BLONDE (a reference to a full backup) - anchor = basis - whence = anchor.when.earliest - genus = anchor.genus - blonded = BLONDE.forDiffBackup(anchor) - - if isinstance(basis, datetime.datetime): - whence = basis - genus = "___" - blonded = BLONDE(asset.ARK, detail, genus) - - opts['since'] = whence - - else: - blonded = BLONDE.forFullBackup(asset.ARK) - - package = "{}.tar".format(str(blonded)) - tag = pathlib.PurePosixPath(ark.site) / ark.zone / ark.asset / package - spooled = (self.scratch / tag) - spool = spooled.parent - - #-- assure that the scratch path exists and all of the subpaths that create the repo folder for this asset. - spool.mkdir(parents = True, exist_ok = True) - - #-- use the built-in python tar archiver, using "pax" (POSIX.1-2001) extensions. - pax((self.scratch / tag), asset.halo, **opts) - - #-- Answer the receipt. - return receipt - - def push(self, asset, basis = None, **kwargs): """ Given an asset, I push a backup of the asset to this vault. @@ -107,9 +58,13 @@ def push(self, asset, basis = None, **kwargs): {asset} - an instance of Bastion.Model.isAsset {basis} - can be a datetime or a BLONDE. """ - packed = self.pack(asset, basis, **kwargs) + packing = self.vault.pack(asset, basis, **kwargs) + + #-- if the pack operation failed, we return the failure report. + if packing.failed: + return packing - transferred = self.put(packed.spooled, packed.opts['tag']) + transferred = self.put(packing.spooled, packed.opts['tag']) if transferred: #-- clean up! diff --git a/lib/Bastion/Packers/CARP.py b/lib/Bastion/Packers/CARP.py index 83799f2..fac1c14 100644 --- a/lib/Bastion/Packers/CARP.py +++ b/lib/Bastion/Packers/CARP.py @@ -8,29 +8,101 @@ logger = logging.getLogger(__name__) -class PackReport(Report): - def __init__(self, request, status, obj = None, **kwargs): - Report.__init__(self, request, status, obj, **kwargs) + +class PackingReceipt: + def __init__(self, asset, tag, packaged, tarp): + #-- asset:Bastion.Model.isAsset + #-- tag:pathlib.PurePosixPath + #-- packaged:list + #-- tarp:pathlib.PosixPath + assert isinstance(asset, isAsset), "asset must be an instance of Bastion.Model.isAsset" + assert isinstance(tag, pathlib.PurePosixPath), "tag must be pathlib.PurePosixPath" + assert isinstance(tarp, pathlib.PosixPath), "tarp must be pathlib.PosixPath" + + self.asset = asset + self.tag = tag + self.packaged = packaged + self.tarp = tarp + self.blonde = BLONDE.decode(tag.name.stem) + + def toJDN(self): + jdn = { + 'asset': str(self.asset.CURIE), + 'tag': str(self.tag), + 'packaged': list(self.packaged), + 'blonde': str(self.blonde), + 'tarp': str(self.tarp) + } + return jdn class PackRequest(Request): - def __init__(self, asset, spooled, basis = None, **kwargs): - #-- Which asset will be packed? - #-- If differential backup, what basis is used for determining changes? - Request.__init__(self, kwargs.get("action", "pack")) - - self.context['asset'] = str(asset.CURIE) - self.context['halo'] = str(asset.halo) - - if basis: - self.context['detail'] = 'D' - self.context['basis'] = str(kwargs['basis']) - self.context['whence'] = kwargs['whence'].isoformat() - if 'genus' in kwargs: - self.context['genus'] = kwargs['genus'] + """ + Constructs a request for packing a given asset. + Can be invoked as... + PackRequest(request) - casts a generic request into an instance of PackRequest + PackRequest(asset) - implies a full backup of the asset + PackRequest(asset, since) - implies a differential backup of the asset relative to the given datetime (since) + PackRequest(asset, basis) - implies a differential backup of the asset relative to the given basis (BLONDE) + """ +# def __init__(self, asset, basis = None, **kwargs): + def __init__(self, subject, *args, **kwargs): + if isinstance(subject, Request): + Request.__init__("PackRequest", ID = subject.ID, opened = subject.when, context = subject.context) + else: - self.context['detail'] = 'F' + asset = subject + basis = args[0] if args else None + whence = None + genus = None + + if basis: + detail = 'D' + if isinstance(basis, BLONDE): + #-- I was given a BLONDE (a reference to a full backup) + whence = basis.when.earliest + genus = basis.genus + blonded = BLONDE.forDiffBackup(basis) + if isinstance(basis, datetime.datetime): + whence = basis + genus = "___" + blonded = BLONDE(asset.ARK, detail, genus) + else: + detail = 'F' + blonded = BLONDE.forFullBackup(asset.ARK) + + xtras = { + 'blonde': blonded, + 'detail': detail, + 'since': whence, + 'basis': basis, + 'genus': genus, + 'asset': asset, + 'halo': str(asset.halo) + } + + Request.__init__("PackRequest", asset, **xtras) + + @property + def asset(self): + return self['asset'] + + @property + def basis(self): + return self['basis'] + + @property + def detail(self): + return self['detail'] + + @property + def blonde(self): + return self['blonde'] - def succeeded(self, catalog, *args, **kwargs): - return PackReport(self, ResultStatus.Ok, catalog) + @property + def since(self): + return self['since'] + @property + def genus(self): + return self['genus'] diff --git a/lib/Bastion/Packers/TARs.py b/lib/Bastion/Packers/TARs.py index 473de1b..e82101d 100644 --- a/lib/Bastion/Packers/TARs.py +++ b/lib/Bastion/Packers/TARs.py @@ -10,7 +10,8 @@ import logging from Bastion.Common import * -from Bastion.CARP import ReportException +from Bastion.CARP import Report +from Bastion.Packers.CARP import PackRequest, PackingReceipt from Bastion.Chronology import logger = logging.getLogger(__name__) @@ -71,124 +72,74 @@ def pax(tarp, folder, **kwargs): -class PaxReportSuccess(Report): - TEMPLATE = """ -# REQUEST {UUID} -Request ({request}) opened at {opened}. -# RESULT {UUID} {code} {gloss} -{cls} answered request at {answered} ({elapsed}). -Archive created for {asset} at host area path {halo}. -Archive is {size} and contains {toc} files. -## CATALOG -The following items are included in the archive... -{items} -""" - TEMPLATE_CATALOG_ITEM = "* {size} {modified} {path}" - - def __init__(self, request, catalog): - Report.__init__(self, request, Resultstatus.Ok, catalog) - - def __str__(self): - """ - Answers a textual representation of self which is the "print" form of self. - """ - #-- ask the request to generate a markdown section for itself. - h1request = self.request.toMD() - - #-- build the "madlibs" structure to substitute into the result section. - h1result = TEMPLATE_RESULT.format(**madlibs) - - #-- build the catalog (sub)section - #-- first, we build the items stanza... - txitems = '\n'.join( [TEMPLATE_CATALOG_ITEM.format(**entry) for entry in self.catalog] ) - h2catalog = TEMPLATE_H2CATALOG.format(items = txitems) - - - - class Packer(Bastion.Model.isPacker): def __init__(self, vault, **kwargs): Bastion.Model.isPacker.__init__(self, vault) - #-- find a scratch folder in order of lowest (default) to highest priority. #-- 1. /tmp #-- 2. vault's scratch folder (if any) #-- 3. "scratch" kwarg override self.scratch = asPath( prefer([kwargs.get("scratch", None), getattr(self.vault, "scratch", None), "/tmp"]) ) - def pack(self, asset, basis = None, **kwargs): + def pack(self, subject, *args, **kwargs): """ Given a local asset, I package (.tar, .zip, etc) the asset into my scratch (spool) space. + Can be invoked as... + .pack(request) + .pack(asset) + .pack(asset, basis) + .pack(asset, since) Without a given basis, I package everything (i.e. a full backup). When a basis is given as either a datetime or an anchor (BLONDE), I do a differential backup. - I answer a tuple like (blonde, tag, repo, packaged) + On success, I answer a report that wraps an instance of Packers.CARP.Packed + On failure, I answer a ReportException Where... * blonde is the BLONDE of the newly created archive * tag is the relative path to the archive file * spool is the absolute path to the folder holding the new archive file. - * package is the name of the archive file in the spool folder + * packaged is the name of the archive file in the spool folder """ - detail = 'F' - whence = None - basis = None - genus = None - blonded = None - tarp = None - ark = asset.ARK - opts = { } - - if basis: - detail = 'D' - if isinstance(basis, BLONDE): - #-- I was given a BLONDE (a reference to a full backup) - anchor = basis - whence = anchor.when.earliest - genus = anchor.genus - blonded = BLONDE.forDiffBackup(anchor) - - if isinstance(basis, datetime.datetime): - whence = basis - genus = "___" - blonded = BLONDE(asset.ARK, detail, genus) - - opts['since'] = whence - + #-- grok my dispatch... + if isinstance(subject, PackRequest): + #-- .pack(request:PackRequest) + request = subject + elif isinstance(subject, Request): + #-- .pack(request:isRequest) + request = PackRequest(subject) else: - blonded = BLONDE.forFullBackup(asset.ARK) + #-- .pack(asset) + #-- .pack(asset, basis) + #-- .pack(asset, since) + request = PackRequest(subject, *args, **kwargs) - package = "{}.tar".format(str(blonded)) - tag = pathlib.PurePosixPath(ark.site) / ark.zone / ark.asset / package - spool = (self.scratch / tag).parent + ark = request.asset.ARK + package = "{}.tar".format(str(request.blonde)) + tag = pathlib.PurePosixPath(ark.site) / ark.zone / ark.asset / package + spool = (self.scratch / tag).parent + tarp = self.scratch / tag #-- assure that the scratch path exists and all of the subpaths that create the repo folder for this asset. spool.mkdir(parents = True, exist_ok = True) - #-- generate a formal request object. - context = { } - if whence: - context['whence'] = whence - if genus: - context['genus'] = genus - request = PackRequest(asset, basis, **context) - - #-- use the built-in python tar archiver, using "pax" (POSIX.1-2001) extensions. try: - pax((self.scratch / tag), asset.halo, **opts) + #-- use the built-in python tar archiver, using "pax" (POSIX.1-2001) extensions. + pax(tarp, asset.halo, **opts) except Exception as err: - report = ReportException(request, err) + report = Report.Failed(request, err) else: - report = PaxReportSuccess(request) - - #↑↑↑↑↑↑ Need to do a clean version of what exactly the catalog is and how its made and also how an exception is described in the failure report. + #-- create an answer frame that will be part of the report. + packed = PackingReceipt(request.asset, tag, catalog(tarp), tarp) + report = Report.Success(request, packed) #-- answer the BLONDE of the newly created package. - return (blonded, tag, spool, package) - + return report - def unpack(self, halo, root, **kwargs): + def unpack(self, subject, *args, **kwargs): """ Given a local packed archive object (e.g. .tar, .zip, etc.) at halo, I unpack the archive into the given (local) root path. + .unpack(request) + .unpack(halo, root) """ raise NotImplementedError @@ -204,6 +155,6 @@ def catalog(tarp): for info in tark.getmembers(): sz = info.size mq = quantim( datetime.datetime.fromtimestamp(info.mtime) ) - pt = pathlib.PurePosixPath(info.name) - cats.append( (sz, mq, pt) ) + #pt = pathlib.PurePosixPath(info.name) + cats.append( (sz, mq, info.name) ) return cats diff --git a/lib/Bastion/Vaults/BFD.py b/lib/Bastion/Vaults/BFD.py index 6172fca..1ae393f 100644 --- a/lib/Bastion/Vaults/BFD.py +++ b/lib/Bastion/Vaults/BFD.py @@ -16,6 +16,7 @@ from Bastion.Curator import Manifest, BLONDE, Snap from Bastion.Clerks.BFD import Clerk from Bastion.Movers.BFD import Mover +import Bastion.Packers.TARs logger = logging.getLogger(__name__) @@ -48,12 +49,11 @@ def configured(self, conf): def changed(self, *args, **kwargs): #-- React to internal events that can invalidate cached items, etc. for aspect in args: - if aspect in ('clerk', 'mover'): + if aspect in ('clerk', 'mover', 'packer'): setattr(self, '_{}'.format(aspect), None) return self @property -<<<<<<< HEAD def sites(self): """ I answer a sorted collection of the names of all sites known to this vault. @@ -92,122 +92,88 @@ def assets(self, site, zone): def manifest(self, *args): """ I answer a manifest of the named asset... - manifest(ark) - manifest(site, zone, asset) + .manifest(ark) + .manifest(asset) + .manifest(site, zone, asset) """ if len(args) == 1: arg = args[0] + if isinstance(arg, Bastion.Model.ARK): + ark = arg + repo = self.bank / ark.site / ark.zone / ark.asset + blondes = [entry.name for entry in repo.iterdir() if entry.is_file()] + return Bastion.Curator.Manifest(ark, blondes) if isinstance(arg, Bastion.Model.isAsset): - ark = arg.ARK - else: - ark = arg - return self._manifest_ark( ark ) + return self.manifest(arg.ARK) elif len(args) == 3: - return self._manifest_site_zone_asset( args[0], args[1], args[2] ) - else: - raise ValueError + site, zone, asset = args + ark = ARK(site, zone, asset) + return self.manifest(ark) + raise ValueError def provision(self, *args): """ - provision(ark) - ensures that the site, zone, and asset folders exist. - provision(site, zone, asset_name) - an alias for provision(ark) + .provision(ark) - ensures that the site, zone, and asset folders exist. + .provision(asset) + .provision(site, zone, asset_name) - an alias for provision(ark) """ if len(args) == 1: - return self._provision_ark( args[0] ) - elif len(args) == 3: - return self._provision_site_zone_asset( args[0], args[1], args[2] ) - else: - raise ValueError - - def pack(self, asset, basis = None, **kwargs): - """ - Given a local asset, I package (.tar, .zip, etc) the asset into my scratch (spool) space. - Without a given basis, I package everything (i.e. a full backup). - When a basis is given as either a datetime or an anchor (BLONDE), I do a differential backup. - I answer the BLONDE for the package, relative to the local scratch (spool) space. - """ - detail = 'F' - whence = None - basis = None - blonded = None - tarp = None - ark = asset.ARK - opts = { } - - if basis: - detail = 'D' - if isinstance(basis, BLONDE): - #-- I was given a BLONDE (a reference to a full backup) - anchor = basis - whence = anchor.when.earliest - genus = anchor.genus - blonded = BLONDE.forDiffBackup(anchor) - - if isinstance(basis, datetime.datetime): - whence = basis - genus = "___" - blonded = BLONDE(asset.ARK, detail, genus) - - opts['since'] = whence - - else: - blonded = BLONDE.forFullBackup(asset.ARK) - - package = "{}.tar".format(str(blonded)) - tag = pathlib.PurePosixPath(ark.site) / ark.zone / ark.asset / package - spool = (self.scratch / tag).parent - - #-- assure that the scratch path exists and all of the subpaths that create the repo folder for this asset. - spool.mkdir(parents = True, exist_ok = True) + arg = args[0] - #-- use the built-in python tar archiver, using "pax" (POSIX.1-2001) extensions. - pax((self.scratch / tag), asset.halo, **opts) + if isinstance(arg, ARK): + ark = arg + repo = self.bank / ark.site / ark.zone / ark.assets + repo.mkdir(0o770, parents = True, exist_ok = True) + request = Request("ProvisionRequest") + request["ARK"] = str(ark.CURIE) + request["repo"] = str(repo) + return Report.Success(request) + if isinstance(arg, isAsset): + return self.provision(arg.ark) - #-- answer the BLONDE of the newly created package. - return (blonded, tag, spool, package) + elif len(args) == 3: + site, zone, asset = args + ark = ARK(site, zone, asset) + return self.provision(ark) + raise ValueError def push(self, asset, basis = None, **kwargs): """ Given an asset, I push a backup of the asset to this vault. - push(asset) - creates a full backup in this vault, creating a new base for differentials - push(asset, basis) - creates a differential backup relative to the given basis. + .push(asset) - creates a full backup in this vault, creating a new base for differentials + .push(asset, basis) - creates a differential backup relative to the given basis. + .push(asset, since) - creates a differential backup relative to the given datetime. {asset} - an instance of Bastion.Model.isAsset {basis} - can be a datetime or a BLONDE. """ - request = CARP.Request("{}.push".format(self.PROTOCOL)) - request.args['asset'] = CURIE(asset) - request.args['vault'] = self.name + #-- Create the request... + request = Request("PushRequest") + request["asset"] = str(asset.CURIE) if basis: if isinstance(basis, datetime.datetime): - request.args['basis'] = basis.isoformat() - elif isinstance(basis, BLONDE): - request.args['basis'] = str(basis) + request['since'] = basis else: - raise ValueError('.push takes basis as an instance of BLONDE or datetime') + request['basis'] = basis - packing_receipt = self.pack(asset, basis, **kwargs) + packing = self.pack(asset, basis, **kwargs) - if packing_receipt.indicates_success: - #-- assure that the bank exists. - (self.bank / tag).parent.mkdir(parents = True, exist_ok = True) + #-- if the pack operation failed, we return the failure report. + if packing.failed: + return packing - copy_receipt = self.put(self.scratch / tag, tag) + transfer = self.put(packing.spooled, packed.opts['tag']) - if copy_receipt.indicates_success: - #-- clean up! - (self.scratch / tag).unlink() - else: - #-- HALT AND CATCH FIRE!!! - #-- WORK HERE WORK HERE> + if transfer.succeeded: + #-- clean up! + (self.scratch / tag).unlink() - return receipt + return (transfer, blonde, receipt) def pull(self, ark, **kwargs): raise NotImplementedError - def put(self, halo, tag, **kwargs): here = halo there = self.bank / tag @@ -224,25 +190,31 @@ def put(self, halo, tag, **kwargs): } return (True, receipt) - def get(self, tag, halo, **kwargs): shutil.copystat(self.bank / tag, halo) -======= + + @property + def packer(self): + if getattr(self, '_packer', None) is None: + self._packer = Bastion.Packers.TARs.Packer(self) + return self._packer + + @property def clerk(self): - if getattr(self, '_clerk', None) is None: - self._clerk = Clerk(self) - return self._clerk ->>>>>>> 2e61ff33808d7cd60a7e31a3942455dfa0d7b0ce + """ + I am my own clerk. + """ + return self @property def mover(self): - if getatrr(self, '_mover', None) is None: - self._mover = Mover(self) - return self._mover + """ + I am my own mover. + """ + return self #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ #-- END Bastion.Model.Vault PROTOCOL | #------------------------------------- - Vault.register()