diff --git a/bin/bastion.py b/bin/bastion.py new file mode 100755 index 0000000..7de70e4 --- /dev/null +++ b/bin/bastion.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 + +import sys +import os +import pathlib +import logging + +logger = logging.getLogger() +logging.basicConfig(level = logging.DEBUG) + + +BIN_PATH = pathlib.Path(sys.argv[0]).absolute().parent +APP_PATH = BIN_PATH.parent +LIB_PATH = APP_PATH / 'lib' +LBX_PATH = APP_PATH / 'lib-exec' + +sys.path.insert(0, str(LIB_PATH)) + + +from Bastion.Common import * +from Bastion.Site import Site +from Bastion.Condo import * + + + +CONF_SEARCH_ORDER = [ + pathlib.Path('/etc/bastion'), + APP_PATH / 'etc', + pathlib.Path('~/.bastion').expanduser() +] + +if __name__ == '__main__': + comargs = dict(enumerate(sys.argv[1:])) + subject = comargs.get(0, 'help') + request = comargs.get(1, 'help') + + conf = Condex() + for folder in CONF_SEARCH_ORDER: + for confile in folder.rglob("conf-*.yaml"): + print(confile) + conf.load(folder / confile) + + if subject == 'keytab': + if request == 'refresh': + raise NotImplementedError + else: + Keytab(conf).help() + + if subject == 'backups': + sname = comargs[2] + site = Site(sname).configured(conf) + + +#bastion backups tidy idifhub +#bastion backups update idifhub +#bastion backups catalog idifhub +#bastion keytab refresh fortress diff --git a/bin/fossil.py b/bin/fossil.py index c2740ea..aa6921e 100644 --- a/bin/fossil.py +++ b/bin/fossil.py @@ -21,6 +21,8 @@ #----------------------------------------------------------------------------------------------------------------------- museum = Fossils(login = ndenny, conf = "idif.xlsx") #-- the default login can also be part of the conf workbook. +museum.vault("idifhub.ecn.purdue.edu", "LiDAR") + #-------------------------------------------------------------------------------------------------------------- #-- This is the "do everything" magic method that uses a lot of lower level methods to accomplish it's goal. | #-------------------------------------------------------------------------------------------------------------- @@ -36,7 +38,7 @@ #-- slug is the base32 encoding of the shake128/5 hash of the dataset's name. | #-- backups are also annotated with provenance in more readable English text. | #-------------------------------------------------------------------------------------------------------------- -report = museum.backup("QL2_3DEP_LiDAR_IN_2011_2013_l2") +report = museum.fossilize("QL2_3DEP_LiDAR_IN_2011_2013_l2") #-- This forces a differential backup of the given dataset RIGHT NOW. #-- Without an optional override, default behavior is to use the most recent full backup as the basis for ths @@ -62,3 +64,16 @@ def backup_full(folder, **kwargs): def backup_differential(folder): return backup(folder, 1, **kwargs) + +SITE = sys.argv[1] + +site = Site(SITE) +manifest = Manifest(SITE) +musuem = HPSS.Museum(site) + +for zone in manifest.zones: + for asset in manifest[zone].assets: + curator = museum.curator(zone, asset) + if curator.branches.most_recent.deposited > asset.longevity: + curator.curate(asset) + diff --git a/bin/out.txt b/bin/out.txt deleted file mode 100644 index 395339f..0000000 --- a/bin/out.txt +++ /dev/null @@ -1,23 +0,0 @@ -/home/ndenny: -drwxr-xr-x 2 ndenny student 253552 512 Wed Jul 19 15:28:56 2023 CfGS --rw------- 1 ndenny student 10 253552 DISK 12800 Thu Dec 7 06:33:23 2023 miniconda_0231207.tar --rw------- 1 ndenny student 10 253552 DISK 1312 Thu Dec 7 06:33:24 2023 miniconda_0231207.tar.idx --rw------- 1 ndenny itap 11 253552 TAPE 2307115900416 Tue Feb 9 10:19:17 2021 nexus_20210209.tar --rw------- 1 ndenny itap 11 253552 TAPE 414114080 Tue Feb 9 10:19:22 2021 nexus_20210209.tar.idx --rw------- 1 ndenny itap 11 253552 TAPE 2837237811712 Fri Nov 19 21:03:24 2021 nexus_20211119.tar --rw------- 1 ndenny itap 11 253552 TAPE 417398048 Fri Nov 19 21:03:30 2021 nexus_20211119.tar.idx --rw------- 1 ndenny itap 11 253552 TAPE 2016443183104 Thu May 19 16:15:49 2022 nexus_catalog_2022-05-19.tar --rw------- 1 ndenny itap 11 253552 TAPE 196347680 Thu May 19 16:15:51 2022 nexus_catalog_2022-05-19.tar.idx --rw------- 1 ndenny itap 11 253552 TAPE 18800947200 Mon Feb 14 15:24:16 2022 nexus_final_report_2022-02-14.tar --rw------- 1 ndenny itap 10 253552 DISK 1675040 Mon Feb 14 15:24:16 2022 nexus_final_report_2022-02-14.tar.idx --rw------- 1 ndenny itap 11 253552 TAPE 1634327330816 Mon Feb 14 17:55:47 2022 nexus_projects_2022-02-14.tar --rw------- 1 ndenny itap 11 253552 TAPE 1659624728576 Thu May 19 12:52:50 2022 nexus_projects_2022-05-19.tar --rw------- 1 ndenny itap 11 253552 TAPE 492911904 Thu May 19 12:52:57 2022 nexus_projects_2022-05-19.tar.idx --rw------- 1 ndenny student 11 253552 DISK 1569479696896 Thu Dec 7 09:18:17 2023 QL2_3DEP_LiDAR_IN_2011_2013_l2.tar --rw------- 1 ndenny student 10 253552 DISK 71481632 Thu Dec 7 10:03:45 2023 QL2_3DEP_LiDAR_IN_2011_2013_l2.tar.idx --rw------- 1 ndenny student 11 253552 DISK 2323521421824 Mon Dec 11 03:08:35 2023 QL2_3DEP_LiDAR_IN_2017_2019_l2.tar --rw------- 1 ndenny student 11 253552 DISK 124447008 Mon Dec 11 04:05:42 2023 QL2_3DEP_LiDAR_IN_2017_2019_l2.tar.idx --rw------- 1 ndenny student 11 253552 DISK 1575000025088 Mon Dec 11 05:53:04 2023 QL2_3DEP_LiDAR_IN_2017_2019_laz.tar --rw------- 1 ndenny student 10 253552 DISK 22099744 Mon Dec 11 06:32:41 2023 QL2_3DEP_LiDAR_IN_2017_2019_laz.tar.idx --rw------- 1 ndenny student 11 253552 DISK 8517799739904 Mon Dec 11 11:38:13 2023 QLX_3DEP_LiDAR_US_South.tar -drwxr-x--- 2 ndenny itap 253552 512 Thu Jul 16 05:51:12 2020 QUDT diff --git a/docs/idifhub_conf.xlsx b/docs/idifhub_conf.xlsx index f7c2748..2129d84 100644 Binary files a/docs/idifhub_conf.xlsx and b/docs/idifhub_conf.xlsx differ diff --git a/etc/conf-idifhub.yaml b/etc/conf-idifhub.yaml new file mode 100644 index 0000000..c98873d --- /dev/null +++ b/etc/conf-idifhub.yaml @@ -0,0 +1,69 @@ +sites: + idifhub: + host: idifhub.ecn.purdue.edu + logging: + level: DEBUG + path: /var/log/bastion + persist: + git: + repo: git@github.purdue.edu/ndenny/bastion-idif + key: /home/ndenny/.ssh/ndenny@rcac + + #-- policy defined with the site is the default policy for all RZs within this site. + policy: + history: 60 + drift: 30 + vault: fortress + + #-- declare any (resource) zones within the scope of this site. + zones: + #-- this is the long form for declaring a resource zone (Rz) + LiDAR: + root: /home/jinha/www/lidar + #-- each resource zone can declare its own policy + policy: + history: 60 + drift: 30 + vault: fortress + #-- this next example is the short form for declaring a reource zone + #-- this resource zone inherits all of its policy, etc. from the site in which it resides. + hyperspectral: /home/jinha/www/hsi + +assets: + idifhub: + #-- Top level keys are the names of (resource) zones for this site. + LiDAR: + #-- short form of an asset is just the folder or file that is immediately subordinate to the RZ's root + #-- assets declared using the short form will inherit their policy, etc. from superior contexts. + - QL2_3DEP-LiDAR_IN_2011_2013_l2 + #-- long form of an asset allows for policy override, etc. + - name: QL2_3DEP_LiDAR_IN_2017_2019 + policy: + history: 90 + drift: 30 + - QL2_3DEP_LiDAR_IN_2017_2019_l2 + - QL2_3DEP_LiDAR_IN_2017_2019_laz + - name: QLX_3DEP-LiDAR_US_South + about: low resolution LiDAR coverage of US South (census) Region + RDN: SOUTHLRZ + #-- this is a special case that includes all files and folders in the RZ's root folder as distinct assets. + hyperspectral: "*" + +vaults: + fortress: + protocol: HPSS + host: fortress.rcac.purdue.edu + login: ndenny + key: + path: /home/ndenny/.private/hpss.unix.keytab + refresh: + period: 60 + ssh: + host: data.rcac.purdue.edu + user: ndenny + key: /home/ndenny/.ssh/ndenny@rcac + + BFD: + protocol: copy + zone: /mnt/BFD/bastion + diff --git a/lab/bootstrap.py b/lab/bootstrap.py new file mode 100644 index 0000000..5b787c0 --- /dev/null +++ b/lab/bootstrap.py @@ -0,0 +1,32 @@ +import sys +import pathlib +import logging +import datetime + + +logger = logging.getLogger(__name__) +logging.basicConfig(level = logging.DEBUG) + +RUN_PATH = pathlib.Path(sys.argv[0]).absolute().parent +APP_PATH = RUN_PATH.parent +LIB_PATH = APP_PATH / 'lib' +LBX_PATH = APP_PATH / 'lib-exec' + +sys.path.insert(0, str(LIB_PATH)) + +from Bastion.Common import * +from Bastion.Site import Site +from Bastion.Condo import * +from Bastion.Curator import Snap + +CONF_SEARCH_ORDER = [ + pathlib.Path('/etc/bastion'), + APP_PATH / 'etc', + pathlib.Path('~/.bastion').expanduser() +] + +conf = Condex().load(APP_PATH / 'etc' / 'conf-idifhub.yaml') +site = Site('idifhub').configured( conf ) + +slug = site.zone('LiDAR').assets.any.RDN +latest = Snap.dub(slug, datetime.datetime.now(), "D", "XXV") diff --git a/lib/Bastion/Common.py b/lib/Bastion/Common.py index 666a5b6..a96e689 100644 --- a/lib/Bastion/Common.py +++ b/lib/Bastion/Common.py @@ -4,26 +4,275 @@ import json import hashlib import base64 +import numbers as pynumbers +import datetime +import string +import operator import yaml +DAY = datetime.timedelta(days = 1) +DAYS = DAY +SECOND = datetime.timedelta(seconds = 1) +SECONDS = SECOND +MINUTE = datetime.timedelta(minutes = 1) +MINUTES = MINUTE +HOUR = datetime.timedelta(hours = 1) +HOURS = HOUR + + + +class Thing: + def __init__(self, **kwargs): + for k, v in kwargs.items(): + setattr(self, k, v) + + +def RDN(x): + """ + Answers the relatively distinguishing name (RDN) of object, x. + If x is a string, it is assumed that x is the name. + If x is an object with a CN attribute, answers x.RDN + """ + if isinstance(x, str): + return x + elif hasattr(x, 'RDN'): + return x.RDN + else: + raise ValueError("RDN(x) - requires that object x have an .RDN property") + + + +class entity: + """ + I am syntactic sugar. + Mostly, I do type checking, e.g. + x = 4 + if entity(x).isNumeric: print("it's a number!") + """ + def __init__(self, subject): + self.subject = subject + + @property + def isNumeric(self): + return isinstance(self.subject, pynumbers.Real) + + @property + def isDuration(self): + return isinstance(self.subject, datetime.timedelta) + + @property + def isString(self): + return isinstance(self.subject, str) + + + +class UnknownType(object): + _ = None + + def __new__(cls): + if UnknownType._ is None: + UnknownType._ = object.__new__(cls) + return UnknownType._ + + def __repr__(self): + return "<|???|>" + +Unknown = UnknownType() + + + +class CURIE(tuple): + """ + a CURIE (or Compact URI) defines a generic, abbreviated syntax for expressing Uniform Resource Identifiers (URIs). + It is an abbreviated URI expressed in a compact syntax, and may be found in both XML and non-XML grammars. + A CURIE may be considered a datatype. + See ... + * https://en.wikipedia.org/wiki/CURIE + * https://www.w3.org/TR/2010/NOTE-curie-20101216/ + examples ... + + [unit:METER] + [doi:10.1000/182] + [RADISH:V9RPl4gIaYzB3_clVjDc] + """ + def __new__(cls, *args, **kwargs): + if len(args) == 1: + arg = args[0] + if isinstance(arg, cls): + return arg + + #-- ask the object if it already has a CURIE representation ... + f = getattr(arg, 'CURIE', None) + if f: + if isinstance(f, cls): + #-- Yes, the object does have an already prepared CURIE representation. + return f + if callable(f): + #-- No, the object doesn't have a CURIE already prepared, + #-- but it does have a method to create a CURIE representation. + return f( ) #-- invoke the object's CURIE creation method. + + #-- If I was given a string, I should be able to parse it. + if isinstance(arg, str): + t, p, q = cls.parse(arg) + return tuple.__new__(cls, (t, p, q)) + + if len(args) == 2: + itype = args[0] + path = args[1] + args = urllib.parse.urlencode(kwargs) if kwargs else None + return tuple.__new__(cls, (itype, path, args)) + + raise TypeError + + itype = property(operator.itemgetter(0)) + path = property(operator.itemgetter(1)) + args = property(operator.itemgetter(2)) + + def __str__(self): + if self.args: + return "[{}:{}?{}]".format(self.itype, self.path, self.args) + else: + return "[{}:{}]".format(self.itype, self.path) + + def __repr__(self): + return str(self) + + @property + def ref(self): + if self.args: + return "{}?{}".format(self.path, self.args) + else: + return self.path + + @staticmethod + def parse(c): + if (c[0] == '[') and (c[-1] == ']'): + cinner = c[1:-1] + if ':' in cinner: + i = cinner.find(':') + itype = cinner[:i] + objref = cinner[i+1:] + if '?' in objref: + j = objref.find('?') + path = objref[:j] + query = objref[j+1:] + else: + path = objref + query = None + + return (itype, path, query) + + raise ValueError + + def __matmul__(self, ns): + """ + I answer my reference iff my itype is the given namespace (ns). + Otherwise, I raise an exception. + """ + if self.itype == ns: + return self.ref + else: + raise TypeError + + +class Quantim: + """ + Quantized Time (Quantim). + I represent with ~ 1 minute of precision a date within the 3rd millenium. + (i.e. 2000 - 2999) + The quantized minute is described in base36 (0...9,A...Z). + With two digits, the day is divided into 1,296 quantums, each of which is ~ 66.67 seconds. + """ + + EN36 = list(string.digits + string.ascii_uppercase) + DE36 = dict([(c, i) for i, c in enumerate(EN36)]) + QUANTUM = 86400.0 / (36**2) + + def __init__(self, whence, separator = None): + self.separator = separator if (separator is not None) else '' + + if isinstance(whence, datetime.datetime): + year_starts = datetime.datetime(whence.year, 1, 1, 0, 0, 0) + adnl_seconds = (whence - year_starts).seconds + + self.dY = whence.year - 2000 + self.dD = (whence - year_starts).days + self.qM = int(adnl_seconds // Quantim.QUANTUM) + + elif isinstance(whence, str): + if self.separator: + words = whence.split(self.separator) + if len(words) == 3: + yW = words[0] + dW = words[1] + qW = words[2] + else: + raise Exception("Quantim:__init__ parse error for '{}'".format(whence)) + else: + if len(whence) == 8: + yW = whence[0:3] + dW = whence[3:6] + qW = whence[6:8] + else: + raise Exception("Quantim:__init__ parse error for '{}'".format(whence)) + + self.dY = int(yW) + self.dD = int(dW) + self.qM = (Quantim.DE36[qW[0]] * 36) + Quantim.DE36[qW[1]] + + else: + raise ValueError("Quantim:__init__ cannot create instance from whence '{}'".format(whence)) + + def __str__(self): + lsq = self.qM % 36 + msq = self.qM // 36 + xmap = list(string.digits + string.ascii_uppercase) + yW = "{:03d}".format(self.dY) + dW = "{:03d}".format(self.dD) + qW = xmap[msq] + xmap[lsq] + return self.separator.join([yW, dW, qW]) + + def datetime(self): + y = datetime.datetime(self.dY + 2000, 1, 1, 0, 0, 0) + elapsed_days = self.dD * DAYS + elapsed_seconds = ((self.qM * Quantim.QUANTUM) + (Quantim.QUANTUM / 2)) * SECONDS + return (y + elapsed_days + elapsed_seconds) + + @classmethod + def now(cls): + return cls(datetime.datetime.now()) + + def Slug40(text): """ - I generate a 5-character slug based on the given text. + I generate a 8-character slug based on the given text. The slug is generated by hashing the text using SHAKE128, then taking a 40-bit digest and encoding the digest using base32. """ h = hashlib.shake_128() h.update(text.encode('utf-8')) bs = h.digest(5) - return base64.b32encode(bs) + return base64.b32encode(bs).decode('utf-8') - -class Sable: +class canTextify: + """ + I am a trait for serialization using JSON and YAML. + """ def toJDN(self, **kwargs): - raise NotImplementedError + raise NotImplementedError(".toJDN is subclass responsibility") + + @classmethod + def fromJDN(cls, jdn, **kwargs): + raise NotImplementedError(".fromJDN is subclass responsibility") + + #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ + #-- host class protocol. | + #-- host classes must implement the methods, above. | + #----------------------------------------------------- def toJSON(self, **kwargs): jdn = self.toJDN(**kwargs) @@ -33,10 +282,6 @@ def toYAML(self, **kwargs): jdn = self.toJDN(**kwargs) return yaml.dump(jdn, default_flow_style = False, indent = 3) - @classmethod - def fromJDN(cls, jdn, **kwargs): - raise NotImplementedError - @classmethod def fromJSON(cls, js, **kwargs): jdn = json.loads(js) diff --git a/lib/Bastion/Condo.py b/lib/Bastion/Condo.py new file mode 100644 index 0000000..4959187 --- /dev/null +++ b/lib/Bastion/Condo.py @@ -0,0 +1,590 @@ +""" +Bastion.Condo + +I provide a nested mapping object, useful for configuration, etc. +Multi-part keys (typically separated by the dot "." element) are parsed into nested keys. + +Typical use starts with constructing a condo using the Condex( ) construction function, e.g. ... + +conf = Condex( ) +conf['some.nested.key'] = 'something' +conf['some.other.key'] = 'else' + +#------------------------------------ +#-- These two lines are equivalent. | +#------------------------------------ +print(conf['some.other.key']) +print(conf['some']['other']['key']) +""" +import sys +import os +import collections.abc as pycollections +import json +import datetime +import fnmatch +import logging +import pathlib +import csv + +try: + import yaml +except: + pass + +try: + import openpyxl +except: + pass + + + +#------------------------------------ +#-- Check that we're using Python 3 | +#------------------------------------ +assert(sys.version_info.major == 3) + + +#------------------------------- +#-- Get a handle to a logger. | +#------------------------------- +logger = logging.getLogger(__name__) + + +class ftuple(tuple): + """ + ftuple is a special case of a tuple that returns its own iterator when called. + This is used as a hack to allow my own (NTD) .keys (as a property) idiom to be + shoe-horned in where standard python might expect .keys to be a callable method + Specifically, dict.update(x) will first check to see if x has an attribute named "keys" + It then assumes that keys is a callable. In my own style, .keys returns a set or a tuple + (depending on the class) and neither sets nor tuples are normally callable. + ftuple should allow both styles to work. + """ + def __call__(self): + return iter(self) + + +class CxKeyView: + def __init__(self, cxnode): + self.nest = cxnode + + def dive(self, prekey = None): + for k, v in sorted(self.nest.children.items()): + k = CxKey(k) if prekey is None else (prekey / k) + if isinstance(v, CxNode): + for subk in v.keys.dive(k): + yield subk + else: + yield k + + @property + def deep(self): + return frozenset( self.dive() ) + + def matching(self, pattern): + spattern = str(pattern) + return list( sorted(filter(lambda k: fnmatch.fnmatch(str(k), spattern), self.deep)) ) + + def __contains__(self, k): + k = CxKey(k) + if k.head in self.nest.children.keys( ): + if k.tail.isNotEmpty: + return (k.tail in self.nest.children[k.head]) + else: + return True + else: + return False + + def __iter__(self): + return iter(sorted(self.nest.children.keys())) + + def __call__(self): + return iter(self) + + +class CxItem(tuple): + def __new__(cls, k, v): + return tuple.__new__(cls, (CxKey(k), v)) + + @property + def key(self): + return self[0] + + @property + def value(self): + return self[1] + + + +class CxKey(tuple): + def __new__(cls, k = None): + if k is None: + return tuple.__new__(cls, [None]) + + if isinstance(k, cls): + return k + + if isinstance(k, str): + tokens = list(map(lambda token: token.strip( ), k.split('.'))) + return cls(tokens) + + if isinstance(k, pycollections.Sequence): + return tuple.__new__(cls, k) + + if isinstance(k, CxNode): + if k.isChild: + return CxKey(k.parent) / k.name + else: + return CxKey(None) + + raise TypeError + + @property + def isEmpty(self): + if len(self) == 0: + return True + elif len(self) == 1: + return ( (self[0] is None) or (self[0] == '') ) + return False + + @property + def isNotEmpty(self): + if len(self) > 0: + if self[0]: + return True + return False + + @property + def head(self): + return self[0] + + @property + def tail(self): + return CxKey(self[1:]) + + def __str__(self): + return '' if self.isEmpty else '.'.join(map(str, self)) + + def __format__(self, *args): + return str(self) + + def __repr__(self): + return str(self) + + def __truediv__(self, other): + other = CxKey(other) + if self[0] is None: + return CxKey(other) + else: + return CxKey(tuple(self) + tuple(other)) + + + +class CxNode(object): + def __init__(self, parent = None, name = None, value = None): + self.children = { } + self.parent = parent + self.name = name + self.value = value + + def show(self, *args, **kwargs): + for item in self.walk(): + print("{item.key:40s} → {item.value}".format(item=item)) + + def walk(self): + """ + I dive into my nested structure and answer each CxItem in key order. + """ + for k in self.keys.dive(): + yield CxItem(k, self[k]) + + @property + def flattened(self): + """ + Answers a dictionary where all keys and subkeys are put into the same 0-level depth. + """ + return dict([(k, self[k]) for k in self.keys.deep]) + + def __iter__(self): + for k in sorted(self.children.keys()): + yield CxItem(k, self.children[k]) + + def __eq__(self, other): + if isinstance(other, CxNode): + return (self.flattened == other.flattened) + + if isinstance(other, pycollections.Mapping): + #-- Flatten the other mapping. + flatter = tuple( [(k, other[k]) for k in sorted(other.keys( ))] ) + return (self.flattened == flatter) + + raise TypeError + + def _dex(self): + return dict( self.walk ) + + def get(self, *args): + """ + gets the value of a given key. + .get(f, key) - answers the value of f(self[key]) + .get(f, key, default) - answers the value of f(self[key] or default) + .get(key) - answers the value associated with key + .get(key, default) - answers the value of self[key] or default + """ + if len(args) == 1: + #-- .get(key) + return self[ args[0] ] + + if len(args) == 2: + #-- .get(key, default) + if isinstance(args[0], str): + k, default = args + try: + return self[k] + except KeyError: + return default + + #-- .get(form, key) + if callable(args[0]): + xform, k = args + return xform(self[k]) + + if len(args) == 3: + #-- .get(form, key, default) + xform, k, default = args + return xform( self.get(k, default) ) + + raise ValueError + + def __contains__(self, k): + return (k in self.keys) + + def __getitem__(self, k): + k = CxKey(k) + if k.isEmpty: + return self + else: + if k.head in self.children: + if isinstance(self.children[k.head], CxNode): + return self.children[k.head][k.tail] + else: + v = self.children[k.head] + if callable(v): + return v(self.root, k) + else: + return v + return self.children[k.head] + else: + raise KeyError(k) + + def __setitem__(self, k, v): + k = CxKey(k) + if len(k) == 1: + self.children[k.head] = v + else: + if (k.head not in self.children): + self.children[k.head] = CxNode(self, k.head) + self.children[k.head][k.tail] = v + + def update(self, d): + if d is not None: + if isinstance(d, CxNode): + for k in d.keys.deep: + self[k] = d[k] + return self + + if isinstance(d, pycollections.Mapping): + for k in d.keys( ): + if isinstance(d[k], pycollections.Mapping): + if k not in self: + self[k] = CxNode(self, k) + self[k].update(d[k]) + else: + self[k] = d[k] + return self + + raise TypeError + + @property + def keys(self): + return CxKeyView(self) + + @property + def root(self): + """ + Answers a reference to the root node of this configuration tree. + """ + if self.parent is None: + return self + else: + return self.parent.root + + @property + def isRoot(self): + return True if self.parent is None else False + + @property + def isChild(self): + return True if self.parent is not None else False + + def toJDN(self): + d = { } + for k in self.keys.deep: + if k.tail.isEmpty: + d[str(k)] = self[k] + else: + d[str(k.head)] = self[k.head].toJDN( ) + return d + + def load(self, srcname, **kwargs): + loader = CxLoader(self) + return loader.load(srcname, **kwargs) + + def sed(self, tokens): + sedr = CxSEDR(self) + sedr.sed(tokens) + return self + + + +class CxLoader: + def __init__(self, node): + self.node = node + + def load(self, srcname, **kwargs): + srcpath = pathlib.Path(srcname) + + if srcpath.exists(): + logger.debug("CxNode/load: reading configuration from {}".format(srcpath)) + + form = kwargs.get('form', None) + if form is None: + form = srcpath.suffix[1:].upper() + else: + form = form.strip( ).upper( ) + + loaders = { + 'JSON': self.loadJSON, + 'JSN': self.loadJSON, + 'JS': self.loadJSON, + 'YML': self.loadYAML, + 'YAML': self.loadYAML, + 'CSV': self.loadCSV, + 'XLSX': self.loadXLSX + } + + loader = loaders.get(form, None) + + if loader: + loader(srcname, **kwargs) + else: + raise Exception("CxNode/load: I don't know how to handle files of form '{}'".format(form)) + else: + logger.error('CxNode/read: I cannot find the specified file: {}'.format(srcpath)) + raise FileNotFoundError(srcname) + + return self.node + + def loadJSON(self, fname, **kwargs): + blob = json.load(open(fname)) + self.node.update(blob) + return self.node + + def loadYAML(self, fname, **kwargs): + if 'yaml' in sys.modules: + blob = yaml.safe_load(open(fname)) + self.node.update(blob) + return self.node + + msg = "CxNode/loadYAML: yaml module not available" + logger.error(msg) + raise Exception(msg) + + def loadCSV(self, fname, **kwargs): + keyCol = kwargs.get('keyCol', 'key') + valCol = kwargs.get('valCol', 'value') + + with open(fname, 'rt') as src: + tbl = csv.DictReader(src) + for row in tbl: + k = row[keyCol] + v = row[valCol] + self.node[k] = v + + return self.node + + def loadXLSX(self, srcpath, sheet = None, **kwargs): + xlspath = pathlib.Path(srcpath) + if 'openpyxl' in sys.modules: + keyCol = kwargs.get('keyCol', 'A') + valCol = kwargs.get('valCol', 'B') + bgnRow = kwargs.get('bgnRow', 2) + + wb = openpyxl.load_workbook(str(xlspath)) + ws = None + if sheet: + if isinstance(sheet, str): + ws = wb[sheet] + elif isinstance(sheet, int): + #-- Get sheet by index + dex = dict(enumerate(wb.sheetnames)) + ws = dex[sheet] + + if not ws: + msg = "Condo/loadXLSX: .xlsx workbook sheets must be by name or index" + logger.error(msg) + raise ValueError(msg) + + for i in range(2, ws.max_row+1): + k = ws["{}{}".format(keyCol,i)].value + v = ws["{}{}".format(valCol,i)].value + if k is not None: + k = k.strip() + if k: + self.node[k] = v + + else: + msg = "CxNode/loadXLSX: openpyxl module not available" + logger.error(msg) + raise Exception(msg) + + return self.node + + + +class CxSEDR: + def __init__(self, node): + self.node + + def sed(self, tokens): + """ + Interpet the given list of tokens as an edit stream (aka "sed"). + + ^file (LOAD) reads the given file name into the configuration + key: (SET) sets a nested key to some value + key+ (SADD) adds a value to the key (set semantics) + key++ (BADD) adds a value to the key (bag semantics) + key- (SREM) removes a value from the key (set semantics) + key-- (BREM) removes all instances of the value from the key (bag semantics) + key! (ON) sets the key to True + key~ (OFF) sets the key to False + """ + state = 'SCANNING' + key = None + nakeds = [ ] + + for token in tokens: + logger.debug("CxSEDR.sed state is {} working on key '{}' ingesting token '{}'".format(state, key, token)) + if state == 'SCANNING': + op = 'SCANNING' #-- default to continuing the scanning state, unless otherwise set. + + if token[0] == '^': + #-- load the file + path = token[1:] + node.load(path) + + elif token[-1] == ':': + key = token[:-1] + op = 'SET' + + elif token[-1] == '!': + key = token[:-1] + self.node[key] = True + + elif token[-1] == '~': + key = token[:-1] + self.node[key] = False + + elif token[-1] == '+': + if token[-2:] == '++': + key = token[:-2] + op = 'BADD' + else: + key = token[:-1] + op = 'SADD' + + elif token[-1] == '-': + if token[-2:] == '--': + key = token[:-2] + op = 'BREM' + else: + key = token[:-1] + op = 'SREM' + + else: + #--------------------------------------------------------- + #-- the token doesn't appear to have any sed semantics, | + #-- so add it to the "naked" list. | + #--------------------------------------------------------- + nakeds.append(token) + + elif state == 'SET': + self.node[key] = token + op = 'SCANNING' + + elif state == 'SADD': + curval = self.node[key] if (key in self.node) else [] + + if isinstance(curval, pycollections.MutableSequence): + if token not in self.node[key]: + self.node[key].append(token) + + elif isinstance(curval, pycollections.MutableSet): + self.node[key].add(token) + + else: + self.node[key] = [curval, token] + + op = 'SCANNING' + + elif state == 'BADD': + curval = self.node[key] if (key in self) else [] + + #---------------------------------------------------------- + #-- if the current value is a mutable set, | + #-- convert the set to a list to handle the bag semantics | + #---------------------------------------------------------- + if isinstance(curval, pycollections.MutableSet): + self.node[key] = list(curval) + curval = self.node[key] + + if isinstance(curval, pycollections.MutableSequence): + self.node[key].append(token) + else: + self.node[key] = [curval, token] + + op = 'SCANNING' + + elif state == 'SREM': + if key in self: + curval = self.node[key] + if isinstance(curval, pycollections.MutableSet): + self.node[key].discard(token) + if isinstance(curval, pycollections.MutableSequence): + if token in self.node[key]: + self.node[key].remove(token) + op = 'SCANNING' + + elif state == 'BREM': + if key in self: + curval = self.node[key] + if isinstance(curval, pycollections.MutableSet): + self.node[key].discard(token) + elif isinstance(curval, pycollections.MutableSequence): + while token in self[key]: + self.node[key].remove(token) + state = op + + return nakeds + + + + + + +def Condex(*args, **kwargs): + c = CxNode( ) + for arg in args: + c.update(dict(arg)) + c.update(kwargs) + return c diff --git a/lib/Bastion/Curator.py b/lib/Bastion/Curator.py index 4732e9c..f564e3f 100644 --- a/lib/Bastion/Curator.py +++ b/lib/Bastion/Curator.py @@ -3,136 +3,77 @@ I provide mostly data structures for working with archives and backups. """ -from .Common import Sable, Slug40 -import hashlib -import base64 +import pathlib +from .Common import * -class Asset(Sable): - def __init__(name, path, about, **kwargs): - self.name = RDN - self.path = pathlib.Path(path) - self.about = about - self.RDN = None +#-- Archives > Anchors > Snaps +#-- An archive is a chronicle (time ordered series) of snaps. +#-- Each snap is a 2-tuple of (anchor, differential) +#-- An "anchor" is a full backup of the dataset. +#-- Each blob in the archive is recorded as ... +#-- {slug}-{quantim}-[A|D]{anchor} +#-- Where {slug} is the Slug40 encoding (a 8 character, base32 word) of the dataset name (relative to the Rz), +#-- {quantim} is the 8 character encoding of timestamp using the Quantim method +#-- A if the blob is an anchor (full backup) and D if the blob is a differential. +#-- {anchor} - is a 3 character random string that cannot conflict with any other anchors currently in the archive. - for kwarg in ['RDN']: - if kwarg in kwargs: - setattr(self, kwarg, kwargs[kwarg]) - if self.RDN is None: - self.RDN = Slug40(self.name) - - def toJDN(self, **kwargs): - jdn = { - '_type': "Curator.Asset", - 'name': self.name, - 'path': str(self.path), - 'about': self.about, - 'RDN': self.RDN - } - return jdn - - -class Archive(Sable): +class Archive(canTextify): """ - I represent a top-level archive of some dataset. - I am analagous to a git repository in that I may contain - multiple branches of object evolution. + I represent the chronicled archive of some dataset. + I hold a series of "snaps". """ - def __init__(self, name, **kwargs): - self.name = name - self.RDN = None - - for kwarg in ['RDN']: - if kwarg in kwargs: - setattr(self, kwarg, kwargs[kwarg]) - - if self.RDN is None: - self.RDN = Slug40(self.name) + def __init__(self, asset, **kwargs): + self.asset = asset + self.zone = asset.zone + self.site = asset.zone.site + self.lscat = [ ] #-- list of files that are in this archive. + + def snaps(self): + #-- Answers a list of all snaps in my lscat that are for the given asset. + #-- Snaps are ordered chronologically from earliest to lastest. + raise NotImplementedError + + def snap(self, whence): + raise NotImplementedError + + def anchor(self, snap): + """ + I answer the snap that is the anchor layer for the given snap. + """ + raise NotImplementedError def toJDN(self, **kwargs): jdn = { - '_type': "Curator.Archive", - 'name': self.name, - 'RDN': self.RDN + '_type': "Bastion.Curator.Archive", + 'site': self.site, + 'zone': self.zone, + 'resource': self.resource, } return jdn -class Branch(Sable): - """ - I represent a branch (timeline of object evolution) relative to an archive. - """ - def __init__(self, RDN): - self.RDN = RDN - self.name = RDN - self._snaps = [ ] - - def head(self): - return self._snaps[-1] - - def base(self): - return self._snap[0] - - def created(self): - return self.base.deposited - - def updated(self): - return self.head.deposited - - def commit(self, snap): - self._snaps.append(snap) - self._snaps = sorted(self._snaps, key = lambda s: s.deposited) - - def __iter__(self): - return iter(self._snaps) - - @property +class Snap: def age(self, whence = None): - whence = whence if (whence is not None) else datetime.datetime.now() - return (whence - self.created) - - - -class BlobRef: - def __init__(self, RDN, archive, branch, deposited): - self.RDN = RDN - self.name = RDN - self.archive = archive.RDN if isinstance(archive, Archive) else str(archive) - self.branch = branch.RDN if isinstance(branch, Branch) else str(branch) - self.deposited = deposited - - - -class Snap(Sable): - """ - I represent a "snapshot" in time and contain the necessary - information to restore a dataset to the state observed when this - snap was deposited. - """ - def __init__(self, RDN, archive, branch, deposited, layers, **kwargs): - self.RDN = RDN - self.name = RDN - self.archive = archive.RDN if isinstance(archive, Archive) else str(archive) - self.branch = branch.RDN if isinstance(branch, Branch) else str(branch) - self.deposited = deposited - self.layers = layers - self.about = kwargs.get('about', "") - - @property - def age(self, whence = None): - whence = whence if (whence is not None) else datetime.datetime.now() - return (whence - self.deposited) - - def toJDN(self): - jdn = { - 'RDN': self.RDN, - 'archive': self.archive, - 'branch': self.branch, - 'deposited': self.deposited.isoformat(), - 'layers': self.layers[:] - } - return jdn - - + """ + I answer a datetime timedelta (elapsed time) between whence and the encoded datetime of this snap. + If no "whence" is explicitly given, I assume the current UTC time. + """ + whence = whence if whence is not None else datetime.datetime.utcnow() + return (whence - self.when.datetime()) + + @staticmethod + def parse(path): + path = pathlib.PurePath(path) + return Thing(**{ + 'slug': path.stem[0:8], + 'when': Quantim(path.stem[8:16]), + 'layer': path.stem[16], + 'anchor': path.stem[17:20], + }) + + @staticmethod + def dub(slug, when, layer, anchor): + return "{}{}{}{}".format(slug, str(Quantim(when)), layer, anchor) diff --git a/bin/HPSS.py b/lib/Bastion/HPSS.py similarity index 97% rename from bin/HPSS.py rename to lib/Bastion/HPSS.py index c383763..45e86af 100644 --- a/bin/HPSS.py +++ b/lib/Bastion/HPSS.py @@ -5,23 +5,10 @@ import datetime import json +from Bastion.Common import Thing, Unknown -class Thing(object): - pass -class UnknownType(object): - _ = None - - def __new__(cls): - if UnknownType._ is None: - UnknownType._ = object.__new__(cls) - return UnknownType._ - - def __repr__(self): - return "<|???|>" - -Unknown = UnknownType() #-------------------------------------------------------- #-- Set up an alias as a way of easily describing that | diff --git a/lib/Bastion/Site.py b/lib/Bastion/Site.py index f82bab8..4eb9cd1 100644 --- a/lib/Bastion/Site.py +++ b/lib/Bastion/Site.py @@ -2,87 +2,419 @@ Bastion.Site """ import logging +import pathlib +import random -import openpyxl - -from .Common import Sable, Slug40 -from .Curator import Asset +from .Common import * +from .Condo import CxNode +#from .Curator import Asset logger = logging.getLogger(__name__) -def loadSiteConfig(path = None): - if path is not None: - src = pathlib.Path(src) - else: - for p in ['~/.bastion/site.xlsx', '/etc/bastion/site.xlsx']: - p = pathlib.Path(p).expanduser() - if p.exists(): - src = p - break +#--------------------- +#-- Module defaults. | +#--------------------- +DEFAULT_ASSET_HISTORY = 60 * DAYS +DEFAULT_ANCHOR_DRIFT = 30 * DAYS +DEFAULT_ARCHIVE_VAULT = "fortress" +DEFAULT_LOGGING_PATH = pathlib.Path("/var/log/bastion") + + +def asLogLevel(x): + if isinstance(x, int): + return x + elif isinstance(x, str): + x = x.upper() + if x in ['DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']: + return getattr(logging, x) + raise ValueError("asLogLevel - cannot interpret '{}' as a log level.".format(x)) + + + +class canConfigure: + """ + Trait for objects that can configure themselves from a nested dictionary. + """ + @classmethod + def fromConf(cls, condex): + raise NotImplementedError(".fromConf is subclass responsibility") + + def configured(condex): + raise NotImplementedError(".configured is subclass responsibility") + + - if src is None: - raise Exception("Cannot find site.xlsx configuration") - else: - logger.info("loading site configuration from {}".format(str(src))) +class Site(canConfigure, canTextify): + SITES = { } - conf = SiteConfig() - conf.loadXLSX(src) + @staticmethod + def named(nm): + return Site.SITES[nm] - return conf + def __new__(cls, site): + if isinstance(site, cls): + return site + else: + if site in Site.SITES: + return Site.SITES[site] + else: + return object.__new__(cls) + def __init__(self, name): + if isinstance(name, str): + if name not in Site.SITES: + Site.SITES[name] = self + self.name = name + self.host = "127.0.0.1" -class SiteConfig: - def __init__(self): - self.site = { } #-- confvar -> confval - self._assets = { } #-- @asset -> Asset + self.logging = Thing() + self.logging.level = logging.WARN + self.logging.path = pathlib.Path("/var/log/bastion") + self.logging.persistence = None + + self.policy = RetentionPolicy() + + self._zones = { } + self._catalogs = { } + + def assets(self, zone): + k = RDN(zone) + if k not in self._catalogs: + self._catalogs[k] = AssetCatalog(self, k) + return self._catalogs[k] + + @property + def zones(self): + return [self._zones[z] for z in sorted(self._zones.keys())] + + def zone(self, z): + return self._zones[RDN(z)] + + @property + def RDN(self): + """ + I am the relatively distinguishing name for this object. + """ + return self.name + + def configured(self, conf): + if conf: + condex = conf['sites'][self.name] + if 'logging' in condex: + self.logging.level = condex.get(asLogLevel, 'logging.level', logging.WARN) + + if 'policy' in condex: + self.policy = RetentionPolicy(condex['policy']) + + if 'zones' in condex: + for zkey, zspec in condex['zones']: + zname = str(zkey) + logger.info("associationg (resource) zone {} to site {}".format(zname, self.name)) + logger.debug("self.zones is type {}".format(type(self.zones))) + self._zones[zname] = Zone(self, zname).configured( zspec ) + + aspecs = conf.get("assets.{}".format(self.name), None) + if aspecs is not None: + for zname in aspecs.keys: + logger.info("reading assets for resource zone {} in site {}".format(zname, self.name)) + zone = self.zone(zname) + for aspec in aspecs[zname]: + #-- Short form.... + if entity(aspec).isString: + #-- Short form.... + zone.assets.add( aspec ) + logger.debug("added asset {} to zone {} by short form description".format(aspec, zname)) + else: + #-- Long form ... + zone.assets.add( Asset(zone, aspec) ) + logger.debug("added asset {} to zone {} by long form description".format(aspec, zname)) + + return self + + def resources(self, zone): + zname = zone.name if isinstance(zone, ResourceZone) else zone + + if zname not in self._catalogs: + self._catalogs[zname] = ResourceCatalog(self, zname) + + return self._catalogs[zname] + + def asset(self, slug): + """ + Will search through all zones to locate the asset identified by the given slug. + Raises an error if there are two or more zones that containe the same slug (i.e. a hash collision). + """ + zoned = None + for zone in self.zones: + if slug in zone.assets: + if zoned is not None: + raise Exception("Site.asset - multiple zones ({}, {], etc.) claim asset {}".format(zone.name, zoned.name, slug)) + else: + zoned = zone + if zoned: + return zoned.assets[slug] + else: + return None + + +class RetentionPolicy(canConfigure, canTextify): + """ + How long to store an object, how many copies, and where the copies are deposited. + """ + def __init__(self, conf = None): + self._history = DEFAULT_ASSET_HISTORY + self._drift = DEFAULT_ANCHOR_DRIFT + self.vault_name = DEFAULT_ARCHIVE_VAULT + + if conf: + self.configured(conf) + + def _axs_drift(self, t = None): + if t is not None: + if entity(t).isDuration: + self._drift = t + elif entity(t).isNumeric: + self._drift = t * DAYS + else: + raise ValueError("RetentitonPolicy.drift: drift must be numeric or datetime.timedelta") + return self._drift + + def _axs_history(self, t = None): + if t is not None: + if entity(t).isDuration: + self._history = t + elif entity(t).isNumeric: + self._history = t * DAYS + else: + raise ValueError("RetentitonPolicy.history: history must be numeric or datetime.timedelta") + return self._history + + history = property(_axs_history, _axs_history) + drift = property(_axs_drift, _axs_drift) + + def configured(self, conf): + if conf: + self.history = conf.get('history', DEFAULT_ASSET_HISTORY) + self.drift = conf.get('drift', DEFAULT_ANCHOR_DRIFT) + self.vault = conf.get('vault', DEFAULT_ARCHIVE_VAULT) + return self + + def toJDN(self, **kwargs): + return {'history': self.history.days, 'drift': self.drift.days} + + @classmethod + def fromJDN(cls, jdn): + depth = jdn['history'] * DAYS + radius = jdn['drift'] * DAYS + return csl(history = depth, drift = radius) + + + +class Zone(canConfigure): + """ + I am a (resource) zone. + A zone is a logical entry point to a collection of assets. + A zone allows for asset collections to be mounted at different paths on different sites, while retaining the same internal hierarchical structure. + For a given site, no two zones should have the same name (i.e. the name is relatively distinguishing with respect to the site) + """ + def __init__(self, site, name, root = None): + self.site = Site(site) + self.name = name + self.root = root if (root is None) else pathlib.Path(root) + self.policy = self.site.policy #--inherit my site's default policy + + @property + def RDN(self): + """ + I am the relatively distinguishing name for this object. + """ + return self.name @property def assets(self): - return iter(self._assets.values()) + return self.site.assets(self.name) + + def configured(self, conf): + if conf: + if entity(conf).isString: + #-- short form + self.root = pathlib.Path(conf) + else: + #-- long form + self.root = pathlib.Path(conf['root']) + if 'policy' in conf: + self.policy = RetentionPolicy(conf['policy']) + return self - def asset(self, k): - return self._assets[k] + def __div__(self, name): + return Asset(self, name) - def loadXLSX(self, confpath): - wb = openpyxl.load_workbook(wb = str(confpath)) - #-- Read site conf... - self.gatherSiteEnv(wb) - #-- Read assets... - self.gatherAssets(wb) - def gatherSiteVars(self): - ws = wb['site'] - raise NotImplementedError - def gatherAssets(self): - pass +class Asset(canTextify): + def __init__(self, zone, *args, **kwargs): + self.zone = zone + self.name = None + self.about = None + self._RDN = None + spec = args[0] + if isinstance(spec, str): + self.name = spec + elif isinstance(spec, CxNode): + self.configured(spec) + elif isinstance(spec, dict): + self.configured(spec) + else: + raise ValueError("Asset.__init__ - cannot initialize from object of type {}".format(type(spec))) -class CurationPolicy(Sable): - def __init__(self, name, path, **kwargs): - self.name = name - self.path = pathlib.Path(path) + if 'RDN' in kwargs: + self._RDN = kwargs['RDN'] + if 'about' in kwargs: + self.about = kwargs['about'] + + if self._RDN: + if len(self._RDN) != 8: + raise ValueError("Asset RDN must be exactly 8 characters.") + + if self.name is None: + raise Exception("Asset.__init__ - name of asset is not defined") + + def __str__(self): + return "{} ({})".format(self.badge, self.RDN) + + @property + def RDN(self): + """ + I am the relatively distinguishing name for this object. + In the case of assets, the RDN defaults to the "badge" of the asset. + In some cases, the operator may want to manually label an asset with an RDN. + Thus, an RDN can be given explicity as kwarg in object construction. + If no RDN is explicitly given, the asset's badge is used. + """ + if self._RDN is None: + self._RDN = self.badge + return self._RDN - self.RDN = kwargs.get('RDN', Slug40(name)) - self.asset = self.RDN - self.LOCKS = kwargs.get('LOCKS', 2) #-- Lots Of Copies Keep us Safe, minimum # of branches to retain - self.longevity = kwargs.get('longevity', datetime.timedelta(days = 30)) #-- maximum time before a new branch is forced - self.about = kwargs.get('about', "") + def _get_policy(self): + return getattr(self, '_policy', self.zone.policy) + + def _set_policy(self, p): + if isinstance(p, RetentionPolicy): + self._policy = p + else: + raise ValueError("Asset.policy - must be set to an instance of PolicyRetention") + + policy = property(_get_policy, _set_policy) + + @property + def path(self): + """ + I answer the local file system path to this asset. + """ + return self.zone.root / pathlib.Path(self.name) + + @property + def CURIE(self): + """ + My CURIE (Compact URI) form is [{site}:{zone}/{asset}] + """ + return CURIE(self.zone.site.name, str(pathlib.PurePath(self.zone.name) / self.name)) + + @property + def badge(self): + """ + I am a compact (40-bit) hash constructed from my CURIE (compact URI) + """ + return Slug40(str(self.CURIE)) def toJDN(self, **kwargs): jdn = { - '_type': "Bastion.Site.CurationPolicy", - 'RDN': self.RDN, - 'name': self.name, - 'LOCKS': self.LOCKS, - 'longevity': self.longevity.total_seconds() - 'path': str(self.path), - 'about': self.about + 'zone': "{}:{}".format(self.zone.site.name, self.zone.name), + 'path': str(self.path), } + if self.about: + jdn['about'] = self.about + jdn['RDN'] = self.RDN - @classmethod - def fromJDN(cls, jdn, **kwargs): - policy = cls(jdn['name'], jdn['path'], **jdn) + return jdn + + def configured(self, aconf): + #-- The name key is mandatory. + self.name = aconf['name'] + + #-- The rest of these keys are optional. + if 'policy' in aconf: + self.policy = RetentionPolicy(aconf['policy']) + if 'about' in aconf: + self.about = aconf['about'] + if 'RDN' in aconf: + self._RDN = aconf['RDN'] + + return self + + @property + def created(self): + return datetime.datetime.fromtimestamp( self.path.stat().st_stime ) + + @property + def modified(self): + return datetime.datetime.fromtimestamp( self.path.stat().st_mtime ) + + + +class AssetCatalog: + def __init__(self, context, *args): + if isinstance(context, Zone): + self.zone = context + self.site = context.site + elif isinstance(context, Site): + self.site = context + self.zone = self.site.zone(args[0]) + elif isinstance(context, str): + self.site = Site(context) + self.zone = self.site.zone(args[0]) + else: + raise Exception("AssetCatalog.__init__ - I don't know how to construct the requested AssetCatalog instance") + + self.xRDNs = { } + self.xnames = { } + + @property + def any(self): + k = random.choice( list(self.xRDNs.keys()) ) + return self.xRDNs[k] + + def add(self, obj): + if isinstance(obj, str): + asset = Asset(self.zone, obj) + elif isinstance(obj, pathlib.PurePath): + asset = Asset(self.zone, str(obj)) + elif isinstance(obj, Asset): + asset = obj + else: + raise ValueError("AssetCatalog.add - I don't know how to add an object of type {}".format(type(obj))) + + if asset.RDN not in self.xRDNs: + self.xRDNs[asset.RDN] = asset + self.xnames[asset.name] = asset + else: + raise Exception("duplicate asset RDN added!") + + def __contains__(self, slug): + return (slug in self.xRDNs) + + def update(self, asset): + self.xRDNs[asset.RDN] = asset + self.xnames[asset.name] = asset + + def named(self, name): + return self.xnames[name] + + def __getitem__(self, slug): + return self.xRDNs[slug] + def __iter__(self): + return iter(sorted(self.xRDNs.values(), key = lambda x: str(x.path))) diff --git a/lib/Bastion/Vault.py b/lib/Bastion/Vault.py deleted file mode 100644 index 4a99469..0000000 --- a/lib/Bastion/Vault.py +++ /dev/null @@ -1,33 +0,0 @@ -""" -Bastion.Vault -""" -class isClerk: - """ - I am an abstract type for "clerk" objects that do data management - in the context of a vault. - """ - @property - def snaps(self): - raise NotImplementedError - - @property - def branches(self): - raise NotImplementedError - - - -class isVault: - """ - I am an abstract base type for specialized Vault classes. - """ - def __getitem__(self, asset): - raise NotImplementedError - - @property - def assets(self): - raise NotImplementedError - - def put(self, asset, latest = None): - pass - - def diff --git a/lib/Bastion/Vaults/Common.py b/lib/Bastion/Vaults/Common.py new file mode 100644 index 0000000..e451cbc --- /dev/null +++ b/lib/Bastion/Vaults/Common.py @@ -0,0 +1,76 @@ +""" +Bastion.Vault.Common +""" +from Curator import Archive, Branch, Snap + +class isClerk: + """ + I am an abstract type for "clerk" objects that do data management in the context of a vault. + """ + @property + def sites(self): + raise NotImplementedError + + @property + def zones(self): + raise NotImplementedError + + @property + def snaps(self): + raise NotImplementedError + + @property + def branches(self): + raise NotImplementedError + + @property + def archives(self): + raise NotImplementedError + + +class isSiteClerk: + def __init__(self, site): + self.site = site + + def zones(self): + raise NotImplementedError + + +class isZoneClerk: + def __init__(self, site, zone): + self.site = site + self.zone = zone + + +class isArchiveClerk: + def __init__(self, site, zone, archive): + self.site = site + self.zone = zone + self.resource = archive + + +class isBranchClerk: + def __init__(self, site, zone, archive, branch): + self.site = site + self.zone = zone + self.resource = archive + self.branch = branch + + @property + def snaps(self): + raise NotImplementedError + +class isVault: + """ + I am an abstract base type for specialized Vault classes. + """ + def __getitem__(self, asset): + raise NotImplementedError + + @property + def assets(self): + raise NotImplementedError + + def put(self, asset, latest = None): + pass + diff --git a/lib/Bastion/Vaults/Local.py b/lib/Bastion/Vaults/Local.py new file mode 100644 index 0000000..ac8850e --- /dev/null +++ b/lib/Bastion/Vaults/Local.py @@ -0,0 +1,8 @@ +""" +Bastion.Vault.Local + +I am a vault for storing backups to a local file system. +""" + +from .Common import * + diff --git a/lib/Bastion/Vaults/__init__.py b/lib/Bastion/Vaults/__init__.py new file mode 100644 index 0000000..e69de29