From 36363db715d90e45d8c635a57fd34d2044379991 Mon Sep 17 00:00:00 2001 From: Nathan Denny Date: Thu, 4 Apr 2024 06:12:28 -0400 Subject: [PATCH] Major refactoring of the underlying model and thus code. New model is based on "snaps" in time that are archived. --- bin/bastion.py | 57 ++++ bin/fossil.py | 17 +- bin/out.txt | 23 -- docs/idifhub_conf.xlsx | Bin 16972 -> 22180 bytes etc/conf-idifhub.yaml | 69 ++++ lab/bootstrap.py | 32 ++ lib/Bastion/Common.py | 263 ++++++++++++++- lib/Bastion/Condo.py | 590 +++++++++++++++++++++++++++++++++ lib/Bastion/Curator.py | 177 ++++------ {bin => lib/Bastion}/HPSS.py | 15 +- lib/Bastion/Site.py | 446 +++++++++++++++++++++---- lib/Bastion/Vault.py | 33 -- lib/Bastion/Vaults/Common.py | 76 +++++ lib/Bastion/Vaults/Local.py | 8 + lib/Bastion/Vaults/__init__.py | 0 15 files changed, 1551 insertions(+), 255 deletions(-) create mode 100755 bin/bastion.py delete mode 100644 bin/out.txt create mode 100644 etc/conf-idifhub.yaml create mode 100644 lab/bootstrap.py create mode 100644 lib/Bastion/Condo.py rename {bin => lib/Bastion}/HPSS.py (97%) delete mode 100644 lib/Bastion/Vault.py create mode 100644 lib/Bastion/Vaults/Common.py create mode 100644 lib/Bastion/Vaults/Local.py create mode 100644 lib/Bastion/Vaults/__init__.py diff --git a/bin/bastion.py b/bin/bastion.py new file mode 100755 index 0000000..7de70e4 --- /dev/null +++ b/bin/bastion.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 + +import sys +import os +import pathlib +import logging + +logger = logging.getLogger() +logging.basicConfig(level = logging.DEBUG) + + +BIN_PATH = pathlib.Path(sys.argv[0]).absolute().parent +APP_PATH = BIN_PATH.parent +LIB_PATH = APP_PATH / 'lib' +LBX_PATH = APP_PATH / 'lib-exec' + +sys.path.insert(0, str(LIB_PATH)) + + +from Bastion.Common import * +from Bastion.Site import Site +from Bastion.Condo import * + + + +CONF_SEARCH_ORDER = [ + pathlib.Path('/etc/bastion'), + APP_PATH / 'etc', + pathlib.Path('~/.bastion').expanduser() +] + +if __name__ == '__main__': + comargs = dict(enumerate(sys.argv[1:])) + subject = comargs.get(0, 'help') + request = comargs.get(1, 'help') + + conf = Condex() + for folder in CONF_SEARCH_ORDER: + for confile in folder.rglob("conf-*.yaml"): + print(confile) + conf.load(folder / confile) + + if subject == 'keytab': + if request == 'refresh': + raise NotImplementedError + else: + Keytab(conf).help() + + if subject == 'backups': + sname = comargs[2] + site = Site(sname).configured(conf) + + +#bastion backups tidy idifhub +#bastion backups update idifhub +#bastion backups catalog idifhub +#bastion keytab refresh fortress diff --git a/bin/fossil.py b/bin/fossil.py index c2740ea..aa6921e 100644 --- a/bin/fossil.py +++ b/bin/fossil.py @@ -21,6 +21,8 @@ #----------------------------------------------------------------------------------------------------------------------- museum = Fossils(login = ndenny, conf = "idif.xlsx") #-- the default login can also be part of the conf workbook. +museum.vault("idifhub.ecn.purdue.edu", "LiDAR") + #-------------------------------------------------------------------------------------------------------------- #-- This is the "do everything" magic method that uses a lot of lower level methods to accomplish it's goal. | #-------------------------------------------------------------------------------------------------------------- @@ -36,7 +38,7 @@ #-- slug is the base32 encoding of the shake128/5 hash of the dataset's name. | #-- backups are also annotated with provenance in more readable English text. | #-------------------------------------------------------------------------------------------------------------- -report = museum.backup("QL2_3DEP_LiDAR_IN_2011_2013_l2") +report = museum.fossilize("QL2_3DEP_LiDAR_IN_2011_2013_l2") #-- This forces a differential backup of the given dataset RIGHT NOW. #-- Without an optional override, default behavior is to use the most recent full backup as the basis for ths @@ -62,3 +64,16 @@ def backup_full(folder, **kwargs): def backup_differential(folder): return backup(folder, 1, **kwargs) + +SITE = sys.argv[1] + +site = Site(SITE) +manifest = Manifest(SITE) +musuem = HPSS.Museum(site) + +for zone in manifest.zones: + for asset in manifest[zone].assets: + curator = museum.curator(zone, asset) + if curator.branches.most_recent.deposited > asset.longevity: + curator.curate(asset) + diff --git a/bin/out.txt b/bin/out.txt deleted file mode 100644 index 395339f..0000000 --- a/bin/out.txt +++ /dev/null @@ -1,23 +0,0 @@ -/home/ndenny: -drwxr-xr-x 2 ndenny student 253552 512 Wed Jul 19 15:28:56 2023 CfGS --rw------- 1 ndenny student 10 253552 DISK 12800 Thu Dec 7 06:33:23 2023 miniconda_0231207.tar --rw------- 1 ndenny student 10 253552 DISK 1312 Thu Dec 7 06:33:24 2023 miniconda_0231207.tar.idx --rw------- 1 ndenny itap 11 253552 TAPE 2307115900416 Tue Feb 9 10:19:17 2021 nexus_20210209.tar --rw------- 1 ndenny itap 11 253552 TAPE 414114080 Tue Feb 9 10:19:22 2021 nexus_20210209.tar.idx --rw------- 1 ndenny itap 11 253552 TAPE 2837237811712 Fri Nov 19 21:03:24 2021 nexus_20211119.tar --rw------- 1 ndenny itap 11 253552 TAPE 417398048 Fri Nov 19 21:03:30 2021 nexus_20211119.tar.idx --rw------- 1 ndenny itap 11 253552 TAPE 2016443183104 Thu May 19 16:15:49 2022 nexus_catalog_2022-05-19.tar --rw------- 1 ndenny itap 11 253552 TAPE 196347680 Thu May 19 16:15:51 2022 nexus_catalog_2022-05-19.tar.idx --rw------- 1 ndenny itap 11 253552 TAPE 18800947200 Mon Feb 14 15:24:16 2022 nexus_final_report_2022-02-14.tar --rw------- 1 ndenny itap 10 253552 DISK 1675040 Mon Feb 14 15:24:16 2022 nexus_final_report_2022-02-14.tar.idx --rw------- 1 ndenny itap 11 253552 TAPE 1634327330816 Mon Feb 14 17:55:47 2022 nexus_projects_2022-02-14.tar --rw------- 1 ndenny itap 11 253552 TAPE 1659624728576 Thu May 19 12:52:50 2022 nexus_projects_2022-05-19.tar --rw------- 1 ndenny itap 11 253552 TAPE 492911904 Thu May 19 12:52:57 2022 nexus_projects_2022-05-19.tar.idx --rw------- 1 ndenny student 11 253552 DISK 1569479696896 Thu Dec 7 09:18:17 2023 QL2_3DEP_LiDAR_IN_2011_2013_l2.tar --rw------- 1 ndenny student 10 253552 DISK 71481632 Thu Dec 7 10:03:45 2023 QL2_3DEP_LiDAR_IN_2011_2013_l2.tar.idx --rw------- 1 ndenny student 11 253552 DISK 2323521421824 Mon Dec 11 03:08:35 2023 QL2_3DEP_LiDAR_IN_2017_2019_l2.tar --rw------- 1 ndenny student 11 253552 DISK 124447008 Mon Dec 11 04:05:42 2023 QL2_3DEP_LiDAR_IN_2017_2019_l2.tar.idx --rw------- 1 ndenny student 11 253552 DISK 1575000025088 Mon Dec 11 05:53:04 2023 QL2_3DEP_LiDAR_IN_2017_2019_laz.tar --rw------- 1 ndenny student 10 253552 DISK 22099744 Mon Dec 11 06:32:41 2023 QL2_3DEP_LiDAR_IN_2017_2019_laz.tar.idx --rw------- 1 ndenny student 11 253552 DISK 8517799739904 Mon Dec 11 11:38:13 2023 QLX_3DEP_LiDAR_US_South.tar -drwxr-x--- 2 ndenny itap 253552 512 Thu Jul 16 05:51:12 2020 QUDT diff --git a/docs/idifhub_conf.xlsx b/docs/idifhub_conf.xlsx index f7c2748e1ac572ffda262bf197d7ec4cdb9a2f85..2129d8448b610627b3c6766ac60a320c44e472b6 100644 GIT binary patch literal 22180 zcmcJX1yogC*Y62wX^>DFq$C9CMoKybrMo$Fhk|rBN=qtThX&~u1j$27!=XFx!T0-K z=19pmmX{Ck+|?0NQDd+oL6Z_T~8ycFym94JIYL?}E7BQ>a>Hgw2mXA34LfUUV6 z0ASAOY-tgy1PV)J#_-ruC2geEur3Z9c#+AkSQ+*;V0 zAk%Ex%Db?tKa`sg3L{DXrer0E6Gv7uOz;3PLt%A>#+G%xA;lK80)|MTly^Z!wv0+5 zETfn_c7c^u2yHJy9Cg}~u`+Jksx-sP0R2H?7)%9S#<+sb)15$mn)f8^u6J=r>}4^e z=4}re(GFd;UyI1vo(W-a##*-@-J3e9@lzZ+30H|BXB%Kl1wCCN3JDlLs{Ce>B;BsS z^wQ5q3%^jj-pOY!xAFym{p*22tTcy!2{qALNJ{1+;L8B|A!6I1u$xWT_-WE>4PbIk z@gX?(WQw1`6GmPN0ddYf(G(E|3MwBK3QGRQPK5fOPGo2AVqs`^-G#s!>O~Wr7@jlQ zn2Yr?q!^vZD?qN|34cu$kdkSZ8Ohzp7;J%&*e_R#iio1=iMtOzLE9<&_B!*XHui=j zXI33q;lfXFNb9GHc+~HVd}Vte6R<2^)^K^^vY(x2ZsSdk4oBJS;Ss8TF*RGMm6Q*A zZW?CuzJ&OFRZ)Mm!A`AiYcv;{!g<^y>((bq!cSw0mAF~w;o3Tbs}=C^WroT_F-{`+ zS44ZPaUPbill9Q7uJ>tV2DWGuw-{C0%h)etg$tcvizmDnw{Zn5M47qS`cm z!au<=<=riBELmo1iEcl&WlR?Wjq169cE$&p`L-2j5rnjmxoa@)x}wBTCWuiuzPD@; zD8kE+63IskvKn-3G~vrInKA($aRfY@b$ig(8;kCyp{f=qkDtXaw|}_*WTS?Ja@wyi z@?;@tTpd%2X%5_F@4qQO-)Bey{VilVh%AIGC~JBUHu&@7bmGM1vSa&w=}Z^?caey2 zsdye9LEiZ5)hmJS;Vg4$c(bO>lk#5d%N=hXaS}#RQW|Kp+(#4HcTYlK(8MgpNhu@X zvs1-mnQ{0s{A%*eR?Q~4pc$_H7{NqKc`;$#hr3i3k-({jh+=4STB1i5WQ3{VD60-t zd6+42Zg0Qp8;{lMjy2w)sm5~R%GDg!+w+wn!hUckwW`T$X#R81%vydns8U72wR!XM zxUTGk&UGRwt&LnQ9C3?MGYhH7zx`o5n*-v(;H7MbW%#pOHuA=S)YT;*Z;V`7UWo~@rxsVn1mN7+1;Pg~l%(Kww z34<;EOTmZQCGoG5&WKqVDxN$aB2+SJQu?Nr!p^xa65D2&UHZwDPv;%-U8tp%NkttT zd^oY!3OZrXL#0-x*%oY8$TI8M15W@*p>~GMVlF3)_x7?J-1A&ExINjX{IuZh$qz?? zxkUpbbSCC;rskcdJO}(MI-eirqh<%OMwEr5fwJ>a<>3vMFMStNzV*UOzT9i&tX=mW zQOG2cC4AxnliBsKAHkp?CN&PZqGeAh(ow{7^Xa?p{^a}*>fRnbFQp7u<_u*ZW^ag@ z&Yb6klA&cH5ZFBs+zS^+4&JU7gp{ekN+{Vi`KG#(9Oy7yqzV)Tlu2e*$cC1)@-^1m ze5SJ*c|Kn4Q?Z$q;z~cFwV~Ox-YJ(en_n+3qg0fs)BP$cn*V84_6(m`IuSHYwQ$})V{>=C`&tgxzV`fAWHcg&)NQg*YTmIP+-;0 zAUc)+mT^1qwNx{PA^1Kx=rtUniFWUm9Yw94fmCQ*%)CkIXG?C}#1Y2CpdzcbLW;7p z0NU`g$<_}E0h+$ZEF5l##T1S9^c&TQO5FEh6Oo45+Me@*2s_)KoCs9=t&<91z;ycc zEWTspBp%xECM=9Tf*-dRy3ZJI9t8F?@LXU=JL?o*U!aC#s(YOnf6kiwS&laMyPhpkOazBH0sG4z|D=Ns8D zs34;BXj5nDF3U&3+A4_mW15QFZAekWMO{KB*gu8!T`i0vbYu0{KB&$f6SoAx4{lq$ zVZ(q4gC0~{4vI^PDhEzT59r>1BLaPl{u42Pr7Xlr|&_GE~w z0;W?vd8J1;&Z400MFHL{^D{Z0+*LxI-<+={44vn z<9n|9>@@1!oDP?sG&t||bPVmSgQ%-{FRX1n+|N|E`uR1d_&_r=mqjUC;KeRLN4V>e zXM=0-4p^>%cP4t-xgp#bz{pg{IIZb*v^4*1xw2jEL5O+j{juc`gCr;?_d=K(8|78JZ;%g@$+&&UvKkg z>fb)xpXy>FDI7hrcCQcasqOIMYgi;9NhumVXxZUn;x8z|O!ho3_MG-dpUwx(AH&QD zcpcBC;^C3_b5N6nsjheJb!cCB*zK(!k!Ti~d+de>Yg-le?Bs-p3uxvYueY_$jQEc% zd(?TVbCBrQe@>Mh@#b#~_D7cu3%_)AJ+o@2W@_-9J#-?`xR^;SGP;){g}E-4CXYinMv%cK6%j*X_?=?hYNiCz(0i`Qk;~7*gBl z)$kRsGQOJs;$XkU#$(T_!E@tyqcU~#3kTEX5PFA{tHWO7dfSEd?&@0J)>a>r>L?zG zKw}<=Y2{)ZRGG7UTFOy$va_(&apZh{RkoLS)mU@caSB%ue|l(|B5<|2*mJOUZmzw% zx*YFL$8@qb-gtSC;^A3uyz4pxDL(0w;;*T%s^m@9NIlFQcP(t+e-#8#`v|x%p4m<-)z3_t#Gu1(=#a zZ9_%(Dn;-i=70+U0KZhD@_akw$< zU#nSnY_@ZL{b5q=w9nYZ(QS_SYG?Fd>sX|+=yW&l=xlzPx<<3^)NJRXUe3POIaF@{ zT;Lqc+fliaN^_nfbi#mbwDhkW78#8mx8RG`e>I7H_WH z-@1@U)jlvMO_^L5IIlbL3&=U*A71WbbgG7Wn)w=i_N4B7Eswhqb@fv9U>CjN zqpDZ8>Vag#N4IyrMaL>ra%;mR9!f=5F@QC;#t&x;^+w7^eQp;BGha_lFW(fcZ5lS7 z6mG6Nahy&xQXe%I)M7RkQp@psEnRI;yESjE^LwXqXnX0(ZDrwlxul4vl00iTADN;c zVCQ0|V#j4iWxsF1Z2=s+I~FpwQT(=KT;J}~L&V3(MXe494#YU-*aSoY zY8HdNEpIYWff&afyMR(a%~EibWxPeL0qBpW!ZF7-AU#mC1RP|kYEi2P!Ug@=)CV*N zYLDtR|<}_47YGI08xO5#}7M!e>RnX z11;q&-1I=$pp>!0Rv*kPFE?W7lI4CE^VzqRbQa5Dn2f>Op0|7gMk))#|) zEtxFbbU`Q}t+B&q;7ukz&~zF2R>EHlh6UYDNOdNKKGpcVazTJTLxX`QzVnC|F~J$5 zc!yMHLFiMr&#Six&^+icevkq_!~|E2;sa8h4I!PJ7Ym$HG|Ml9xdyPh1LNew%HP;# zQ^h=GNKZ0DT2zA+y%gpe!|LvW6anAZ0;ytJAVpS4Z@n-|u1LwDghZgd1Mb|xL`5Vg z#}H8_4d@ia`0|>R{Emk(jec9P-66$8he#!M(Yz2N@xs3d88Obh(5)n9UWk_XwxT>QL`D3+ zVrV?u<|E0Q+xka&$(!5eb1N}v^%3L!Ng8%$K_G7{fQiShz@+gxpqgoAuZRESG&}W- z18X|8p*r?>^@7Kzfp=)CL&o!}zSf@wGAT(yCM811q~rjZlnAQ`t0!#O>?)SmKB-{X z-V-U6GKlYHjMpTqm2|AwU@tW{tc=xUs#S1|-Y^4KPbI#z|mzzT`TJ7w)y^5Elo@^6hcT(^=B&JjTfx0Scf5rGJ|lVF5u=ZIU0ud{pr!fhqX zSw0Bib`ppX>nwjOsc@DLM!2nnIu8UO+)9cu(b?yl#CXR75bp2jP!LrZz~(EKFQ9-*yxV)O|raWfe1W1I@H8B8DH=wwE5gh)LMN+d2cIY!3b_U zI!weHZ9etR#dzq4^G#a3HyIIjW=V-3wEFlsTVtT3%{K}0at0vWVAQD}MQI_$z#OTq zH=uo(f8^GssI0bi9FTS;uhmFmWEuCpzS6yK&w;tfF z&K_ zfNmxA(_nkAdw25P>PG{44(q2^TZvG#x9sM_LW`WZY&yC}-IA*GmxVqJ_`J$0I^wYD z7#XcitSTFNWSc}4v-c?dfeF&0h!l-D1jvT4y3^z2Wh>s;=2OMkF{D4RKw30_6sZey zfv~#gAVrmLdIbW;u3V|Hop5yO!vlE;i>KRKiE!1nbmzm+@|~D%#F0kM;;S;?)~BQ< z6?_h+sWw}E)OZ8hZY?6b5*zM`w+f^GzeLc1|BawQfvW!lYTs$?50Zv~j=Xa_k=#kc zLBFlg?WAF$|F4v?ZvRYcKe+AtCkEoL^Fp_hC-XuS#6QW5^ETL0s2|{2e0h$(h2nZS zRfc!9d*6q@ag*c9G!O4+>+UJ9>(0T-*;ysLhf8*Cf8zP2n-jVr*Yc|`8Z)Qto{ll;GNEaRj7{qrn#%eOw3OmMbn2lQ+jCNbm*J?RtY&;zAD^)je znq|COg+ejjRw`u}-&Gi)Nn88UF?8eQxI^i>AxEcG)Re4a@W$b!D72FT4G%Oa7Bo2- zDHbfb5GfWMxh^Rd0{LrFEF|)DQY;kmCQ>X^@;Op0biEzYd$*KZaR2{&Me&yK|NVO` zw_gGHLr?n8g1|otkZ;+$zJE{h_Fe2p@A}&(vz@ei=(iQZoit4JTgl(J>=%M7pBJJh zzOC5L3sDo_O8&-WzYyF?n~xUnZ3Vy8M~L@!qQHya=5s4aX!Uu=%Oj3zz*$aIR%|enO<=`IQhnlU4&j@GQ4@dY4MdMz?-dDvrFx z6%_xd&5so0fw!Q&8UTW4$HmjWLX2Uw7!ngwB>dXAX7#LXDXVA zLs%^jXF?ZJMDOdUL0i5DDN;iQw<^*Y+=)ymnZ|f1rJ!F=B^@{4?y3XS3^`WIF5f0=QM%FkN_tAVLpo(m*)2xFeXsfg`p^;y68YE>EW9x?NayM1ASAX7A+h<7 zOJ*%9J;QN%DS453fq8H9eDh$R>%LOKx52fGwT-n=u~xBFu^YA?wjH(^wu8)_Hjue< z&MaI7M+ete zHZAOLX|x~E_SZDpf1&NR*)+)aZktVm1!)1CY6MVVM(U?GT8YlI01oqEzC|3|HXZGw zAPA=|uFJ?ymh6%0tO$J?9K8>*tgnM*CdWc8QZmgCx@4Q^im~vSp@7i}F%5 zIS_6e!RoGt6hRVcrBpE)kRnT@MQccry6_T-$c9&aY#Gq#FUN=GQ1M4J`isEcWLuc=9RdW zNb`2K`rKADcstvCAOdd9`%C8I|4+Un=+}J5IDzxe@b+7H{r?p6_gPxbeIpt)J=aj6r5@>Zx2KFU}O-ehA=f5iCc1K#br96*IHBca{i!w1^JVW@xWYTZPd&9Ln{BJ663*CkLT4{VuRbtQ9mZc zTTuMz&Ymih*1U|DCke;a>1yMyW>0YAF{}3Z+0}NiK#i-Ht1D6J_gj8)ddl|J4~GuM z+fIgkjv#ld^?&wiINePW&`zl@2$!P^znjx|Ra;ZVd$!zma?s_!vVVTMzk8(WzP-M> zcrbqda`sqm_WH)LzOKc7;|{z_>(k9+i||GsLR*)!bgN5ukDb*exemZ`5#DG=xcgD% zL1>Crp_kjvLRTKO9I=4M;l)r+cq)n4@>E+}#nsB5Iq}RfQ!24L_ldnLSJK7azH?ys z6=OKS=3?mEH$yikM?S4yV?fc&%=yYvkyHHX@zDY8L{FkB#3ktD_;SCF&$Ij6K_f4Z zXJ_<*(0Rk)D*cuFZ>EJj_K%G
    ssspl6;JlESY-tfQW?H?(elT^n^RO3vjDW(pUz0%I0Drj2J$r~>+S z#~4)AX-^1vc~o#m#4~GdcC*+a$=my12FaO$;ppMx%H?WjN7c)2P3P= zN%%@Ol#-Rd3TYg4%PFMsemkkW5Vl+ssU)Z|A>-G-&%o7briPW1T0@ZxF@@WQ1(u z9OH4dQeJh(&K6`bo%>W7FbM8k#Kk*@vgb>6%}z0ND|E|eZ`F>MBt=j1<2q4pVHudf zI&ooBS#t}kyZ6&;O=JzQF4I1vqS1?4o=*bC28{_gU(ys87z+o1aF#TvFsdfbQf<#L za9y=ps7s?uw^ibcf&eW{j zacvRNyQr=(Vo>*UD~Oq;J8vccF@nb_-}&=zH_K9OcBxY0EQ{LhzvcPTKo5RKvCB4* zktE)uXCGn=_`LKWvgn@1z{^XV!KLDlzAkj8HF*TI$||9J>Pd#V53*vD;M^D3cI7m> z``_^)xxE>XAryEg{aSw6gO*QpHB{bj&fxSiq(N_=M~r!krpW^4{Al?uj-Kai zt8$WV$Lhf#%&ctP$yu~m(37=iucw6-)pJ&-mYnzvtSizMbiQP79itU30&42%U$KqD z`#wo$axXA&XvaM1f?E{r?{XS_FkMtP-aH=6qSCvl)_czR@cHA0d*5n19;@mY(lHaG?J(wB_g7IQ;1IXNe&NPU!kMJ(6?GCJrO0RNzjQ&fF z&^LDt<&=A+fwju=WV}~3cy$#O(-zG+H9KwcU{jG9dhW<|XWkTq!C-4h z+V#~WVD*DvLUIfrHn2HwfXgT?g#z)ja4nQs)0y zpFphe{HI~?!Y!j@SUv;$!>_P(ri(vy(ck0jC(6r8Htk?btMXl%a6zR}ke@YMR>4^5 zUpImE3!0CbQnod+w8X?|IrPkSo#|Kd?_^kuCBqSjj%wl+qk5`*KYjz#kdqBB$%y+L zbbc-z_36k0QTLmaEJx(D z%5yk+5x@?snIFB`65UdctzrQ@uVB4nnIs5{)+*Pazcc;3M062VfXXRJ&r^cyLVE%? zJw&+Id5%k(gt2>L^j?|Ov85r7ncWi3qKh>04&cqpkf0t4g(A8~HG|;cjmQ{fwJM4! zOXU|*(D`=(>>4{!1&<5dKB6^rl^V=A?U)Ib=D&^Hga49pq90xIy4E;3q|x!p=fwrW z^}s%^!#5HQxz)Xc;NNa_C;ZQ??%KA77Iwcp1;==8sgLS6Yhk|YzvY3PaW+qRUq6Ji zK1QfW{RpaEVSb>MJ?bzAAP^qP;H&6J!$Z(@cyuOcHnO#ssTJuQ(j=%7lg3;%7OKm< zUFTL+0Wm0C=Kop(-M49yg^Sz)wn1Jr1JLpHM?X+&S0DMY<~=qRWezgUn> z1v^2JJ>dv%0#ZqH^nc@}+tENmWcA+d?c>fs#6H`E2-Yuyuk>c~7+DbZSC3Dlb^wuw z8<>luvpKPg{NZa!y$1q4UzZK67`gEQNVU}Ib%s4Bk3cgld154BtuyHB7P4L}UY&r{ zzZCL-*Ub-i;ryqC>`ZiR4Gom+ZB4C=zdz$u@LUs+$c*DLQzYmao*e!`ojxr&P9_gW z#j;e=?J2xf)BUj&ONk-vMyWy5RVjyON3VRrcAJlvwLq=$;g;A3vvn-ox~%+$m@CaO zlB!6xSC(4b0rVtow)pk?LgI~uxN(j?+a6p*IZ9b zWnh9Q%1J%xG^I+WtVA;;S0#D@m5HbT>8vM6g;E3WjL!L4#%UiVydvt?BnvLl0%v(P zg{c#)Vfqm@h*_^IGrPBtT!l_2B|qJ}XqzA`--~UBL#6*?^qf3}RCsRi0XC8tFpv59YwB!a)dBkX*ixo}N@j$;7Om$wC2+cD__Yz?o2B@e|W zS;!StzlW=`X*p5wF3pSwX+Tz5cmg9pU80V0AX6c#05lU_ZqA`2+}(8O=5lBk)sOxj z?Jx}$VDLJMMhseFf|)T7ram8UjsZD@Dk`!jQa+6W{|OVo_9M%LiStM(j3rmXC!}Z5 z3My#@6?83Q0}P0I&fRdYc_JdPv4xk!C@iS8vp?)SN{;Xoi zf%+4TAK@Jdb89TB4$^dW{pJfbHDc*mGl{3*8_s|qy zu4U|nWPO&sqpm5&3b*-~-PKWdxjA>Xr&-zA@f|ZaDgAl99!pdY7sRXcj@|p$rwJEJ z2-yU3nn3@0ePjH5nslwLug_9)1mv+@=DU7dXSgb@70*mhuq1uhWq8Oe{n5bB4Fgf$ zzC)*?rMvQkelp9OYSB2Nf0mV~a5RqTEdvQhb_sBV-jz$DS zWibZ)_`aY6h>mkf4FYw9L(v}Bu#pKp5ZrqeZ4ov2qR2%$YX%$SAo(c|*9RW^*$W&L z82vjGLQ9+QBC7MJ_*W>g&y%)LYL^zU!oR?mue;n4>*BMtSn$YrQhbsx*<6s>|M@_2 zJAqeM{Qx|7QTpbQjc;ab7ZC4SiX%1q>@!LA0 z57|;7>58KArKEDiqRG#Qu)wQF0#b7Ljj;fu*g`IjXf*usf`-4xypLFmGS;7KjZ?cc8ZJ%SjWv(Y+KIlIhQl(j?rVcb`?V(d&eV z`kU+Rz?4C*6&w!JGslbVMKQoUyXBlIC4-()MI+PU686wwL_w?%`db#E6JEA++Vx#s zsIgz4Yp$GDwnGlT2Bq!}gK!+?jLK6j|A7uu}SleSI} z=pWHmM?8pVAMD6`FDUFS;+vg-1oPOajfQE3$tX-h1_ZAaw6${kQ_@Bq8ws)=^L(9pyg(#(#o}uY7VX%oy;q^^N4a(^b#X zNn2?`IIJ6e6!4}a7*z} zx7|zBY>rrcRK(wQiN*F>Dz`_0{eA69OA2|Aj%b_+?yhvDQTu5qi2EkfS=PW+0FmF9}$*Vh8pINcq^;3M3}1Ge2AuOmOv$lF8}vp=kP zr%36PB7e0fa`p~?)zx)SD&bJ(*-#7P7NY&Vp?t=IF8z0jQs!~m?_M;8dq%8%e=0H6 znmG&&(u~e`{;e6WuebkPW@?reOs4jRmfs&p?0abm0I_3SHfWQXr0PEdT6Bm_kq6!< z6^y5cPmwCyYo8y5ROid_Nm8toaF`M@X_nmHYS|*C_E@ZVHgy1;M+p^YMyhbn44q$j`llUr82I3*$neAggF{ z>aHm5>K->0jFyl%!7~!=aVNPO1XCm9ong&#LGW~BPz+ksfR8-v`D(k35DlUk3*`Zx zk=_gi)m=WN6J$aDB@7t(5U)1QI1Kv3*U+JLM-r=2$BnL6^7-+~Ei=MPAF?MTg;N3? zQuR}5qns}a2w5BmM$m-TF|_h*MRs27sH=xPsq-?mik5fiENn3;dy=4(?feFWq(6y= zIk+ye_Vv^+b4htx=tT^c@$AWFbHMm=^dgQ;LvS@fj-0@cA>7nw~Pg?3dX|@EOlvNeUWGL2+RWLC3&OU%@ep z)0-nj>N~ZpylI6mpGA~>Ip-w2DJs!0P_P+n@a2W04%c#14eeVA2Nn^(m(+&uOHB|> zNzr4(IQ6&F{m5vvrA(fQSUAHPyp`nhrp-b8a)`1>U(c?kfC1ZN%{$$sG}p*`NI0>M z)~Q>WYaTO9#&iF~OTjP2Y8PP&MtPw)y3JcMk<&hwWWWurP(4bE2yK$cAKo$LILIao zv-ip3ak2Ta!Xuv;Umbdm7cV}4Dse2Ijqa{eFqCkTl!Q(`l-_y-xhE<-sgY@4e0N`3 zH*H?HMjT@gIS>!=JY({dNUXCgqZ5gSHMjvWq*O1;E@BJ?U#?v4#I}~9_{9qWA!IeK zQj5`2e^NUr|J2kM?H3ip8@Rc!HM5=NRrHW3Dfx61>!O(ly$33dUL5_zOPKy^W?$Fs_PGiEixq}MzdUUsF2JeDs*)%;L68wOc1W3BPjEtvRLl?*rOwYU^qhBz59? zp;Ob-*DE~>p@q24c9es;CNI+pq8@V2<=xFyG<#SYvexXU^rpi!WhN$j?x`mMr;Pe5 z7~En?i@ELOw7b)g=Vg;@lZx1+d>kxathWQvlBHlCegh|cAyC&{BKevVm91Kf*4w%& z#=jxPzb3ZnNI`Oi5c{DCuRwS|HMPPbLCDnWi)mH-LWJb=arfxNay0naQh2|7{w& zUJCwo8v0H+-{&IKULh18QYg6DGf!V<4u`ylk#GzMQygO|y^<24<&W{tw)T#vzj-*E zmC!q{^WkrCW)i#>5xiPLH_-Fbo*;Hzb|%Mel9`tX2xjWnaUxZ zN+-nAmwXY?#W@G7!<&cj#joru{$_-A|70Lsd zSC|FHT;N5Hj>pusW8~>7rnHAfn(tkz0*u{IWD??Osl0M!3lNy;yTHNR1y{vTHIVw;$P0^f8w<3;DYr!nExKIg@*SbH#1|r+2R)+tCh(kC6{r- zOy`J1)_PiF>C=ba^?VuuuO&rMlkBi+oTNCR({C@jlb@1z{=w0&Ab zy{`F|$iQ+Tq+WRcPc{F3qW@X*DIfx-=|nJMeW3C-;h^Px3r`8fFO?k5rL#@nK7V#^ zQSMXf-w@)bG3Ug2fff={Ue7||Z|*d)nf`%e*w+X%no7DGE{^4)b%LHfLMtitAi2Rv zs6gY%+v5zFMXdM3 z9DJ$uGNJj{o|&COqJtzKda_?Tno~&tF%r8JF0hZd(>sruPx@Wq~$l+Rj_lPcXXU$|6 zz_AOPZqI@S-?9>%gl##j3q)}e2zol zh-2`f>XV2$@4<2z1))=zNUP4Egni9-GhP^@Ue|qFh&yxkJ*02{-wt2jujkOnCfSAS zg$-WcSXN7JR_41IvRtG#FDWV27*shNCITo6qwIU?_*S8Mh&f3YXE{#X9n zz9wTNtsnF6M5s;@%#gHh<<#m?TfHlDoeY01j@%{x=FyIg0C8)?U{yTJ$JYOUr*WU3)>r0NA zvOu0y-4fdJj7hjG;MDhC5`yBZcDlsZ{G%xwS*;J=1 z;^;F*6eQ`mTzUvDvk0eGbd<*BI3~XIgWpdO zYVQ>Ce`ZiVQD;EhbMr6g1tcq;K2t$2kUpfd!|QMHGxc*N6rpQYOqA=darR_C7T8vv?VK z{Uh%ktuiIsMU1fbcT0xUcd48(oYkFe*XXy2{9^2Fyvd#w5lCBM`W>-TM=a>G%vzZg zX`aF6Ucz42G&z^@DDzgmQ2i;u%HGh*UR&A4+R#q(dRA$Bu5RwbjN`dggrV)9jfb!Q z^-wpyqo5e=ke!9k?71D;v)aQMrY|ZZdc*I_<@975dIGi4WS7<7aovvyvc6YE;kCm3 zNthTj?#|}f?s%=;a#@poHFnyAviJ|5WGe0*A%0+n3ySTBV^#{R&QwcOeM|bf1y>d- z)W1a?$2R-5shm2R6T8P7{32E`uVtTYy7_G3p8Q6c;_G1FPKkgDS_*a1SWcw3AEB+( z>=!NOu~rVOaM#=l-u=cBfi2PQ8= z`R;b)8e>ykz2$FL-Q6+O_lQ7fTuw_35)7ge0la(%nsLLrv>QEPKK5M@ep^j8=Z<~W zz%YUBhnV^B0P_(ps>5<&ldL!5d%KF3ioqnuI3|;qL3l#7B2~!hBu}1-z*^SK76k5Q ztn)KU4vOaK^Xv~ip?ZmltKB>(!{{)R*V-|PL!zlw*DHUdFhlz^T8l{pTRq+gUMc1Z z<+JDa;WeM)K3p-RYjY!C2NTdRI8Z-csQB|+@~&T~`0MuD>lJ_B`R7+MUB5{0$0iE7 ziXc1xc&*;=yZ(Id!u3VU2t{^Jc!zkB&PQTD4B>c@Zd@_n-S-OJC(r0b;G zPaDT~FE^yve)scpZs9uh^kXCE{7-ZIo{0M0!_Uj!*ExtEn>zO&G7`V<{`2DR_1gH4 z4Tkrp-T!B${P(?oUJ<-ru==s_@cp#+kLBXu9sIl&cD-=;V>1x=Wf}4J9e<{mUw=7I z@ONVQ9_Re-;AbxQwI8sE|L))i^8D`Q=b-!7!yu6O-OC?4($BH$b!`4)gL(F|gFi*; zzaPxc(crJX&TsJbBli8>*U$0Quf7f?|9o6OqRHQV{T##m+SJ=pKl{4oknc_X$Cmne zw*9rK$E1Jv@_lms{V;x>#C|;td)eO|d_Rmodii-|{Pi%jZ*FS)UI7}lAvH{KoCGsKtMo5K>oaWdU3#jJ4hfPR3Io2O+i~5Cu18Y zT_txrV@GXzH)|`xJTMT-93T+D`~N=v7xzG)@`y|?BT_rbIbv*8i9Ikh(QE=}GzlRv ziYlrtPMObeTj0&w(%&GgHrLW*9X`3|mY6!x>oe0Z!=4AK; zm5{}G?-g`taJZEWpee@(A3gJfjlpP}mkr6V^W4bE=HWR4jYBrwO;F+#`>M>T^Gi)F zU@pu!xMQ8qz_K7_4MNTjP{=K8J~eBc?I*^8VF=sTXe>G?3-p^wSA-;f!K~`4hMR{Y zQ{7`-ObLOc6&#WYa@jKta9*R+jE@NVDnr&PA7wlgm{>(LmL-^yn9k3s6g1p7n(NMwoKvcE<7P+T>>Bn%waf(y(?TM+vVHD*aY=%@Rqyo4a_;8&>``?;}0g zs~?uLs7ikTPm7j0hgzZomd2F?Vkwzc;x1m$2DaST+x~GeC@x3PJl?zZ`*t>V=_>OP z58L5F5}KGIq45~?yO4t1nQC|p$5kiTQO4J69{|1ZPn9t!@rI-T z1p+z)ct1qI8*Ww%uC@-A2DY}Ae|WxfWo_GZMl|p2`d7avnHp%MfHWB*Kb23JMf8?U z#lzq;YTyx8GX^xT>-C_W}wTPP6Xo}q~@fx!zl@S`jj!6)h zSL&5te^;6qfnhP0Qb{w|&tYfr1?MfH&;=Hpd=j^5=ulf4s|QeQc^>vb8LRDFh|2!p z=Q*Wy*NV*WR`3u_9sB3S55#5T`Il`XVji0QE1F+5HKDVCI>CCsMkT$!`l2-ENR>c& z4gH=A^icH$^%&`b7;I&Mi34HSk<@dOJlIo(*^Q)3FB%>9Gt`UeTad!J1qrZjPm;8* z{k^=pnK;#B4@&Lu#_tpw=~Bu=r6tqm-K;cwW-y{P`oXY>>a~{JQL2Q_#rUbj`&PQ+ z*@ep0aW08E{6x`blW+JL9li5g5~r|^Cp0w99=Th9KIk5Rra)4JYddFw`rK_(Nv|eI zDOh=i(>8SM1}y5c_1;)DM$8!1T$yyfq6(0Fpv0`7`IadO;TXle`6}Tg-jx4wvt+x! z`vrBKr*X+wQ)EeYJJ@{Pa-8jGv;CdM3eGy30cg$n=hVS-q3Y;zX@=IVKpjD`_A|TN z*z-2lGx(Ze$6+$CJ}1r+qVT;mzH0Q2@3JvIz3x#ucG3`)Z@3%Pb?lt8V!uN|#hkd7 z9mZD@E-)5H@S{32Qjr+)XJPi;J!mCU_rDzt?O=1d&{<(!9ycQ{Rz?8VZ^U;Ism8&E zZ=A!FxNVygN2o+XWGHgmTjI#PYs4Ja;<_TQ?6@L)et9cB-8@nICTZ#w>y2DUU-ST# zm0IPeSsL+6&y+RQ@`z6X9>;60ey5JLH3S20AZQ&CMvbC5*xMXuR(!+cFA1b0vlCL{ z2u6{QyEr&(P7h6Q<}yF}A`q@+lhm2CtvnmQ(>iu|Ft`v_72ojeUGY_X0$Eo*7TZ03 z5bNJ99+}&XZN)r18kBY^zxc{5gE9Dxp|TQVL-nv}>f`009|APHrEbx<8o6oLZ*__v z?%fbRzzv;)0RdqHK>-7F>Q7Jfvu6FvD**!nOTfGTyN~t+Y0F+lxS-Py&%v9X+oWr) z%tu$uFda5{#}>G|9|l~iXRYsVS^QaH(N0kun8bQc9|@P1zjry~cn9H5dkZBCgEg1*z>dUIeI~5TXsCS~~_NutyzW>@_t4p1f1@0N~5SE8IZY28)U{Q5x1%lCg`YKpQnq7&NeAvNv3t6%l z$}a%B{7;JwP7dtX1qg5^z=43U0JQlJi|uG;Z0zL7@Z*Q+57V6yzY>+s2=H_?*LbuW zkf8pw;e=%tWy-ger5?>Lqe$h8Da+&f%GYaLDY(?e4PhAmgN#obql?=|TRG@U{?5}^ ztkQWYCmNLX_X>;AR>XKAVy>VterM(^8%=`7R81 zXPRlO2F$G5H1Me|wx|Z%?kh;EPH=oD!Ic%YGIhv5~A0m#uyX&gM&DWgiqdVmwk|W${44 ztHV-1%+Z&;weu*MtW07bJXjhM*jSVucVREHR15Qbso1mr%uZp*RT;4q5oHR8O498Q z!o&u#W#)1QB{7Fa)%T1r3OzL5XY8DseIT3b-=#7Aex&J(FUunX;X>x%OkDImRQ{PR zZfMdra?kiyu-=Tc zN=jZo@-=&U@d`uJxUnvqoYqv$b?Ur^{m-wgr4o~py_Tkxm>8u|84{PKsZnO0WKmV1 z{312sS0i}=`#G;j0!I9vjO@OJQAfnWK*9n!#I(F1CMjDNWcy%c=^%L@(-_mVpeSka z4_o+8((6jhKb!;9AqeijBt7$=q+bG%9_`XrmkxUyCIS-G&AZ{tM$RCr?&(<>DJLBzhD03AE`^@Av~#d9k(f<1p|*@xuL3_Y z6^e7k{QP9@Am?F8JzfF%abTyUwh_N54Eo=&t00X&z5>B4OY6=)GqGmd*yX%93A0&9>&d1&QQG7 zQ>%GG(KqZ1-nbOz(~_;QO?)k*)mun1suUHGL^dGi6GI;<-;bgxwKRuJuf>d{18Jwi z3!ibQAJoK1TWa%gtj_q=JOZ6}r?)dH9jy#r9GbeA@QZtb={5ZMtq#| zV$%#Pdo|;v0&{8}`7+Z!Iu#M2N-FHeX%Bj(dG?`2(=(~k_biQNg#RO9Rr*AO!pg}e zjTa2C`KmvHcq+Ol@4je+`OqGfci08pF(320krI~1)yQ72a0(Q=I0XgP;U|lAOzG4VBqQjG6$eW*5LE>y6+Z^V`uv(_?@`fsjv%+2KV3xppsx zLMqPp<5A4|}rv%2KGQ7-R2MS?W`7*RQbU8-mTV|>@7W;^2ikIa4YLL+#l5=t2{Tw${i@A4&Y z63pO;ZlPiw^bPFvYe29j!cd!f)N{53fT|N1I%boB;@rrwNYm-AqEtDO?jqB3Rznvt zX4S~1V9aTVtXm%VwW~6~5Ou~LIt2>4)9m5qq>lXWJvL0>Ukx3B-g%JWkXo@sF{Mi* ztyrPkut+gmj4Va$r^7=*7yTi_nr?43X3{b0RHG|ZxAc@egZsX{P!$4tCBksj1ho&q zg_R6zGfDGiq@`NYM214|Ct+GzZkj-b$bDou+0XX7Z57-k)C*T>fRu{{NeaZcL<3 z8Nf_tfbfr8-VfjCWM*t_%<$v-!z~`GkA>r~qjh0E@WDB{KCx~`ldW$~SjDfA8f7Nn zHZ&f~tFSP~wd0_IkiENBpe!g%7O>?>n0^lm-*FWUOv>I9&{E_BJ4~sBw|=TcS;jKD^w+TyC4{sqzTM`lPOQT?<}mbl^gcl5ceo z0bJ{*Fl-e+6|YuraHdJI>9>U(G+oW!1mfYh+|DNYvH~Rl`xZaWm)}mTG*W$(ds(WXR(2d^tP34DRCb zy*jw>)vax5pq8|OBvVv8nTytaeSXGV*X?>aJ341SoTWJL>FavBA4%?dJ?Ffx{<@0J z(CK|~aTt%e;eEX?8;!TTf{T}OMig3eR=H((N*=E7kMMX7%$M26D9{)89{Iqfebd(R54+5mVvcbDZ-E5UgSX;f;JNDcU^VT22oE9Yiu+Q`|NU6#~Z{AEjM zTj*r0y>vNpkx-31=zuTs^aS0_kS zA3|ox1ReCq6zJ;(4MIkGj0rz3UyQ9=SM`=y&hdB|k>~@ygV87C)9%2C8NK|m!aE@* zR=mz|qa9`k-)>RzN(NFPk1(ps2SeJAl6#!N*Vz1BFmT%lkNDbsY;lqf2h%O#SS+zT zE{lXmqaYKDR{m0UO7aTFz5BBty{{-<~H}SDV_}2e@HSh9Yki++z!^NoaHIW+-`NMj=RV zvV!aq?1axyEr1^|#7_I``ZLuy}{b2A|M${R^(eLS}cIaiSN=_Q)pKcNa(ih z$@wUlm!l`)Gz5*;K4*kUT`ReNQ!22W0RszRw3Fge ztG&C0vw3NUZ~+}jgi~E4<1}Lc<`ym2Skj~VKx=C7EVFEjK?{xp%TXjEvZXLFMYBIS zx#DCWm(6{H1ymC?$y>{N)B~*E1viKml37j6JjG_0H{w|~cdq(LZ!l|vg%UWm;$^-- zFvX^LCiKzi()lLBzn#L(v!+q?L*mugPJ2&7kZzCU#5Lr`+)ZL0w1y?0B}3r9rC4eA zQrf?PS5CTw+tWc18QlTOxzCPVpr*z^0~r9(Z#rbx*a_yoI6|23nHKd^AI&<& z9|m*nz?WSJ&el>F+C0n9IaFHGw;-U*xl6Qf&3cD}b~Dmtk1|WEx6AZ~)z!(b)U(h+ z!O70D-B~+}aPXX0Qt%CBj%ql+r~M|8dH3*TwSCS6($$~a!_JqZSm8iUz&4qL@?jch zD1l8cd^KJ@5^AIU9R=O150w?_fyBWG&FO}$uo zErgkG55QXblfo|MRf7xkC)>MpV}Z!w;ZnMzmQrc+!h5V$PvLT(b&$X#EYI1p9?fHC7Jh@er%I*L6csf2&HCr6rro zn9cZY$4*D~DBibZb2^Mw&}9Z|opny|s&7Wb8zQdxvJ^=s8#HpLMH7@c=9`-5`AEB7 zQ05FI&u3r^bNxd~qplN?zI`&EUxG<4=`^&|>gEIACPS&-k*w2kjJVLSq5gEJmSme0 zqU1rNuVJzPMTPuoS;xk#QW;AKD-gO!t=_6)G9x<@W?!EJ)KH_HU>^D1QPk3{N=P2@ zDuHyn+$69Ww1u1oCT>fxIrmK(B@U_oPLp^o&y2&3mF*o4Squ3fgHVvHY+itJ-aK>T zC!z93JfhH}s%6+bBpprmcuK%E$kw+-Z>zDGrx`9$%kPW;UjOW>*b`nQQ>+5&3;DG&YYmVzV{#S zF8(OzE-z?+Byr1v0woSsNW)mApk}sqdFcXwug0HjekpN2{N1!U8Q0o+u+2wsjB?mm zo0b5fRBviZEO8rRU+z71;25$>>gmEJXD<0HE5DJshU=|97DOWf82^Ec)q*nm%?p&$ zQD+x+G5ajkY0nw&8O3c9SL&UQo_q8_nYFS^u#pC1R&U7>&MFwlQe0ffSAqA2TShH$ zT>%wap4}h)T(X4T@k^=@QRAp|8MdQ};7Zl0VbY|CY&eMp&L>=OohPTF!+?KfW+43T zuVZ2Z-)bd$8k&t}fe9L77DVM8*ZEqv7&sRM^)^t&`b#}pb`HXp6mhZCD-`%5Pa}~P zCUd=^dBt&!+NrcdL(w7?N8lF-KRoK8C#KRh8~BO^r}Tg~4tp*%i!9~53bo}DtIVXx za9>@hWK9E%9DGdA7;FY@6f;~<^zKqO+pAnvBI2ty=VN_!PHTp(@jt9`C1SHn*r4tj zS>Q71%y+l!#PmD02Xqe4l_iwQ6c3yxa&{B!ALW;Yupr=Vb;-=rXt`ywo^0~lTEXwQ zZ_v=IK1xN_(v-fo?c7oo;|Z)=iZ}Ur3F8L{24~zM67j!%oL+>by_$K|)I%i7hB|gm zgt%uBI(UB*L;j2uRKu0_-80^sJ)iX+1?!vc&NC36yITzrn#{0h5$Dio`c_8Hc&$L^ zw=&c|8CbtDdw(`1?$eiu>#ckCjCg5i6(_B(cuewy5H*4<%}Ti&w$+dc){K`;)fGFG zZ@zrvC%yNEM_7Pd8=PwZ56P8@W3MBD$)!8CM%4A$#KFK;&?AI1KgeajOEGJpCiME7 z6@OoBvl?CvuOvwQYkE_VW2T$Q^NQ#!>o367yPCu=O6E_z`kOwCJqX z1%4pkyL0T_y%TW^Nlwv=-UEz!bj>HM`q@4O|1%1lPns6bDz-K9*k;yPe%3e{WW(?_ zRG5^d1*nbhmym>I&SW8@DPp4JQgL+l8kLqIE!rk(=p4O_DAuP2_@NFpIB^8XrK8W& zY*RD`*0-y5u_AhnUU`_Y&`<3}c)@Vn%#jvkAu0QCwpW7Lp$kP!aF%{y2;1 z35$2S;lY#|yzGn9InKs~bV9u7j6Y%?D`_$IGT$8w6o>RyC+c*kFXxqqeHCyK zlb}&>KpK56rz97LuPfLHbU~XWtm#Lf6IQ2}aNk4;qW4i}Bo-YVpIpC`QfwPs#zx_& z5{GuW<#FbN6L|C%z?Mkk3dM;(eJZEg#@JHXg74Fyv>-|nmn*D@0gs?>> z6l_aaTykZyyotM#*qhS1rr^Ry!%yAaYpB9Jyt;yiWse|L74p(O9;ytSvoMu)64Nhg zE5jbV+zalgCL@ANpql8VF1v9f*)2k&cVf}LlS6iU$8vV=?ET(o5Tt8|p8^Ic48UZh zpINMLXZLRs13KnEj*J*-*&oX=Vo$-8x2$i(P}8QRwk5U5IU?5bXTK9`a7pyczIei+ zg4^$yJT?$?H7-R6GRVX@5SH)OPAdR;Sb(@Rc&+i#+Y}7->u-AqlZNHv`ga@w#MOJjrS!03U8hEDAc3%gMS;&0lJ&o z%$xWsfOC=ka-q*B%w{#lHo)tqdMD8a%PU{h7!o`s$dFZo2qp4-!o(F^`N1eMEuRZJ zuQycE@v85xZ^pKX9u(HsfgFFls$oRAu&Uu@`9hoJXg~Dkx9ch>o7Q|!0D?q9{HO!_ z$<#SJI@wzPnB0fm1LC(Q6ob+VqMgJFoV zA&vUTqH9%c<2QxHUhXFYq}piT{iF5#!)rlrD4C*fWdq{^Uf}wA0xU-?7Q` zC)-w!o+i~;Uuj&9#8VeX$EL>w%)RQ% zSEIlpzmO>qCNkDIY}7sj-AkbutuTY8T6cTKWEm~jxA}6lxd}MiNi8gx*W~XhFI*Vc z4S2hqdKR1HOYY^4G`DZYjLq4*ib`hk{RRa2`5xyxODfom$E$(BuqDAZgZucfsFmWN zMk%+`A`JN9PGY`v5Qqg)-Mu7>QOY3qD&&ITM4@z};(bd=0*f~2Dc*Hdd-}#mpKuBN zM6c?m6(;mhcP6?OtcngUK`N$X7UVAAXP^^dAc)X#9v5rAJKpjrD)0i&;?X9$Ckq(< ze}-cONU3k>0SVVj;y;vz63Fm}uBcgCF_=3UTmMM766TQtQRWc-#h(A-ZdAkCc7+Y; zg*WGwf0B3bqRlX0Cbo}GF2_8a4C|&MN9@Zybttts*?8AR>}(sIW?Y><-DDrisoUx8 zYRf6#aqnl3BCR`Gjg|cF$wsq}Z_e|-FUu6TI$QfTyGQTB)QL@t@pcScA)lIs(7N3j zTCb>tTx1st>dK2P+yrfrf-!rSUfo=pSR>x{B0u4+Z+N$BN{P;Ft$jCmmpPs^B6}Nt zXvc{=PC-yyx5D{7JGH*^t43NyUG0q%%2I;Eem*e4^6f4DJ) z*7UOf8fC1wZWyEu_0Ie4ZTqepCxIvZDx6DrAIwNYM?AC6yQoPO;PC5f>YUM97CcFowMon<$hGH1VM!yIm7f(BEi({Qj&HlNkFYF2wee z5-&1SPmY~yZg^XL4ToV)+Ya;`0Z%rB5ao=1J+5|}5mOr6+NUO=@X)4qmyYvQO&Mr= z%trLknoul5K_r&s{oP&TGv+spoyz1m?|gZ@n5L4J%o&!mBN>!uh9JLIet$!DV#J-T zHAci-S=7ta>ab683R7@Z8S2W@FUY{&N-z*1Nkcul7PdN^Ap%+&C&Hq(Le^^vNm%>U z??jn-FRG{uWoY%uF;Nub6_HXvbBr7F28sa<%vqk}dcM(N6_kwEqu}XH&jS&>SqKo{ zLy&z>H<#e!`{?|>?3fr#rn|it>6%qJ=vU?}74RtyjGbDcxx$k;K83#X6VEKq!4wYa zdj+ne4NH^x2?YWw&2JNVJH0nTttS~Hw<^G9g&^Jr+^cekWWWV?Rbb$}1?*G`XS!ch zQ80$RJJaxk>Pd(Q2!W)Q2b#4X9%Ba$NL@=eX){hkMnWz;4_q?9h4FZ#4(HM9r zPx9xJZ{%&V7LE$O8AX|Nh}>3s|Dm_EOOqajkm%#+5l#H;EN;9IdaXy+r~2}wfm+Un zv;zqD^9gNznfRX%z=RP6?3q&8p*qiO}9QHq#QH}+3+*Y?ppyz-l>ju!o}Jm zMKvp@b%Q4JVgVWjh~^;_E^rnJM7jaG9f3GPE5atoL|lI4jrVoL$7IuL6=_K#1v=u% z$bn|Bfg+kMby@N;?=_-K>*8W1%r)yWnlz1HWULFC6$L3ZwBt=a|}7>0xGPxCb&9& zQMgH?f4bc6Pb7d21+p*|`+z1!BIA}rCte+|A z)K8g7LFWLu2%*D$I%`O?LvOX5^&#_qQ*tffE>u=pfmJ-@?BO;NU&O~76<{iqM=|8% zLg&0@0~ew1JH3E8AnvVIVeYCoC!!64W{pa@=56F(UaUEw%jMCm-y_y!X4@MvW2#$0 z&gC#N!yJ|}%Sz`N2XD9}Shh7WGmZ%{riP^3$sAj~6Qv=K(wd+-S_kG4Wa`(Y?%q$Zg3?WdBiuCRP; z3a3G6I(9ip)*|bd^NJcjxlEl_dj&g5ZQvavsP3ZA>+Iw=B1BR5wsS3nq(<_^_s58Op1T=amN#RK*kz66IMvnn7e z7zMvq#i2@m(!!P~KG2Bg?t~HVouGPBRc$06lY)`X4V5wXWE+Lykt&#NnDAF)T}&I3 zH#NN7vFl^wc{e9L7bo2ygC4ukr1WM)N2MArnpW>62Y>g7D-FMpVOK~`TgkuIT0&O@ zwk&qNJ|5N#otZ)Kj-I*f)J=A}lfQ=d>*0xU-PBx%I+gA%1%xT!v;~~5fYaBN!bgE7 zaa~$M-ZzNBUq){Zfu~d%y=uNx%dfLgf@xB_&@=0>ZQ|=EcClK40iPG;oKxDuB$kap z&cT})<4|vj~V;_$QQ#!c8B#aq6yyyydtghl20{aFu^QC zotS>M7SnB*Y^RV$(@*hbFB|EAJXs>(vq!Pbm>DwaE`JME@cCKH6}F2Hb16VE@4vUCQo~!`Qf5s+1%0>QPV-u?=E`%ONB=D+$>3F(>$**tyVdQr~1~|eB%NH)_ z2{!N{`TY|-vP|$@NZXxp=S1-&E;74OO|iKPmr9ez!uKeHO%$X(f$ZZcXz0L`)!k<2 znf#^L9{Y%>xn0g0%p1DZw5PF-KPc%k@w9Qx&A!vB8#5LsdLqQP4zR>Rk_~t%lR!+kh%>{T2yM$E zIK zB*0V(0_i}kR zw17O}KL?-X&)MAnTVNUy`y;gzq#HQt-k4D^R{Pl=v90p6AN*ovz0{Hb*;LFyi~Hm7 zVe3h?ATS=K#M4W*(c>8|_fuX1-eCChTqH*V*)`!2mM`_^yhl|jqzVtC@+gnxtuA_XU||;_$fC=6mHks+5VKFp+DxLAeOH%C&yLNji@9dh=Df4Df*g=O*3JS|K}WqjBt7$2 zUwnl;I|+S{RclAxEi{?$u3^H6TY;1OQDGi<-W?>J1MMT#I*P}cl@7i(tpU~r1Z(rqns@&FW%;0vlz`~kv%13>rumt$Au3j{n9_DQpj$R`LUfXU@*ng|DlryfN z{})5{KN>r@NNHFAm4*H&_dn(-(zQTW{L~QO3SLFDW|6y;_y?6;9kEUDM_0b{*`V2G zJbf-8(k3O0p>+jaW7T4yP!BlkmNOM0pM4MaXu-SU?6Z=p=d}qv0h}4qi*)HVduF|2 zyj~zoe!1&G7A_q(EU3N5%pfbA&T8FZb3_nQQslPH$}Qk#lX!9CKYj0igkiS95ECOC-AwoR^t)y#1>q}W3@b&U4V=FJ8tW|n?)#Wc!?8w^<*+G8f@ z`zMk@vlqJQtlS-_V5m6olqM&^Hr*jx+;gKDgRC)ds%*SoJc|2QIg|z28sAYLV zY>c4=ne)fwa7i>0SR+O`xaA4V(VXh84J1d-+O|$F*6(liO9gES2_7J3?^7i%UfJM3 zxZVk$U@(3mKwe%`qtxp08ck>(VvDAPXxDRVN-JH3ZMD1}^j-T=S^2A#*Gi%w?gcdN zzhl7w{CD@2{?FK>H00ZB4_>HvQp(T{>3r4 z+%7_u<%w}pgr;>YT~HxL(8(fuku3_imKnObTldh$0#`aD)W0}8zV>DW9k zLGu4^H27g6|4oD6(m?+Q4V?aG4;owtZ#0(@(Be|Q zIY`4mBNtT^TmQU&c-Ib~!HkirgxO{QT=16pjPIsJA^kZDb6F`VTXo1z$NVuyL=w)I zlp~`el#1C}x6F#iM*QnWYhC9SR=sAHIgU4!+q3?h+78k_Rx{j?!L~c&$nj68xofF` zUsbg>V2vwl>G1pC1E!?&l8UF`JXa4VU0aC%q5&`n9bnD#_xqNAYU_{N-|S+Rll(it z-*-#?3@`#nkpHxI@>juM_l*4|`U1$E|FVPZSK+^Jj`>Rz2uK0uhwy)~RpwWmU$;U0 zg;Wiw5&y3?Mf{5LYhC$YD20G-{|(9y3;Qd|uVtryp^O5mPyr~v7NPzM@N41BUjUMT zWmy2gUu87EivGGF{+DPa!JnePE|33;@b@*|zvO{{@Q8qb{;}fwtN7m+*M1h?CizMH zKbG2lmHzvx%g@qKHIx1o z55Rv0h-Cd0 z;KyhD7Uk~|>(3~8Y(Jy?E`I&>i~b(>{ESt?@q4VlqM%=~{_Z?~#;W4{J=R~YR8A5c UVBLU#&;fsb0N32e{l|a*2X1dM1^@s6 diff --git a/etc/conf-idifhub.yaml b/etc/conf-idifhub.yaml new file mode 100644 index 0000000..c98873d --- /dev/null +++ b/etc/conf-idifhub.yaml @@ -0,0 +1,69 @@ +sites: + idifhub: + host: idifhub.ecn.purdue.edu + logging: + level: DEBUG + path: /var/log/bastion + persist: + git: + repo: git@github.purdue.edu/ndenny/bastion-idif + key: /home/ndenny/.ssh/ndenny@rcac + + #-- policy defined with the site is the default policy for all RZs within this site. + policy: + history: 60 + drift: 30 + vault: fortress + + #-- declare any (resource) zones within the scope of this site. + zones: + #-- this is the long form for declaring a resource zone (Rz) + LiDAR: + root: /home/jinha/www/lidar + #-- each resource zone can declare its own policy + policy: + history: 60 + drift: 30 + vault: fortress + #-- this next example is the short form for declaring a reource zone + #-- this resource zone inherits all of its policy, etc. from the site in which it resides. + hyperspectral: /home/jinha/www/hsi + +assets: + idifhub: + #-- Top level keys are the names of (resource) zones for this site. + LiDAR: + #-- short form of an asset is just the folder or file that is immediately subordinate to the RZ's root + #-- assets declared using the short form will inherit their policy, etc. from superior contexts. + - QL2_3DEP-LiDAR_IN_2011_2013_l2 + #-- long form of an asset allows for policy override, etc. + - name: QL2_3DEP_LiDAR_IN_2017_2019 + policy: + history: 90 + drift: 30 + - QL2_3DEP_LiDAR_IN_2017_2019_l2 + - QL2_3DEP_LiDAR_IN_2017_2019_laz + - name: QLX_3DEP-LiDAR_US_South + about: low resolution LiDAR coverage of US South (census) Region + RDN: SOUTHLRZ + #-- this is a special case that includes all files and folders in the RZ's root folder as distinct assets. + hyperspectral: "*" + +vaults: + fortress: + protocol: HPSS + host: fortress.rcac.purdue.edu + login: ndenny + key: + path: /home/ndenny/.private/hpss.unix.keytab + refresh: + period: 60 + ssh: + host: data.rcac.purdue.edu + user: ndenny + key: /home/ndenny/.ssh/ndenny@rcac + + BFD: + protocol: copy + zone: /mnt/BFD/bastion + diff --git a/lab/bootstrap.py b/lab/bootstrap.py new file mode 100644 index 0000000..5b787c0 --- /dev/null +++ b/lab/bootstrap.py @@ -0,0 +1,32 @@ +import sys +import pathlib +import logging +import datetime + + +logger = logging.getLogger(__name__) +logging.basicConfig(level = logging.DEBUG) + +RUN_PATH = pathlib.Path(sys.argv[0]).absolute().parent +APP_PATH = RUN_PATH.parent +LIB_PATH = APP_PATH / 'lib' +LBX_PATH = APP_PATH / 'lib-exec' + +sys.path.insert(0, str(LIB_PATH)) + +from Bastion.Common import * +from Bastion.Site import Site +from Bastion.Condo import * +from Bastion.Curator import Snap + +CONF_SEARCH_ORDER = [ + pathlib.Path('/etc/bastion'), + APP_PATH / 'etc', + pathlib.Path('~/.bastion').expanduser() +] + +conf = Condex().load(APP_PATH / 'etc' / 'conf-idifhub.yaml') +site = Site('idifhub').configured( conf ) + +slug = site.zone('LiDAR').assets.any.RDN +latest = Snap.dub(slug, datetime.datetime.now(), "D", "XXV") diff --git a/lib/Bastion/Common.py b/lib/Bastion/Common.py index 666a5b6..a96e689 100644 --- a/lib/Bastion/Common.py +++ b/lib/Bastion/Common.py @@ -4,26 +4,275 @@ import json import hashlib import base64 +import numbers as pynumbers +import datetime +import string +import operator import yaml +DAY = datetime.timedelta(days = 1) +DAYS = DAY +SECOND = datetime.timedelta(seconds = 1) +SECONDS = SECOND +MINUTE = datetime.timedelta(minutes = 1) +MINUTES = MINUTE +HOUR = datetime.timedelta(hours = 1) +HOURS = HOUR + + + +class Thing: + def __init__(self, **kwargs): + for k, v in kwargs.items(): + setattr(self, k, v) + + +def RDN(x): + """ + Answers the relatively distinguishing name (RDN) of object, x. + If x is a string, it is assumed that x is the name. + If x is an object with a CN attribute, answers x.RDN + """ + if isinstance(x, str): + return x + elif hasattr(x, 'RDN'): + return x.RDN + else: + raise ValueError("RDN(x) - requires that object x have an .RDN property") + + + +class entity: + """ + I am syntactic sugar. + Mostly, I do type checking, e.g. + x = 4 + if entity(x).isNumeric: print("it's a number!") + """ + def __init__(self, subject): + self.subject = subject + + @property + def isNumeric(self): + return isinstance(self.subject, pynumbers.Real) + + @property + def isDuration(self): + return isinstance(self.subject, datetime.timedelta) + + @property + def isString(self): + return isinstance(self.subject, str) + + + +class UnknownType(object): + _ = None + + def __new__(cls): + if UnknownType._ is None: + UnknownType._ = object.__new__(cls) + return UnknownType._ + + def __repr__(self): + return "<|???|>" + +Unknown = UnknownType() + + + +class CURIE(tuple): + """ + a CURIE (or Compact URI) defines a generic, abbreviated syntax for expressing Uniform Resource Identifiers (URIs). + It is an abbreviated URI expressed in a compact syntax, and may be found in both XML and non-XML grammars. + A CURIE may be considered a datatype. + See ... + * https://en.wikipedia.org/wiki/CURIE + * https://www.w3.org/TR/2010/NOTE-curie-20101216/ + examples ... + + [unit:METER] + [doi:10.1000/182] + [RADISH:V9RPl4gIaYzB3_clVjDc] + """ + def __new__(cls, *args, **kwargs): + if len(args) == 1: + arg = args[0] + if isinstance(arg, cls): + return arg + + #-- ask the object if it already has a CURIE representation ... + f = getattr(arg, 'CURIE', None) + if f: + if isinstance(f, cls): + #-- Yes, the object does have an already prepared CURIE representation. + return f + if callable(f): + #-- No, the object doesn't have a CURIE already prepared, + #-- but it does have a method to create a CURIE representation. + return f( ) #-- invoke the object's CURIE creation method. + + #-- If I was given a string, I should be able to parse it. + if isinstance(arg, str): + t, p, q = cls.parse(arg) + return tuple.__new__(cls, (t, p, q)) + + if len(args) == 2: + itype = args[0] + path = args[1] + args = urllib.parse.urlencode(kwargs) if kwargs else None + return tuple.__new__(cls, (itype, path, args)) + + raise TypeError + + itype = property(operator.itemgetter(0)) + path = property(operator.itemgetter(1)) + args = property(operator.itemgetter(2)) + + def __str__(self): + if self.args: + return "[{}:{}?{}]".format(self.itype, self.path, self.args) + else: + return "[{}:{}]".format(self.itype, self.path) + + def __repr__(self): + return str(self) + + @property + def ref(self): + if self.args: + return "{}?{}".format(self.path, self.args) + else: + return self.path + + @staticmethod + def parse(c): + if (c[0] == '[') and (c[-1] == ']'): + cinner = c[1:-1] + if ':' in cinner: + i = cinner.find(':') + itype = cinner[:i] + objref = cinner[i+1:] + if '?' in objref: + j = objref.find('?') + path = objref[:j] + query = objref[j+1:] + else: + path = objref + query = None + + return (itype, path, query) + + raise ValueError + + def __matmul__(self, ns): + """ + I answer my reference iff my itype is the given namespace (ns). + Otherwise, I raise an exception. + """ + if self.itype == ns: + return self.ref + else: + raise TypeError + + +class Quantim: + """ + Quantized Time (Quantim). + I represent with ~ 1 minute of precision a date within the 3rd millenium. + (i.e. 2000 - 2999) + The quantized minute is described in base36 (0...9,A...Z). + With two digits, the day is divided into 1,296 quantums, each of which is ~ 66.67 seconds. + """ + + EN36 = list(string.digits + string.ascii_uppercase) + DE36 = dict([(c, i) for i, c in enumerate(EN36)]) + QUANTUM = 86400.0 / (36**2) + + def __init__(self, whence, separator = None): + self.separator = separator if (separator is not None) else '' + + if isinstance(whence, datetime.datetime): + year_starts = datetime.datetime(whence.year, 1, 1, 0, 0, 0) + adnl_seconds = (whence - year_starts).seconds + + self.dY = whence.year - 2000 + self.dD = (whence - year_starts).days + self.qM = int(adnl_seconds // Quantim.QUANTUM) + + elif isinstance(whence, str): + if self.separator: + words = whence.split(self.separator) + if len(words) == 3: + yW = words[0] + dW = words[1] + qW = words[2] + else: + raise Exception("Quantim:__init__ parse error for '{}'".format(whence)) + else: + if len(whence) == 8: + yW = whence[0:3] + dW = whence[3:6] + qW = whence[6:8] + else: + raise Exception("Quantim:__init__ parse error for '{}'".format(whence)) + + self.dY = int(yW) + self.dD = int(dW) + self.qM = (Quantim.DE36[qW[0]] * 36) + Quantim.DE36[qW[1]] + + else: + raise ValueError("Quantim:__init__ cannot create instance from whence '{}'".format(whence)) + + def __str__(self): + lsq = self.qM % 36 + msq = self.qM // 36 + xmap = list(string.digits + string.ascii_uppercase) + yW = "{:03d}".format(self.dY) + dW = "{:03d}".format(self.dD) + qW = xmap[msq] + xmap[lsq] + return self.separator.join([yW, dW, qW]) + + def datetime(self): + y = datetime.datetime(self.dY + 2000, 1, 1, 0, 0, 0) + elapsed_days = self.dD * DAYS + elapsed_seconds = ((self.qM * Quantim.QUANTUM) + (Quantim.QUANTUM / 2)) * SECONDS + return (y + elapsed_days + elapsed_seconds) + + @classmethod + def now(cls): + return cls(datetime.datetime.now()) + + def Slug40(text): """ - I generate a 5-character slug based on the given text. + I generate a 8-character slug based on the given text. The slug is generated by hashing the text using SHAKE128, then taking a 40-bit digest and encoding the digest using base32. """ h = hashlib.shake_128() h.update(text.encode('utf-8')) bs = h.digest(5) - return base64.b32encode(bs) + return base64.b32encode(bs).decode('utf-8') - -class Sable: +class canTextify: + """ + I am a trait for serialization using JSON and YAML. + """ def toJDN(self, **kwargs): - raise NotImplementedError + raise NotImplementedError(".toJDN is subclass responsibility") + + @classmethod + def fromJDN(cls, jdn, **kwargs): + raise NotImplementedError(".fromJDN is subclass responsibility") + + #↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑ + #-- host class protocol. | + #-- host classes must implement the methods, above. | + #----------------------------------------------------- def toJSON(self, **kwargs): jdn = self.toJDN(**kwargs) @@ -33,10 +282,6 @@ def toYAML(self, **kwargs): jdn = self.toJDN(**kwargs) return yaml.dump(jdn, default_flow_style = False, indent = 3) - @classmethod - def fromJDN(cls, jdn, **kwargs): - raise NotImplementedError - @classmethod def fromJSON(cls, js, **kwargs): jdn = json.loads(js) diff --git a/lib/Bastion/Condo.py b/lib/Bastion/Condo.py new file mode 100644 index 0000000..4959187 --- /dev/null +++ b/lib/Bastion/Condo.py @@ -0,0 +1,590 @@ +""" +Bastion.Condo + +I provide a nested mapping object, useful for configuration, etc. +Multi-part keys (typically separated by the dot "." element) are parsed into nested keys. + +Typical use starts with constructing a condo using the Condex( ) construction function, e.g. ... + +conf = Condex( ) +conf['some.nested.key'] = 'something' +conf['some.other.key'] = 'else' + +#------------------------------------ +#-- These two lines are equivalent. | +#------------------------------------ +print(conf['some.other.key']) +print(conf['some']['other']['key']) +""" +import sys +import os +import collections.abc as pycollections +import json +import datetime +import fnmatch +import logging +import pathlib +import csv + +try: + import yaml +except: + pass + +try: + import openpyxl +except: + pass + + + +#------------------------------------ +#-- Check that we're using Python 3 | +#------------------------------------ +assert(sys.version_info.major == 3) + + +#------------------------------- +#-- Get a handle to a logger. | +#------------------------------- +logger = logging.getLogger(__name__) + + +class ftuple(tuple): + """ + ftuple is a special case of a tuple that returns its own iterator when called. + This is used as a hack to allow my own (NTD) .keys (as a property) idiom to be + shoe-horned in where standard python might expect .keys to be a callable method + Specifically, dict.update(x) will first check to see if x has an attribute named "keys" + It then assumes that keys is a callable. In my own style, .keys returns a set or a tuple + (depending on the class) and neither sets nor tuples are normally callable. + ftuple should allow both styles to work. + """ + def __call__(self): + return iter(self) + + +class CxKeyView: + def __init__(self, cxnode): + self.nest = cxnode + + def dive(self, prekey = None): + for k, v in sorted(self.nest.children.items()): + k = CxKey(k) if prekey is None else (prekey / k) + if isinstance(v, CxNode): + for subk in v.keys.dive(k): + yield subk + else: + yield k + + @property + def deep(self): + return frozenset( self.dive() ) + + def matching(self, pattern): + spattern = str(pattern) + return list( sorted(filter(lambda k: fnmatch.fnmatch(str(k), spattern), self.deep)) ) + + def __contains__(self, k): + k = CxKey(k) + if k.head in self.nest.children.keys( ): + if k.tail.isNotEmpty: + return (k.tail in self.nest.children[k.head]) + else: + return True + else: + return False + + def __iter__(self): + return iter(sorted(self.nest.children.keys())) + + def __call__(self): + return iter(self) + + +class CxItem(tuple): + def __new__(cls, k, v): + return tuple.__new__(cls, (CxKey(k), v)) + + @property + def key(self): + return self[0] + + @property + def value(self): + return self[1] + + + +class CxKey(tuple): + def __new__(cls, k = None): + if k is None: + return tuple.__new__(cls, [None]) + + if isinstance(k, cls): + return k + + if isinstance(k, str): + tokens = list(map(lambda token: token.strip( ), k.split('.'))) + return cls(tokens) + + if isinstance(k, pycollections.Sequence): + return tuple.__new__(cls, k) + + if isinstance(k, CxNode): + if k.isChild: + return CxKey(k.parent) / k.name + else: + return CxKey(None) + + raise TypeError + + @property + def isEmpty(self): + if len(self) == 0: + return True + elif len(self) == 1: + return ( (self[0] is None) or (self[0] == '') ) + return False + + @property + def isNotEmpty(self): + if len(self) > 0: + if self[0]: + return True + return False + + @property + def head(self): + return self[0] + + @property + def tail(self): + return CxKey(self[1:]) + + def __str__(self): + return '' if self.isEmpty else '.'.join(map(str, self)) + + def __format__(self, *args): + return str(self) + + def __repr__(self): + return str(self) + + def __truediv__(self, other): + other = CxKey(other) + if self[0] is None: + return CxKey(other) + else: + return CxKey(tuple(self) + tuple(other)) + + + +class CxNode(object): + def __init__(self, parent = None, name = None, value = None): + self.children = { } + self.parent = parent + self.name = name + self.value = value + + def show(self, *args, **kwargs): + for item in self.walk(): + print("{item.key:40s} → {item.value}".format(item=item)) + + def walk(self): + """ + I dive into my nested structure and answer each CxItem in key order. + """ + for k in self.keys.dive(): + yield CxItem(k, self[k]) + + @property + def flattened(self): + """ + Answers a dictionary where all keys and subkeys are put into the same 0-level depth. + """ + return dict([(k, self[k]) for k in self.keys.deep]) + + def __iter__(self): + for k in sorted(self.children.keys()): + yield CxItem(k, self.children[k]) + + def __eq__(self, other): + if isinstance(other, CxNode): + return (self.flattened == other.flattened) + + if isinstance(other, pycollections.Mapping): + #-- Flatten the other mapping. + flatter = tuple( [(k, other[k]) for k in sorted(other.keys( ))] ) + return (self.flattened == flatter) + + raise TypeError + + def _dex(self): + return dict( self.walk ) + + def get(self, *args): + """ + gets the value of a given key. + .get(f, key) - answers the value of f(self[key]) + .get(f, key, default) - answers the value of f(self[key] or default) + .get(key) - answers the value associated with key + .get(key, default) - answers the value of self[key] or default + """ + if len(args) == 1: + #-- .get(key) + return self[ args[0] ] + + if len(args) == 2: + #-- .get(key, default) + if isinstance(args[0], str): + k, default = args + try: + return self[k] + except KeyError: + return default + + #-- .get(form, key) + if callable(args[0]): + xform, k = args + return xform(self[k]) + + if len(args) == 3: + #-- .get(form, key, default) + xform, k, default = args + return xform( self.get(k, default) ) + + raise ValueError + + def __contains__(self, k): + return (k in self.keys) + + def __getitem__(self, k): + k = CxKey(k) + if k.isEmpty: + return self + else: + if k.head in self.children: + if isinstance(self.children[k.head], CxNode): + return self.children[k.head][k.tail] + else: + v = self.children[k.head] + if callable(v): + return v(self.root, k) + else: + return v + return self.children[k.head] + else: + raise KeyError(k) + + def __setitem__(self, k, v): + k = CxKey(k) + if len(k) == 1: + self.children[k.head] = v + else: + if (k.head not in self.children): + self.children[k.head] = CxNode(self, k.head) + self.children[k.head][k.tail] = v + + def update(self, d): + if d is not None: + if isinstance(d, CxNode): + for k in d.keys.deep: + self[k] = d[k] + return self + + if isinstance(d, pycollections.Mapping): + for k in d.keys( ): + if isinstance(d[k], pycollections.Mapping): + if k not in self: + self[k] = CxNode(self, k) + self[k].update(d[k]) + else: + self[k] = d[k] + return self + + raise TypeError + + @property + def keys(self): + return CxKeyView(self) + + @property + def root(self): + """ + Answers a reference to the root node of this configuration tree. + """ + if self.parent is None: + return self + else: + return self.parent.root + + @property + def isRoot(self): + return True if self.parent is None else False + + @property + def isChild(self): + return True if self.parent is not None else False + + def toJDN(self): + d = { } + for k in self.keys.deep: + if k.tail.isEmpty: + d[str(k)] = self[k] + else: + d[str(k.head)] = self[k.head].toJDN( ) + return d + + def load(self, srcname, **kwargs): + loader = CxLoader(self) + return loader.load(srcname, **kwargs) + + def sed(self, tokens): + sedr = CxSEDR(self) + sedr.sed(tokens) + return self + + + +class CxLoader: + def __init__(self, node): + self.node = node + + def load(self, srcname, **kwargs): + srcpath = pathlib.Path(srcname) + + if srcpath.exists(): + logger.debug("CxNode/load: reading configuration from {}".format(srcpath)) + + form = kwargs.get('form', None) + if form is None: + form = srcpath.suffix[1:].upper() + else: + form = form.strip( ).upper( ) + + loaders = { + 'JSON': self.loadJSON, + 'JSN': self.loadJSON, + 'JS': self.loadJSON, + 'YML': self.loadYAML, + 'YAML': self.loadYAML, + 'CSV': self.loadCSV, + 'XLSX': self.loadXLSX + } + + loader = loaders.get(form, None) + + if loader: + loader(srcname, **kwargs) + else: + raise Exception("CxNode/load: I don't know how to handle files of form '{}'".format(form)) + else: + logger.error('CxNode/read: I cannot find the specified file: {}'.format(srcpath)) + raise FileNotFoundError(srcname) + + return self.node + + def loadJSON(self, fname, **kwargs): + blob = json.load(open(fname)) + self.node.update(blob) + return self.node + + def loadYAML(self, fname, **kwargs): + if 'yaml' in sys.modules: + blob = yaml.safe_load(open(fname)) + self.node.update(blob) + return self.node + + msg = "CxNode/loadYAML: yaml module not available" + logger.error(msg) + raise Exception(msg) + + def loadCSV(self, fname, **kwargs): + keyCol = kwargs.get('keyCol', 'key') + valCol = kwargs.get('valCol', 'value') + + with open(fname, 'rt') as src: + tbl = csv.DictReader(src) + for row in tbl: + k = row[keyCol] + v = row[valCol] + self.node[k] = v + + return self.node + + def loadXLSX(self, srcpath, sheet = None, **kwargs): + xlspath = pathlib.Path(srcpath) + if 'openpyxl' in sys.modules: + keyCol = kwargs.get('keyCol', 'A') + valCol = kwargs.get('valCol', 'B') + bgnRow = kwargs.get('bgnRow', 2) + + wb = openpyxl.load_workbook(str(xlspath)) + ws = None + if sheet: + if isinstance(sheet, str): + ws = wb[sheet] + elif isinstance(sheet, int): + #-- Get sheet by index + dex = dict(enumerate(wb.sheetnames)) + ws = dex[sheet] + + if not ws: + msg = "Condo/loadXLSX: .xlsx workbook sheets must be by name or index" + logger.error(msg) + raise ValueError(msg) + + for i in range(2, ws.max_row+1): + k = ws["{}{}".format(keyCol,i)].value + v = ws["{}{}".format(valCol,i)].value + if k is not None: + k = k.strip() + if k: + self.node[k] = v + + else: + msg = "CxNode/loadXLSX: openpyxl module not available" + logger.error(msg) + raise Exception(msg) + + return self.node + + + +class CxSEDR: + def __init__(self, node): + self.node + + def sed(self, tokens): + """ + Interpet the given list of tokens as an edit stream (aka "sed"). + + ^file (LOAD) reads the given file name into the configuration + key: (SET) sets a nested key to some value + key+ (SADD) adds a value to the key (set semantics) + key++ (BADD) adds a value to the key (bag semantics) + key- (SREM) removes a value from the key (set semantics) + key-- (BREM) removes all instances of the value from the key (bag semantics) + key! (ON) sets the key to True + key~ (OFF) sets the key to False + """ + state = 'SCANNING' + key = None + nakeds = [ ] + + for token in tokens: + logger.debug("CxSEDR.sed state is {} working on key '{}' ingesting token '{}'".format(state, key, token)) + if state == 'SCANNING': + op = 'SCANNING' #-- default to continuing the scanning state, unless otherwise set. + + if token[0] == '^': + #-- load the file + path = token[1:] + node.load(path) + + elif token[-1] == ':': + key = token[:-1] + op = 'SET' + + elif token[-1] == '!': + key = token[:-1] + self.node[key] = True + + elif token[-1] == '~': + key = token[:-1] + self.node[key] = False + + elif token[-1] == '+': + if token[-2:] == '++': + key = token[:-2] + op = 'BADD' + else: + key = token[:-1] + op = 'SADD' + + elif token[-1] == '-': + if token[-2:] == '--': + key = token[:-2] + op = 'BREM' + else: + key = token[:-1] + op = 'SREM' + + else: + #--------------------------------------------------------- + #-- the token doesn't appear to have any sed semantics, | + #-- so add it to the "naked" list. | + #--------------------------------------------------------- + nakeds.append(token) + + elif state == 'SET': + self.node[key] = token + op = 'SCANNING' + + elif state == 'SADD': + curval = self.node[key] if (key in self.node) else [] + + if isinstance(curval, pycollections.MutableSequence): + if token not in self.node[key]: + self.node[key].append(token) + + elif isinstance(curval, pycollections.MutableSet): + self.node[key].add(token) + + else: + self.node[key] = [curval, token] + + op = 'SCANNING' + + elif state == 'BADD': + curval = self.node[key] if (key in self) else [] + + #---------------------------------------------------------- + #-- if the current value is a mutable set, | + #-- convert the set to a list to handle the bag semantics | + #---------------------------------------------------------- + if isinstance(curval, pycollections.MutableSet): + self.node[key] = list(curval) + curval = self.node[key] + + if isinstance(curval, pycollections.MutableSequence): + self.node[key].append(token) + else: + self.node[key] = [curval, token] + + op = 'SCANNING' + + elif state == 'SREM': + if key in self: + curval = self.node[key] + if isinstance(curval, pycollections.MutableSet): + self.node[key].discard(token) + if isinstance(curval, pycollections.MutableSequence): + if token in self.node[key]: + self.node[key].remove(token) + op = 'SCANNING' + + elif state == 'BREM': + if key in self: + curval = self.node[key] + if isinstance(curval, pycollections.MutableSet): + self.node[key].discard(token) + elif isinstance(curval, pycollections.MutableSequence): + while token in self[key]: + self.node[key].remove(token) + state = op + + return nakeds + + + + + + +def Condex(*args, **kwargs): + c = CxNode( ) + for arg in args: + c.update(dict(arg)) + c.update(kwargs) + return c diff --git a/lib/Bastion/Curator.py b/lib/Bastion/Curator.py index 4732e9c..f564e3f 100644 --- a/lib/Bastion/Curator.py +++ b/lib/Bastion/Curator.py @@ -3,136 +3,77 @@ I provide mostly data structures for working with archives and backups. """ -from .Common import Sable, Slug40 -import hashlib -import base64 +import pathlib +from .Common import * -class Asset(Sable): - def __init__(name, path, about, **kwargs): - self.name = RDN - self.path = pathlib.Path(path) - self.about = about - self.RDN = None +#-- Archives > Anchors > Snaps +#-- An archive is a chronicle (time ordered series) of snaps. +#-- Each snap is a 2-tuple of (anchor, differential) +#-- An "anchor" is a full backup of the dataset. +#-- Each blob in the archive is recorded as ... +#-- {slug}-{quantim}-[A|D]{anchor} +#-- Where {slug} is the Slug40 encoding (a 8 character, base32 word) of the dataset name (relative to the Rz), +#-- {quantim} is the 8 character encoding of timestamp using the Quantim method +#-- A if the blob is an anchor (full backup) and D if the blob is a differential. +#-- {anchor} - is a 3 character random string that cannot conflict with any other anchors currently in the archive. - for kwarg in ['RDN']: - if kwarg in kwargs: - setattr(self, kwarg, kwargs[kwarg]) - if self.RDN is None: - self.RDN = Slug40(self.name) - - def toJDN(self, **kwargs): - jdn = { - '_type': "Curator.Asset", - 'name': self.name, - 'path': str(self.path), - 'about': self.about, - 'RDN': self.RDN - } - return jdn - - -class Archive(Sable): +class Archive(canTextify): """ - I represent a top-level archive of some dataset. - I am analagous to a git repository in that I may contain - multiple branches of object evolution. + I represent the chronicled archive of some dataset. + I hold a series of "snaps". """ - def __init__(self, name, **kwargs): - self.name = name - self.RDN = None - - for kwarg in ['RDN']: - if kwarg in kwargs: - setattr(self, kwarg, kwargs[kwarg]) - - if self.RDN is None: - self.RDN = Slug40(self.name) + def __init__(self, asset, **kwargs): + self.asset = asset + self.zone = asset.zone + self.site = asset.zone.site + self.lscat = [ ] #-- list of files that are in this archive. + + def snaps(self): + #-- Answers a list of all snaps in my lscat that are for the given asset. + #-- Snaps are ordered chronologically from earliest to lastest. + raise NotImplementedError + + def snap(self, whence): + raise NotImplementedError + + def anchor(self, snap): + """ + I answer the snap that is the anchor layer for the given snap. + """ + raise NotImplementedError def toJDN(self, **kwargs): jdn = { - '_type': "Curator.Archive", - 'name': self.name, - 'RDN': self.RDN + '_type': "Bastion.Curator.Archive", + 'site': self.site, + 'zone': self.zone, + 'resource': self.resource, } return jdn -class Branch(Sable): - """ - I represent a branch (timeline of object evolution) relative to an archive. - """ - def __init__(self, RDN): - self.RDN = RDN - self.name = RDN - self._snaps = [ ] - - def head(self): - return self._snaps[-1] - - def base(self): - return self._snap[0] - - def created(self): - return self.base.deposited - - def updated(self): - return self.head.deposited - - def commit(self, snap): - self._snaps.append(snap) - self._snaps = sorted(self._snaps, key = lambda s: s.deposited) - - def __iter__(self): - return iter(self._snaps) - - @property +class Snap: def age(self, whence = None): - whence = whence if (whence is not None) else datetime.datetime.now() - return (whence - self.created) - - - -class BlobRef: - def __init__(self, RDN, archive, branch, deposited): - self.RDN = RDN - self.name = RDN - self.archive = archive.RDN if isinstance(archive, Archive) else str(archive) - self.branch = branch.RDN if isinstance(branch, Branch) else str(branch) - self.deposited = deposited - - - -class Snap(Sable): - """ - I represent a "snapshot" in time and contain the necessary - information to restore a dataset to the state observed when this - snap was deposited. - """ - def __init__(self, RDN, archive, branch, deposited, layers, **kwargs): - self.RDN = RDN - self.name = RDN - self.archive = archive.RDN if isinstance(archive, Archive) else str(archive) - self.branch = branch.RDN if isinstance(branch, Branch) else str(branch) - self.deposited = deposited - self.layers = layers - self.about = kwargs.get('about', "") - - @property - def age(self, whence = None): - whence = whence if (whence is not None) else datetime.datetime.now() - return (whence - self.deposited) - - def toJDN(self): - jdn = { - 'RDN': self.RDN, - 'archive': self.archive, - 'branch': self.branch, - 'deposited': self.deposited.isoformat(), - 'layers': self.layers[:] - } - return jdn - - + """ + I answer a datetime timedelta (elapsed time) between whence and the encoded datetime of this snap. + If no "whence" is explicitly given, I assume the current UTC time. + """ + whence = whence if whence is not None else datetime.datetime.utcnow() + return (whence - self.when.datetime()) + + @staticmethod + def parse(path): + path = pathlib.PurePath(path) + return Thing(**{ + 'slug': path.stem[0:8], + 'when': Quantim(path.stem[8:16]), + 'layer': path.stem[16], + 'anchor': path.stem[17:20], + }) + + @staticmethod + def dub(slug, when, layer, anchor): + return "{}{}{}{}".format(slug, str(Quantim(when)), layer, anchor) diff --git a/bin/HPSS.py b/lib/Bastion/HPSS.py similarity index 97% rename from bin/HPSS.py rename to lib/Bastion/HPSS.py index c383763..45e86af 100644 --- a/bin/HPSS.py +++ b/lib/Bastion/HPSS.py @@ -5,23 +5,10 @@ import datetime import json +from Bastion.Common import Thing, Unknown -class Thing(object): - pass -class UnknownType(object): - _ = None - - def __new__(cls): - if UnknownType._ is None: - UnknownType._ = object.__new__(cls) - return UnknownType._ - - def __repr__(self): - return "<|???|>" - -Unknown = UnknownType() #-------------------------------------------------------- #-- Set up an alias as a way of easily describing that | diff --git a/lib/Bastion/Site.py b/lib/Bastion/Site.py index f82bab8..4eb9cd1 100644 --- a/lib/Bastion/Site.py +++ b/lib/Bastion/Site.py @@ -2,87 +2,419 @@ Bastion.Site """ import logging +import pathlib +import random -import openpyxl - -from .Common import Sable, Slug40 -from .Curator import Asset +from .Common import * +from .Condo import CxNode +#from .Curator import Asset logger = logging.getLogger(__name__) -def loadSiteConfig(path = None): - if path is not None: - src = pathlib.Path(src) - else: - for p in ['~/.bastion/site.xlsx', '/etc/bastion/site.xlsx']: - p = pathlib.Path(p).expanduser() - if p.exists(): - src = p - break +#--------------------- +#-- Module defaults. | +#--------------------- +DEFAULT_ASSET_HISTORY = 60 * DAYS +DEFAULT_ANCHOR_DRIFT = 30 * DAYS +DEFAULT_ARCHIVE_VAULT = "fortress" +DEFAULT_LOGGING_PATH = pathlib.Path("/var/log/bastion") + + +def asLogLevel(x): + if isinstance(x, int): + return x + elif isinstance(x, str): + x = x.upper() + if x in ['DEBUG', 'INFO', 'WARN', 'ERROR', 'CRITICAL']: + return getattr(logging, x) + raise ValueError("asLogLevel - cannot interpret '{}' as a log level.".format(x)) + + + +class canConfigure: + """ + Trait for objects that can configure themselves from a nested dictionary. + """ + @classmethod + def fromConf(cls, condex): + raise NotImplementedError(".fromConf is subclass responsibility") + + def configured(condex): + raise NotImplementedError(".configured is subclass responsibility") + + - if src is None: - raise Exception("Cannot find site.xlsx configuration") - else: - logger.info("loading site configuration from {}".format(str(src))) +class Site(canConfigure, canTextify): + SITES = { } - conf = SiteConfig() - conf.loadXLSX(src) + @staticmethod + def named(nm): + return Site.SITES[nm] - return conf + def __new__(cls, site): + if isinstance(site, cls): + return site + else: + if site in Site.SITES: + return Site.SITES[site] + else: + return object.__new__(cls) + def __init__(self, name): + if isinstance(name, str): + if name not in Site.SITES: + Site.SITES[name] = self + self.name = name + self.host = "127.0.0.1" -class SiteConfig: - def __init__(self): - self.site = { } #-- confvar -> confval - self._assets = { } #-- @asset -> Asset + self.logging = Thing() + self.logging.level = logging.WARN + self.logging.path = pathlib.Path("/var/log/bastion") + self.logging.persistence = None + + self.policy = RetentionPolicy() + + self._zones = { } + self._catalogs = { } + + def assets(self, zone): + k = RDN(zone) + if k not in self._catalogs: + self._catalogs[k] = AssetCatalog(self, k) + return self._catalogs[k] + + @property + def zones(self): + return [self._zones[z] for z in sorted(self._zones.keys())] + + def zone(self, z): + return self._zones[RDN(z)] + + @property + def RDN(self): + """ + I am the relatively distinguishing name for this object. + """ + return self.name + + def configured(self, conf): + if conf: + condex = conf['sites'][self.name] + if 'logging' in condex: + self.logging.level = condex.get(asLogLevel, 'logging.level', logging.WARN) + + if 'policy' in condex: + self.policy = RetentionPolicy(condex['policy']) + + if 'zones' in condex: + for zkey, zspec in condex['zones']: + zname = str(zkey) + logger.info("associationg (resource) zone {} to site {}".format(zname, self.name)) + logger.debug("self.zones is type {}".format(type(self.zones))) + self._zones[zname] = Zone(self, zname).configured( zspec ) + + aspecs = conf.get("assets.{}".format(self.name), None) + if aspecs is not None: + for zname in aspecs.keys: + logger.info("reading assets for resource zone {} in site {}".format(zname, self.name)) + zone = self.zone(zname) + for aspec in aspecs[zname]: + #-- Short form.... + if entity(aspec).isString: + #-- Short form.... + zone.assets.add( aspec ) + logger.debug("added asset {} to zone {} by short form description".format(aspec, zname)) + else: + #-- Long form ... + zone.assets.add( Asset(zone, aspec) ) + logger.debug("added asset {} to zone {} by long form description".format(aspec, zname)) + + return self + + def resources(self, zone): + zname = zone.name if isinstance(zone, ResourceZone) else zone + + if zname not in self._catalogs: + self._catalogs[zname] = ResourceCatalog(self, zname) + + return self._catalogs[zname] + + def asset(self, slug): + """ + Will search through all zones to locate the asset identified by the given slug. + Raises an error if there are two or more zones that containe the same slug (i.e. a hash collision). + """ + zoned = None + for zone in self.zones: + if slug in zone.assets: + if zoned is not None: + raise Exception("Site.asset - multiple zones ({}, {], etc.) claim asset {}".format(zone.name, zoned.name, slug)) + else: + zoned = zone + if zoned: + return zoned.assets[slug] + else: + return None + + +class RetentionPolicy(canConfigure, canTextify): + """ + How long to store an object, how many copies, and where the copies are deposited. + """ + def __init__(self, conf = None): + self._history = DEFAULT_ASSET_HISTORY + self._drift = DEFAULT_ANCHOR_DRIFT + self.vault_name = DEFAULT_ARCHIVE_VAULT + + if conf: + self.configured(conf) + + def _axs_drift(self, t = None): + if t is not None: + if entity(t).isDuration: + self._drift = t + elif entity(t).isNumeric: + self._drift = t * DAYS + else: + raise ValueError("RetentitonPolicy.drift: drift must be numeric or datetime.timedelta") + return self._drift + + def _axs_history(self, t = None): + if t is not None: + if entity(t).isDuration: + self._history = t + elif entity(t).isNumeric: + self._history = t * DAYS + else: + raise ValueError("RetentitonPolicy.history: history must be numeric or datetime.timedelta") + return self._history + + history = property(_axs_history, _axs_history) + drift = property(_axs_drift, _axs_drift) + + def configured(self, conf): + if conf: + self.history = conf.get('history', DEFAULT_ASSET_HISTORY) + self.drift = conf.get('drift', DEFAULT_ANCHOR_DRIFT) + self.vault = conf.get('vault', DEFAULT_ARCHIVE_VAULT) + return self + + def toJDN(self, **kwargs): + return {'history': self.history.days, 'drift': self.drift.days} + + @classmethod + def fromJDN(cls, jdn): + depth = jdn['history'] * DAYS + radius = jdn['drift'] * DAYS + return csl(history = depth, drift = radius) + + + +class Zone(canConfigure): + """ + I am a (resource) zone. + A zone is a logical entry point to a collection of assets. + A zone allows for asset collections to be mounted at different paths on different sites, while retaining the same internal hierarchical structure. + For a given site, no two zones should have the same name (i.e. the name is relatively distinguishing with respect to the site) + """ + def __init__(self, site, name, root = None): + self.site = Site(site) + self.name = name + self.root = root if (root is None) else pathlib.Path(root) + self.policy = self.site.policy #--inherit my site's default policy + + @property + def RDN(self): + """ + I am the relatively distinguishing name for this object. + """ + return self.name @property def assets(self): - return iter(self._assets.values()) + return self.site.assets(self.name) + + def configured(self, conf): + if conf: + if entity(conf).isString: + #-- short form + self.root = pathlib.Path(conf) + else: + #-- long form + self.root = pathlib.Path(conf['root']) + if 'policy' in conf: + self.policy = RetentionPolicy(conf['policy']) + return self - def asset(self, k): - return self._assets[k] + def __div__(self, name): + return Asset(self, name) - def loadXLSX(self, confpath): - wb = openpyxl.load_workbook(wb = str(confpath)) - #-- Read site conf... - self.gatherSiteEnv(wb) - #-- Read assets... - self.gatherAssets(wb) - def gatherSiteVars(self): - ws = wb['site'] - raise NotImplementedError - def gatherAssets(self): - pass +class Asset(canTextify): + def __init__(self, zone, *args, **kwargs): + self.zone = zone + self.name = None + self.about = None + self._RDN = None + spec = args[0] + if isinstance(spec, str): + self.name = spec + elif isinstance(spec, CxNode): + self.configured(spec) + elif isinstance(spec, dict): + self.configured(spec) + else: + raise ValueError("Asset.__init__ - cannot initialize from object of type {}".format(type(spec))) -class CurationPolicy(Sable): - def __init__(self, name, path, **kwargs): - self.name = name - self.path = pathlib.Path(path) + if 'RDN' in kwargs: + self._RDN = kwargs['RDN'] + if 'about' in kwargs: + self.about = kwargs['about'] + + if self._RDN: + if len(self._RDN) != 8: + raise ValueError("Asset RDN must be exactly 8 characters.") + + if self.name is None: + raise Exception("Asset.__init__ - name of asset is not defined") + + def __str__(self): + return "{} ({})".format(self.badge, self.RDN) + + @property + def RDN(self): + """ + I am the relatively distinguishing name for this object. + In the case of assets, the RDN defaults to the "badge" of the asset. + In some cases, the operator may want to manually label an asset with an RDN. + Thus, an RDN can be given explicity as kwarg in object construction. + If no RDN is explicitly given, the asset's badge is used. + """ + if self._RDN is None: + self._RDN = self.badge + return self._RDN - self.RDN = kwargs.get('RDN', Slug40(name)) - self.asset = self.RDN - self.LOCKS = kwargs.get('LOCKS', 2) #-- Lots Of Copies Keep us Safe, minimum # of branches to retain - self.longevity = kwargs.get('longevity', datetime.timedelta(days = 30)) #-- maximum time before a new branch is forced - self.about = kwargs.get('about', "") + def _get_policy(self): + return getattr(self, '_policy', self.zone.policy) + + def _set_policy(self, p): + if isinstance(p, RetentionPolicy): + self._policy = p + else: + raise ValueError("Asset.policy - must be set to an instance of PolicyRetention") + + policy = property(_get_policy, _set_policy) + + @property + def path(self): + """ + I answer the local file system path to this asset. + """ + return self.zone.root / pathlib.Path(self.name) + + @property + def CURIE(self): + """ + My CURIE (Compact URI) form is [{site}:{zone}/{asset}] + """ + return CURIE(self.zone.site.name, str(pathlib.PurePath(self.zone.name) / self.name)) + + @property + def badge(self): + """ + I am a compact (40-bit) hash constructed from my CURIE (compact URI) + """ + return Slug40(str(self.CURIE)) def toJDN(self, **kwargs): jdn = { - '_type': "Bastion.Site.CurationPolicy", - 'RDN': self.RDN, - 'name': self.name, - 'LOCKS': self.LOCKS, - 'longevity': self.longevity.total_seconds() - 'path': str(self.path), - 'about': self.about + 'zone': "{}:{}".format(self.zone.site.name, self.zone.name), + 'path': str(self.path), } + if self.about: + jdn['about'] = self.about + jdn['RDN'] = self.RDN - @classmethod - def fromJDN(cls, jdn, **kwargs): - policy = cls(jdn['name'], jdn['path'], **jdn) + return jdn + + def configured(self, aconf): + #-- The name key is mandatory. + self.name = aconf['name'] + + #-- The rest of these keys are optional. + if 'policy' in aconf: + self.policy = RetentionPolicy(aconf['policy']) + if 'about' in aconf: + self.about = aconf['about'] + if 'RDN' in aconf: + self._RDN = aconf['RDN'] + + return self + + @property + def created(self): + return datetime.datetime.fromtimestamp( self.path.stat().st_stime ) + + @property + def modified(self): + return datetime.datetime.fromtimestamp( self.path.stat().st_mtime ) + + + +class AssetCatalog: + def __init__(self, context, *args): + if isinstance(context, Zone): + self.zone = context + self.site = context.site + elif isinstance(context, Site): + self.site = context + self.zone = self.site.zone(args[0]) + elif isinstance(context, str): + self.site = Site(context) + self.zone = self.site.zone(args[0]) + else: + raise Exception("AssetCatalog.__init__ - I don't know how to construct the requested AssetCatalog instance") + + self.xRDNs = { } + self.xnames = { } + + @property + def any(self): + k = random.choice( list(self.xRDNs.keys()) ) + return self.xRDNs[k] + + def add(self, obj): + if isinstance(obj, str): + asset = Asset(self.zone, obj) + elif isinstance(obj, pathlib.PurePath): + asset = Asset(self.zone, str(obj)) + elif isinstance(obj, Asset): + asset = obj + else: + raise ValueError("AssetCatalog.add - I don't know how to add an object of type {}".format(type(obj))) + + if asset.RDN not in self.xRDNs: + self.xRDNs[asset.RDN] = asset + self.xnames[asset.name] = asset + else: + raise Exception("duplicate asset RDN added!") + + def __contains__(self, slug): + return (slug in self.xRDNs) + + def update(self, asset): + self.xRDNs[asset.RDN] = asset + self.xnames[asset.name] = asset + + def named(self, name): + return self.xnames[name] + + def __getitem__(self, slug): + return self.xRDNs[slug] + def __iter__(self): + return iter(sorted(self.xRDNs.values(), key = lambda x: str(x.path))) diff --git a/lib/Bastion/Vault.py b/lib/Bastion/Vault.py deleted file mode 100644 index 4a99469..0000000 --- a/lib/Bastion/Vault.py +++ /dev/null @@ -1,33 +0,0 @@ -""" -Bastion.Vault -""" -class isClerk: - """ - I am an abstract type for "clerk" objects that do data management - in the context of a vault. - """ - @property - def snaps(self): - raise NotImplementedError - - @property - def branches(self): - raise NotImplementedError - - - -class isVault: - """ - I am an abstract base type for specialized Vault classes. - """ - def __getitem__(self, asset): - raise NotImplementedError - - @property - def assets(self): - raise NotImplementedError - - def put(self, asset, latest = None): - pass - - def diff --git a/lib/Bastion/Vaults/Common.py b/lib/Bastion/Vaults/Common.py new file mode 100644 index 0000000..e451cbc --- /dev/null +++ b/lib/Bastion/Vaults/Common.py @@ -0,0 +1,76 @@ +""" +Bastion.Vault.Common +""" +from Curator import Archive, Branch, Snap + +class isClerk: + """ + I am an abstract type for "clerk" objects that do data management in the context of a vault. + """ + @property + def sites(self): + raise NotImplementedError + + @property + def zones(self): + raise NotImplementedError + + @property + def snaps(self): + raise NotImplementedError + + @property + def branches(self): + raise NotImplementedError + + @property + def archives(self): + raise NotImplementedError + + +class isSiteClerk: + def __init__(self, site): + self.site = site + + def zones(self): + raise NotImplementedError + + +class isZoneClerk: + def __init__(self, site, zone): + self.site = site + self.zone = zone + + +class isArchiveClerk: + def __init__(self, site, zone, archive): + self.site = site + self.zone = zone + self.resource = archive + + +class isBranchClerk: + def __init__(self, site, zone, archive, branch): + self.site = site + self.zone = zone + self.resource = archive + self.branch = branch + + @property + def snaps(self): + raise NotImplementedError + +class isVault: + """ + I am an abstract base type for specialized Vault classes. + """ + def __getitem__(self, asset): + raise NotImplementedError + + @property + def assets(self): + raise NotImplementedError + + def put(self, asset, latest = None): + pass + diff --git a/lib/Bastion/Vaults/Local.py b/lib/Bastion/Vaults/Local.py new file mode 100644 index 0000000..ac8850e --- /dev/null +++ b/lib/Bastion/Vaults/Local.py @@ -0,0 +1,8 @@ +""" +Bastion.Vault.Local + +I am a vault for storing backups to a local file system. +""" + +from .Common import * + diff --git a/lib/Bastion/Vaults/__init__.py b/lib/Bastion/Vaults/__init__.py new file mode 100644 index 0000000..e69de29