diff --git a/bin/HPSS.py b/bin/HPSS.py new file mode 100644 index 0000000..30ec84e --- /dev/null +++ b/bin/HPSS.py @@ -0,0 +1,210 @@ +import os +import pathlib +import subprocess +import operator +import datetime +import json + + + + +def asDateTime(x): + if (x is None): + return x + if isinstance(x, datetime.datetime): + return x + if isinstance(x, (int, float)): + return datetime.datetime.fromtimestamp(x) + + raise ValueError("cannot interpret value of type {} as instance of datetime.datetime".format( str(type(x)) )) + + + +class fstat(tuple): + def __new__(cls, path, size, **kwargs): + obj = { } + obj['path'] = pathlib.Path(path) + obj['size'] = int(size) + + for kwarg in ('perms', 'owner', 'group', 'stored', 'xtype'): + obj[kwarg] = kwargs.get(kwarg, None) + + obj['links'] = None if ('links' not in kwargs) else int(kwargs['links']) + + for dtarg in ('created', 'modified'): + obj[dtarg] = None if (dtarg not in kwargs) else asDateTime(kwargs[dtarg]) + + s = [obj[tag] for tag in ('path', 'size', 'perms', 'links', 'owner', 'group', 'stored', 'created', 'modified', 'xtype')] + + return tuple.__new__(cls, s) + + path = property(operator.itemgetter(0)) + size = property(operator.itemgetter(1)) + perms = property(operator.itemgetter(2)) + links = property(operator.itemgetter(3)) + owner = property(operator.itemgetter(4)) + group = property(operator.itemgetter(5)) + stored = property(operator.itemgetter(6)) + created = property(operator.itemgetter(7)) + modified = property(operator.itemgetter(8)) + xtype = property(operator.itemgetter(9)) + + def __add__(self, other): + me = self.toJDN() + you = other.toJDN() + me.update(you) + return fstat.fromJDN( me ) + + @property + def parent(self): + return self.path.parent + + @property + def folder(self): + return self.path.parent + + def isdir(self): + return (self.perms[0] == 'd') + + def isfile(self): + return (self.perms[0] != 'd') + + @property + def name(self): + return self.path.name + + def toJSON(self, **kwargs): + return json.dumps( self.toJDN(**kwargs), sort_keys = True, indent = 3 ) + + def toJDN(self, **kwargs): + jdn = { + 'path': str(self.path), + 'size': self.size + } + for attr in ('perms', 'links', 'owner', 'group', 'stored', 'created', 'modified'): + jdn[attr] = getattr(self, attr) + return jdn + + @classmethod + def fromJDN(cls, jdn): + path = pathlib.Path(jdn['path']) + size = int(jdn['size']) + kwargs = { } + for attr in ('perms', 'links', 'owner', 'group', 'stored', 'created', 'modified'): + if attr in jdn: + kwargs[attr] = jdn[attr] + return cls(path, size, **kwargs) + + + + +class HSI: + def __init__(self, xpath = None, **kwargs): + self.xpath = pathlib.Path("/usr/local/bin/hsi") + self.login = os.getlogin() + + if xpath is not None: + self.xpath = pathlib.Path(xpath) + + if 'login' in kwargs: + self.login = kwargs['login'] + + + def do(self, command): + comargs = [str(self.xpath), "-l", "ndenny", "-q", "-P", "{}".format(command)] + proc = subprocess.run(comargs, capture_output = True, check = True) + return proc + + def stat(self, target): + """ + Answers a slightly different set of stats for the given path. + """ + if isinstance(target, pathlib.Path): + return self.stat( self.ls(target) ) + elif isinstance(target, str): + return self.stat( pathlib.Path(target) ) + elif isinstance(target, fstat): + proc = self.do("ls -d -P {}".format( str(path) )) + lines = [line.strip() for line in proc.stdout.decode('utf-8').split('\n')] + line = lines[0] + tokens = [token.strip() for token in line.split()] + size = 0 + + kwargs = { + 'xtype': tokens[0] + } + + if kwargs['xtype'] in ('FILE', 'HARDLINK'): + size = int(tokens[2]) + created_mdy = tokens[9] + created_time = tokens[10] + + modified_mdy = tokens[11] + modified_time = tokens[12] + + kwargs['created'] = datetime.datetime.strptime("{} {}".format(created_mdy, created_time), "%m/%d/%Y %H:%M:%S") + kwargs['modified'] = datetime.datetime.strptime("{} {}".format(modified_mdy, modified_time), "%m/%d/%Y %H:%M:%S") + + return fstat(path, size, **kwargs) + + def ls(self, path = None): + """ + Answers a list of stat-like objects for entries at the given path. + """ + if path: + proc = self.do("ls -lUD {}".format( str(path) )) + else: + proc = self.do("ls -lUD") + + entries = [ ] + + lines = [line.strip() for line in proc.stdout.decode('utf-8').split('\n')] + lines = [line for line in lines if len(line) > 0] + folder = pathlib.Path(lines[0][:-1]) + for line in lines[1:]: + tokens = [token.strip( ) for token in line.split()] + + perms = tokens[0] + links = int(tokens[1]) + owner = tokens[2] + group = tokens[3] + name = tokens[-1] + + #for i, token in enumerate(tokens): + # print("{:02d}: {}".format(i, token)) + if len(tokens) == 14: + #-- this is a normal file + stored = tokens[6] + size = int(tokens[7]) + dow = tokens[8] + month3 = tokens[9] + dom = tokens[10] + hms = tokens[11] + year4 = tokens[12] + + elif len(tokens) == 12: + #-- this is likely a folder. + stored = None + size = int(tokens[5]) + dow = tokens[6] + month3 = tokens[7] + dom = tokens[8] + hms = tokens[9] + year4 = tokens[10] + + dts = " ".join([dow, month3, dom, hms, year4]) + modified = datetime.datetime.strptime(dts, "%c") + + p = folder / name + basic = fstat(p, size, perms = perms, owner = owner, group = group, stored = stored, links = links, modified = modified) + detail = self.stat(p) + entry = detail + basic + + + entries.append( entry ) + + return (proc, entries) + + + +hsi = HSI("/opt/hsi/bin/hsi", login = "ndenny") diff --git a/bin/fossil.py b/bin/fossil.py new file mode 100644 index 0000000..2d67aac --- /dev/null +++ b/bin/fossil.py @@ -0,0 +1,28 @@ +""" +fossil - archive a data set with high assurance using HPSS tools. + +e.g. +fossilize QL2_3DEP_LiDAR_IN_2011_2013_l2 +""" + +HSI = "/opt/hsi/bin/hsi" +HTAR = "/opt/his/bin/htar" + + + +def backup(folder, level = 1, **kwargs): + """ + Given a folder, I do a backup of the folder using htar. + + backup("/home/ndenny/lidar.digitalforestry.org/QL2_3DEP_LiDAR_IN_2011_2013_l2") + """ + raise NotImplementedError + + +def backup_full(folder, **kwargs): + return backup(folder, 0, **kwargs) + + +def backup_differential(folder): + return backup(folder, 1, **kwargs) + diff --git a/bin/out.txt b/bin/out.txt new file mode 100644 index 0000000..395339f --- /dev/null +++ b/bin/out.txt @@ -0,0 +1,23 @@ +/home/ndenny: +drwxr-xr-x 2 ndenny student 253552 512 Wed Jul 19 15:28:56 2023 CfGS +-rw------- 1 ndenny student 10 253552 DISK 12800 Thu Dec 7 06:33:23 2023 miniconda_0231207.tar +-rw------- 1 ndenny student 10 253552 DISK 1312 Thu Dec 7 06:33:24 2023 miniconda_0231207.tar.idx +-rw------- 1 ndenny itap 11 253552 TAPE 2307115900416 Tue Feb 9 10:19:17 2021 nexus_20210209.tar +-rw------- 1 ndenny itap 11 253552 TAPE 414114080 Tue Feb 9 10:19:22 2021 nexus_20210209.tar.idx +-rw------- 1 ndenny itap 11 253552 TAPE 2837237811712 Fri Nov 19 21:03:24 2021 nexus_20211119.tar +-rw------- 1 ndenny itap 11 253552 TAPE 417398048 Fri Nov 19 21:03:30 2021 nexus_20211119.tar.idx +-rw------- 1 ndenny itap 11 253552 TAPE 2016443183104 Thu May 19 16:15:49 2022 nexus_catalog_2022-05-19.tar +-rw------- 1 ndenny itap 11 253552 TAPE 196347680 Thu May 19 16:15:51 2022 nexus_catalog_2022-05-19.tar.idx +-rw------- 1 ndenny itap 11 253552 TAPE 18800947200 Mon Feb 14 15:24:16 2022 nexus_final_report_2022-02-14.tar +-rw------- 1 ndenny itap 10 253552 DISK 1675040 Mon Feb 14 15:24:16 2022 nexus_final_report_2022-02-14.tar.idx +-rw------- 1 ndenny itap 11 253552 TAPE 1634327330816 Mon Feb 14 17:55:47 2022 nexus_projects_2022-02-14.tar +-rw------- 1 ndenny itap 11 253552 TAPE 1659624728576 Thu May 19 12:52:50 2022 nexus_projects_2022-05-19.tar +-rw------- 1 ndenny itap 11 253552 TAPE 492911904 Thu May 19 12:52:57 2022 nexus_projects_2022-05-19.tar.idx +-rw------- 1 ndenny student 11 253552 DISK 1569479696896 Thu Dec 7 09:18:17 2023 QL2_3DEP_LiDAR_IN_2011_2013_l2.tar +-rw------- 1 ndenny student 10 253552 DISK 71481632 Thu Dec 7 10:03:45 2023 QL2_3DEP_LiDAR_IN_2011_2013_l2.tar.idx +-rw------- 1 ndenny student 11 253552 DISK 2323521421824 Mon Dec 11 03:08:35 2023 QL2_3DEP_LiDAR_IN_2017_2019_l2.tar +-rw------- 1 ndenny student 11 253552 DISK 124447008 Mon Dec 11 04:05:42 2023 QL2_3DEP_LiDAR_IN_2017_2019_l2.tar.idx +-rw------- 1 ndenny student 11 253552 DISK 1575000025088 Mon Dec 11 05:53:04 2023 QL2_3DEP_LiDAR_IN_2017_2019_laz.tar +-rw------- 1 ndenny student 10 253552 DISK 22099744 Mon Dec 11 06:32:41 2023 QL2_3DEP_LiDAR_IN_2017_2019_laz.tar.idx +-rw------- 1 ndenny student 11 253552 DISK 8517799739904 Mon Dec 11 11:38:13 2023 QLX_3DEP_LiDAR_US_South.tar +drwxr-x--- 2 ndenny itap 253552 512 Thu Jul 16 05:51:12 2020 QUDT diff --git a/lab/webhookx3.py b/lab/webhookx3.py new file mode 100644 index 0000000..c1f47e4 --- /dev/null +++ b/lab/webhookx3.py @@ -0,0 +1,129 @@ +import sys +import socket +import json +import urllib.request + +WEB_HOOK = "https://purdue0.webhook.office.com/webhookb2/992d9034-7971-4c00-9894-60210e6044d6@4130bd39-7c53-419c-b1e5-8758d6d63f21/IncomingWebhook/fdb087d3f75a4c668258d8b546470b39/ee3bcfc7-20f8-4e83-acdc-91828dd8722e" +THIS_HOST = socket.gethostname() + +#card = { +# "text": "Hello, this is a message from {} posted via MS Webhook".format(THIS_HOST) +#} + + +failed = { + "@type": "MessageCard", + "@context": "http://schema.org/extensions", + "themeColor": "0076D7", + "summary": "IDIF hub backup failed", + "sections": [{ + "activityTitle": "FAILURE: Backup Failed", + "activitySubtitle": "idifhub.ecn.purdue.edu", + "activityImage": "https://raw.github.itap.purdue.edu/ndenny/bastion-cdn/master/red_X.png", + "facts": [{ + "name": "dataset", + "value": "QLX_3DEP_LiDAR_US_South" + }, { + "name": "message", + "value": "CRC checksum failed" + }, { + "name": "Status", + "value": "FAILED" + }], + "markdown": True + }], + "potentialAction": [ + { + "@type": "OpenUri", + "name": "Read log file", + "targets": [{ + "os": "default", + "uri": "https://github.itap.purdue.edu/ndenny/bastion-cdn" + }] + } + ] +} + + +completed = { + "@type": "MessageCard", + "@context": "http://schema.org/extensions", + "themeColor": "0076D7", + "summary": "IDIF hub backup succeeded", + "sections": [{ + "activityTitle": "SUCCESS: Backup Succeeded", + "activitySubtitle": "idifhub.ecn.purdue.edu", + "activityImage": "https://raw.github.itap.purdue.edu/ndenny/bastion-cdn/master/green_check.png", + "facts": [{ + "name": "dataset", + "value": "QL2_3DEP_LiDAR_IN_2017_2019_laz" + }, + { + "name": "Status", + "value": "SUCCESS" + }], + "markdown": True + }], + "potentialAction": [ + { + "@type": "OpenUri", + "name": "Read log file", + "targets": [{ + "os": "default", + "uri": "https://github.itap.purdue.edu/ndenny/bastion-cdn" + }] + } + ] +} + + +quota = { + "@type": "MessageCard", + "@context": "http://schema.org/extensions", + "themeColor": "0076D7", + "summary": "IDIF hub inventory alert", + "sections": [{ + "activityTitle": "ALERT: Storage Quota", + "activitySubtitle": "idifhub.ecn.purdue.edu", + "activityImage": "https://raw.github.itap.purdue.edu/ndenny/bastion-cdn/master/warning.png", + "text": "Assets on Fortress are approaching warning threshold of 100 TiBs", + "markdown": True + }], + "potentialAction": [ + { + "@type": "OpenUri", + "name": "Read log file", + "targets": [{ + "os": "default", + "uri": "https://github.itap.purdue.edu/ndenny/bastion-cdn" + }] + } + ] +} + + + +# "actions": [ +# { +# "type": "Action.OpenUrl", +# "title": "View Log", +# "url": "https://itap.github.purdue.edu" +# } +# ] + + +opts = { + 'quota': quota, + 'failed': failed, + 'completed': completed +} + +card = opts[sys.argv[1]] + +payload = json.dumps(card).encode('utf-8') + + +request = urllib.request.Request(WEB_HOOK, payload) +request.add_header('Content-Type', 'application/json') + +conn = urllib.request.urlopen(request)