initial import

COAG-Research-Services · Dec 13, 2023 · b7009b2 · b7009b2
1 parent 1186089
commit b7009b2
Show file tree

Hide file tree

Showing 4 changed files with 390 additions and 0 deletions.
diff --git a/bin/HPSS.py b/bin/HPSS.py
@@ -0,0 +1,210 @@
+import os
+import pathlib
+import subprocess
+import operator
+import datetime
+import json
+
+
+
+
+def asDateTime(x):
+	if (x is None):
+		return x
+	if isinstance(x, datetime.datetime):
+		return x
+	if isinstance(x, (int, float)):
+		return datetime.datetime.fromtimestamp(x)
+
+	raise ValueError("cannot interpret value of type {} as instance of datetime.datetime".format( str(type(x)) ))
+
+
+
+class fstat(tuple):
+	def __new__(cls, path, size, **kwargs):
+		obj = { }
+		obj['path']  = pathlib.Path(path)
+		obj['size']  = int(size)
+
+		for kwarg in ('perms', 'owner', 'group', 'stored', 'xtype'):
+			obj[kwarg] = kwargs.get(kwarg, None)
+
+		obj['links'] = None if ('links' not in kwargs) else int(kwargs['links'])
+
+		for dtarg in ('created', 'modified'):
+			obj[dtarg] = None if (dtarg not in kwargs) else asDateTime(kwargs[dtarg])
+
+		s = [obj[tag] for tag in ('path', 'size', 'perms', 'links', 'owner', 'group', 'stored', 'created', 'modified', 'xtype')]
+
+		return tuple.__new__(cls, s)
+
+	path     = property(operator.itemgetter(0))
+	size     = property(operator.itemgetter(1))
+	perms    = property(operator.itemgetter(2))
+	links    = property(operator.itemgetter(3))
+	owner    = property(operator.itemgetter(4))
+	group    = property(operator.itemgetter(5))
+	stored   = property(operator.itemgetter(6))
+	created  = property(operator.itemgetter(7))
+	modified = property(operator.itemgetter(8))
+	xtype    = property(operator.itemgetter(9))
+
+	def __add__(self, other):
+		me  = self.toJDN()
+		you = other.toJDN()
+		me.update(you)
+		return fstat.fromJDN( me )
+
+	@property
+	def parent(self):
+		return self.path.parent
+
+	@property
+	def folder(self):
+		return self.path.parent
+
+	def isdir(self):
+		return (self.perms[0] == 'd')
+
+	def isfile(self):
+		return (self.perms[0] != 'd')
+
+	@property
+	def name(self):
+		return self.path.name
+
+	def toJSON(self, **kwargs):
+		return json.dumps( self.toJDN(**kwargs), sort_keys = True, indent = 3 )
+
+	def toJDN(self, **kwargs):
+		jdn = {
+			'path': str(self.path),
+			'size': self.size
+		}
+		for attr in ('perms', 'links', 'owner', 'group', 'stored', 'created', 'modified'):
+			jdn[attr] = getattr(self, attr)
+		return jdn
+
+	@classmethod
+	def fromJDN(cls, jdn):
+		path = pathlib.Path(jdn['path'])
+		size = int(jdn['size'])	
+		kwargs = { }
+		for attr in ('perms', 'links', 'owner', 'group', 'stored', 'created', 'modified'):
+			if attr in jdn:
+				kwargs[attr] = jdn[attr]
+		return cls(path, size, **kwargs)
+
+
+
+
+class HSI:
+	def __init__(self, xpath = None, **kwargs):
+		self.xpath = pathlib.Path("/usr/local/bin/hsi")
+		self.login = os.getlogin()
+
+		if xpath is not None:
+			self.xpath = pathlib.Path(xpath)
+
+		if 'login' in kwargs:
+			self.login = kwargs['login']
+
+
+	def do(self, command):
+		comargs = [str(self.xpath), "-l", "ndenny", "-q", "-P", "{}".format(command)]
+		proc = subprocess.run(comargs, capture_output = True, check = True)
+		return proc
+
+	def stat(self, target):
+		"""
+		Answers a slightly different set of stats for the given path.
+		"""
+		if isinstance(target, pathlib.Path):
+			return self.stat( self.ls(target) )
+		elif isinstance(target, str):
+			return self.stat( pathlib.Path(target) )
+		elif isinstance(target, fstat):
+			proc   = self.do("ls -d -P {}".format( str(path) ))
+			lines  = [line.strip() for line in proc.stdout.decode('utf-8').split('\n')]
+			line   = lines[0]
+			tokens = [token.strip() for token in line.split()]
+			size   = 0
+
+			kwargs = {
+				'xtype': tokens[0]
+			}
+
+			if kwargs['xtype'] in ('FILE', 'HARDLINK'):
+				size          = int(tokens[2])
+				created_mdy   = tokens[9]
+				created_time  = tokens[10]
+
+				modified_mdy  = tokens[11]
+				modified_time = tokens[12]
+
+				kwargs['created']  = datetime.datetime.strptime("{} {}".format(created_mdy, created_time), "%m/%d/%Y %H:%M:%S")
+				kwargs['modified'] = datetime.datetime.strptime("{} {}".format(modified_mdy, modified_time), "%m/%d/%Y %H:%M:%S")
+
+			return fstat(path, size, **kwargs)
+
+	def ls(self, path = None):
+		"""
+		Answers a list of stat-like objects for entries at the given path.
+		"""
+		if path:
+			proc = self.do("ls -lUD {}".format( str(path) ))
+		else:
+			proc = self.do("ls -lUD")
+
+		entries = [ ]
+
+		lines = [line.strip() for line in proc.stdout.decode('utf-8').split('\n')]
+		lines = [line for line in lines if len(line) > 0]
+		folder = pathlib.Path(lines[0][:-1])
+		for line in lines[1:]:
+			tokens = [token.strip( ) for token in line.split()]
+
+			perms  = tokens[0]
+			links  = int(tokens[1])
+			owner  = tokens[2]
+			group  = tokens[3]
+			name   = tokens[-1]
+
+			#for i, token in enumerate(tokens):
+			#	print("{:02d}: {}".format(i, token))
+			if len(tokens) == 14:
+				#-- this is a normal file
+				stored = tokens[6]
+				size   = int(tokens[7])
+				dow    = tokens[8]
+				month3 = tokens[9]
+				dom    = tokens[10]
+				hms    = tokens[11]
+				year4  = tokens[12]
+
+			elif len(tokens) == 12:
+				#-- this is likely a folder.
+				stored = None
+				size   = int(tokens[5])
+				dow    = tokens[6]
+				month3 = tokens[7]
+				dom    = tokens[8]
+				hms    = tokens[9]
+				year4  = tokens[10]
+
+			dts      = " ".join([dow, month3, dom, hms, year4])
+			modified = datetime.datetime.strptime(dts, "%c")
+
+			p      = folder / name
+			basic  = fstat(p, size, perms = perms, owner = owner, group = group, stored = stored, links = links, modified = modified)
+			detail = self.stat(p)
+			entry  = detail + basic
+
+
+			entries.append( entry )
+
+		return (proc, entries)
+
+
+
+hsi = HSI("/opt/hsi/bin/hsi", login = "ndenny")
diff --git a/bin/fossil.py b/bin/fossil.py
@@ -0,0 +1,28 @@
+"""
+fossil - archive a data set with high assurance using HPSS tools.
+
+e.g.
+fossilize QL2_3DEP_LiDAR_IN_2011_2013_l2 
+"""
+
+HSI  = "/opt/hsi/bin/hsi"
+HTAR = "/opt/his/bin/htar"
+
+
+
+def backup(folder, level = 1, **kwargs):
+	"""
+	Given a folder, I do a backup of the folder using htar.
+
+	backup("/home/ndenny/lidar.digitalforestry.org/QL2_3DEP_LiDAR_IN_2011_2013_l2")
+	""" 
+	raise NotImplementedError
+
+
+def backup_full(folder, **kwargs):
+	return backup(folder, 0, **kwargs)
+
+
+def backup_differential(folder):
+	return backup(folder, 1, **kwargs)
+
diff --git a/bin/out.txt b/bin/out.txt
@@ -0,0 +1,23 @@
+/home/ndenny:
+drwxr-xr-x    2 ndenny    student               253552                512 Wed Jul 19 15:28:56 2023 CfGS
+-rw-------    1 ndenny    student      10       253552 DISK         12800 Thu Dec  7 06:33:23 2023 miniconda_0231207.tar
+-rw-------    1 ndenny    student      10       253552 DISK          1312 Thu Dec  7 06:33:24 2023 miniconda_0231207.tar.idx
+-rw-------    1 ndenny    itap         11       253552 TAPE   2307115900416 Tue Feb  9 10:19:17 2021 nexus_20210209.tar
+-rw-------    1 ndenny    itap         11       253552 TAPE     414114080 Tue Feb  9 10:19:22 2021 nexus_20210209.tar.idx
+-rw-------    1 ndenny    itap         11       253552 TAPE   2837237811712 Fri Nov 19 21:03:24 2021 nexus_20211119.tar
+-rw-------    1 ndenny    itap         11       253552 TAPE     417398048 Fri Nov 19 21:03:30 2021 nexus_20211119.tar.idx
+-rw-------    1 ndenny    itap         11       253552 TAPE   2016443183104 Thu May 19 16:15:49 2022 nexus_catalog_2022-05-19.tar
+-rw-------    1 ndenny    itap         11       253552 TAPE     196347680 Thu May 19 16:15:51 2022 nexus_catalog_2022-05-19.tar.idx
+-rw-------    1 ndenny    itap         11       253552 TAPE   18800947200 Mon Feb 14 15:24:16 2022 nexus_final_report_2022-02-14.tar
+-rw-------    1 ndenny    itap         10       253552 DISK       1675040 Mon Feb 14 15:24:16 2022 nexus_final_report_2022-02-14.tar.idx
+-rw-------    1 ndenny    itap         11       253552 TAPE   1634327330816 Mon Feb 14 17:55:47 2022 nexus_projects_2022-02-14.tar
+-rw-------    1 ndenny    itap         11       253552 TAPE   1659624728576 Thu May 19 12:52:50 2022 nexus_projects_2022-05-19.tar
+-rw-------    1 ndenny    itap         11       253552 TAPE     492911904 Thu May 19 12:52:57 2022 nexus_projects_2022-05-19.tar.idx
+-rw-------    1 ndenny    student      11       253552 DISK   1569479696896 Thu Dec  7 09:18:17 2023 QL2_3DEP_LiDAR_IN_2011_2013_l2.tar
+-rw-------    1 ndenny    student      10       253552 DISK      71481632 Thu Dec  7 10:03:45 2023 QL2_3DEP_LiDAR_IN_2011_2013_l2.tar.idx
+-rw-------    1 ndenny    student      11       253552 DISK   2323521421824 Mon Dec 11 03:08:35 2023 QL2_3DEP_LiDAR_IN_2017_2019_l2.tar
+-rw-------    1 ndenny    student      11       253552 DISK     124447008 Mon Dec 11 04:05:42 2023 QL2_3DEP_LiDAR_IN_2017_2019_l2.tar.idx
+-rw-------    1 ndenny    student      11       253552 DISK   1575000025088 Mon Dec 11 05:53:04 2023 QL2_3DEP_LiDAR_IN_2017_2019_laz.tar
+-rw-------    1 ndenny    student      10       253552 DISK      22099744 Mon Dec 11 06:32:41 2023 QL2_3DEP_LiDAR_IN_2017_2019_laz.tar.idx
+-rw-------    1 ndenny    student      11       253552 DISK   8517799739904 Mon Dec 11 11:38:13 2023 QLX_3DEP_LiDAR_US_South.tar
+drwxr-x---    2 ndenny    itap                  253552                512 Thu Jul 16 05:51:12 2020 QUDT
diff --git a/lab/webhookx3.py b/lab/webhookx3.py
@@ -0,0 +1,129 @@
+import sys
+import socket
+import json
+import urllib.request
+
+WEB_HOOK  = "https://purdue0.webhook.office.com/webhookb2/992d9034-7971-4c00-9894-60210e6044d6@4130bd39-7c53-419c-b1e5-8758d6d63f21/IncomingWebhook/fdb087d3f75a4c668258d8b546470b39/ee3bcfc7-20f8-4e83-acdc-91828dd8722e"
+THIS_HOST = socket.gethostname()
+
+#card = {
+#	"text": "Hello, this is a message from {} posted via MS Webhook".format(THIS_HOST)
+#}
+
+
+failed = {
+    "@type": "MessageCard",
+    "@context": "http://schema.org/extensions",
+    "themeColor": "0076D7",
+    "summary": "IDIF hub backup failed",
+    "sections": [{
+        "activityTitle": "FAILURE: Backup Failed",
+        "activitySubtitle": "idifhub.ecn.purdue.edu",
+        "activityImage": "https://raw.github.itap.purdue.edu/ndenny/bastion-cdn/master/red_X.png",
+        "facts": [{
+            "name": "dataset",
+            "value": "QLX_3DEP_LiDAR_US_South"
+        }, {
+            "name": "message",
+            "value": "CRC checksum failed"
+        }, {
+            "name": "Status",
+            "value": "FAILED"
+        }],
+        "markdown": True
+    }],
+    "potentialAction": [
+	{
+        "@type": "OpenUri",
+        "name": "Read log file",
+        "targets": [{
+            "os": "default",
+            "uri": "https://github.itap.purdue.edu/ndenny/bastion-cdn"
+        }]
+    }
+    ]
+}
+
+
+completed = {
+    "@type": "MessageCard",
+    "@context": "http://schema.org/extensions",
+    "themeColor": "0076D7",
+    "summary": "IDIF hub backup succeeded",
+    "sections": [{
+        "activityTitle": "SUCCESS: Backup Succeeded",
+        "activitySubtitle": "idifhub.ecn.purdue.edu",
+        "activityImage": "https://raw.github.itap.purdue.edu/ndenny/bastion-cdn/master/green_check.png",
+        "facts": [{
+            "name": "dataset",
+            "value": "QL2_3DEP_LiDAR_IN_2017_2019_laz"
+        },  
+        {
+            "name": "Status",
+            "value": "SUCCESS"
+        }],
+        "markdown": True
+    }],
+    "potentialAction": [
+	{
+        "@type": "OpenUri",
+        "name": "Read log file",
+        "targets": [{
+            "os": "default",
+            "uri": "https://github.itap.purdue.edu/ndenny/bastion-cdn"
+        }]
+    }
+    ]
+}
+
+
+quota = {
+    "@type": "MessageCard",
+    "@context": "http://schema.org/extensions",
+    "themeColor": "0076D7",
+    "summary": "IDIF hub inventory alert",
+    "sections": [{
+        "activityTitle": "ALERT: Storage Quota",
+        "activitySubtitle": "idifhub.ecn.purdue.edu",
+        "activityImage": "https://raw.github.itap.purdue.edu/ndenny/bastion-cdn/master/warning.png",
+        "text": "Assets on Fortress are approaching warning threshold of 100 TiBs",
+        "markdown": True
+    }],
+    "potentialAction": [
+	{
+        "@type": "OpenUri",
+        "name": "Read log file",
+        "targets": [{
+            "os": "default",
+            "uri": "https://github.itap.purdue.edu/ndenny/bastion-cdn"
+        }]
+    }
+    ]
+}
+
+
+
+#    "actions": [
+#		 {
+#			"type": "Action.OpenUrl",
+#			"title": "View Log",
+#			"url": "https://itap.github.purdue.edu"
+#		 }
+#    ]
+
+
+opts = {
+	'quota': quota,
+	'failed': failed,
+	'completed': completed
+}
+
+card = opts[sys.argv[1]]
+
+payload = json.dumps(card).encode('utf-8')
+
+
+request = urllib.request.Request(WEB_HOOK, payload)
+request.add_header('Content-Type', 'application/json')
+
+conn = urllib.request.urlopen(request)