debugging SCURLER

lab/depot_example.py

@@ -0,0 +1,38 @@
+import pathlib
+import getpass
+import os
+import sys
+import logging
+
+from Bastion.NetOps.sCURL import SCURLer
+
+logger = logging.getLogger(__name__)
+logging.basicConfig(level = logging.DEBUG)
+
+SSHROOT = pathlib.Path("~/.ssh").expanduser()
+
+if len(sys.argv) > 1:
+	USER = sys.argv[1]
+else:
+	USER    = getpass.getuser()
+
+for candidate in ['bastion_bot', 'id_ecdsa', 'id_rsa']:
+	keypath = SSHROOT / candidate
+	if keypath.exists():
+		logger.debug("selected key at {} for connection".format(str(keypath)))
+		KEYPATH = keypath
+		break
+
+LOCAL   = pathlib.Path("/mnt/BFD/bastion/bank")
+REMOTE  = pathlib.PurePosixPath("/depot/agdata/data/bastion")
+
+SITE    = "rusina"
+ZONE    = "soundscapes"
+ASSET   = "HackathonData"
+BLONDE  = "3AQXEGFS024A03CMFZMT.tar"
+
+depot    = SCURLer('data.rcac.purdue.edu', '/depot', user = USER, keyfile = KEYPATH, silent = False, verbose = True)
+agdata   = depot / 'agdata' / 'data'
+bastion  = agdata / 'bastion'
+
+

lib/Bastion/Clerks/BFD.py

@@ -0,0 +1,107 @@
+import os
+import pathlib
+import subprocess
+import operator
+import datetime
+import json
+import socket
+import logging
+import getpass
+import shutil
+import tarfile
+import logging
+
+from Bastion.Common import Thing, Unknown
+from Bastion.Model import isAsset, isClerk
+from Bastion.Curator import Manifest, BLONDE, Snap
+from Bastion.Utils   import pax
+
+
+logger = logging.getLogger(__name__)
+
+class Clerk(isClerk):
+	def __init__(self, vault, **kwargs):
+		isClerk.__init__(self)
+		self.vault   = vault
+
+	@property
+	def bfd(self):
+		return self.vault.bfd
+
+	@property
+	def bank(self):
+		return self.vault.bank
+
+	@property
+	def scratch(self):
+		return self.vault.scratch
+
+#-----------------------------------------
+#-- BEGIN Bastion.Model.isClerk PROTOCOL |
+#↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓
+	@property
+	def sites(self):
+		"""
+		I answer a sorted collection of the names of all sites known to this vault.
+		"""
+		#-- sites are top level elements relative to the root of the vault.
+		sites = [ ]
+		for entry in self.root.iterdir():
+			if entry.is_dir():
+				sites.append(entry.name)
+
+		return tuple(sorted(sites))
+
+	def zones(self, site):
+		"""
+		Given a site name, I answer a sorted collection of zone names that are known to this vault.
+		"""
+		#-- a zone will be a subdirectory (subfolder) of the given site.
+		#-- look for all of the subfolders of root / site
+		zones = [ ]
+		sroot = self.root / RDN(site)
+		if sroot.exists( ):
+			for entry in sroot.iterdir( ):
+				if entry.is_dir():
+					zones.append(entry.name)
+		return tuple(sorted(zones))
+
+	def assets(self, site, zone):
+		#-- assets will be subdirectories (subfolders) of a given site, zone.
+		assets = [ ]
+		zroot = self.root / site / zone
+		if zroot.exists( ):
+			for entry in zroot.is_dir():
+				assets.append(entry.name)
+		return tuple(sorted(assets))
+
+	def manifest(self, *args):
+		"""
+		I answer a manifest of the named asset...
+		manifest(ark)
+		manifest(site, zone, asset)
+		"""
+		ark = None
+		if len(args) == 1:
+			arg = args[0]
+			if isinstance(arg, Bastion.Model.isAsset):
+				ark = arg.ARK
+			else:
+				ark = arg
+		elif len(args) == 3:
+			ark = ARK(site, zone, asset)
+		else:
+			raise ValueError
+
+		cell = self.bank / ark.site / ark.zone / ark.asset
+		if cell.exists():
+			blondes = [ ]
+			for item in cell.iterdir():
+				if not item.is_dir():
+					blondes.append( BLONDE.decode(item.stem) )
+		manifest = Bastion.Curator.Manifest(ark, blondes)
+
+		return manifest
+#↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑
+#-- END Bastion.Model.isClerk PROTOCOL |
+#---------------------------------------

lib/Bastion/Clerks/SFTP.py

@@ -0,0 +1,98 @@
+import os
+import pathlib
+import subprocess
+import operator
+import datetime
+import json
+import socket
+import logging
+import getpass
+import shutil
+import tarfile
+import logging
+
+from Bastion.Common       import Thing, Unknown
+from Bastion.Model        import isAsset, isClerk
+from Bastion.Curator      import Manifest, BLONDE, Snap
+from Bastion.Packers.TARs import pax
+from Bastion.NetOps.sCURL import SCURLer
+
+
+logger = logging.getLogger(__name__)
+
+
+class Clerk(isClerk):
+	def __init__(self, vault, **kwargs):
+		isClerk.__init__(self)
+		self.vault   = vault
+		self.scurler = SCURLer(self.vault.sfURL, keyfile = self.vault.keypath)
+
+#-----------------------------------------
+#-- BEGIN Bastion.Model.isClerk PROTOCOL |
+#↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓↓
+	@property
+	def sites(self):
+		"""
+		I answer a sorted collection of the names of all sites known to this vault.
+		"""
+		#-- sites are top level elements relative to the root of the vault.
+		sites = [ ]
+		for alien in self.scurler.lsall():
+			if alien.is_dir():
+				sites.append(alien.name)
+
+		return tuple(sorted(sites))
+
+	def zones(self, site):
+		"""
+		Given a site name, I answer a sorted collection of zone names that are known to this vault.
+		"""
+		#-- a zone will be a subdirectory (subfolder) of the given site.
+		#-- look for all of the subfolders of root / site
+		zones = [ ]
+		sroot = self.scurler / RDN(site)
+		if sroot.exists( ):
+			for alien in sroot.lsall( ):
+				if alien.is_dir():
+					zones.append(alien.name)
+		return tuple(sorted(zones))
+
+	def assets(self, site, zone):
+		#-- assets will be subdirectories (subfolders) of a given site, zone.
+		assets = [ ]
+		zroot = self.scurler / site / zone
+		if zroot.exists( ):
+			for alien in zroot.is_dir():
+				assets.append(alien.name)
+		return tuple(sorted(assets))
+
+	def manifest(self, *args):
+		"""
+		I answer a manifest of the named asset...
+		manifest(ark)
+		manifest(site, zone, asset)
+		"""
+		ark = None
+		if len(args) == 1:
+			arg = args[0]
+			if isinstance(arg, Bastion.Model.isAsset):
+				ark = arg.ARK
+			else:
+				ark = arg
+		elif len(args) == 3:
+			ark = ARK(site, zone, asset)
+		else:
+			raise ValueError
+
+		cell = self.scurler / ark.site / ark.zone / ark.asset
+		if cell.exists():
+			blondes = [ ]
+			for alien in cell.lsall():
+				if not alien.is_dir():
+					blondes.append( BLONDE.decode(alien.stem) )
+		manifest = Bastion.Curator.Manifest(ark, blondes)
+
+		return manifest
+#↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑↑
+#-- END Bastion.Model.isClerk PROTOCOL |
+#---------------------------------------