From 21772c3a6596b02be42ebae1d6e5fa0161c66e17 Mon Sep 17 00:00:00 2001 From: Nathan Denny Date: Mon, 7 Oct 2024 09:32:38 -0400 Subject: [PATCH] Added initial Dockerfile to create doppelganger image; ported in "scurler" code from a previous demo project. --- Dockerfile | 48 +++++ lab/fortress_example.py | 19 ++ lib/Bastion/Movers/__init__.py | 5 + lib/Bastion/Movers/sCURL.py | 356 +++++++++++++++++++++++++++++++++ 4 files changed, 428 insertions(+) create mode 100644 Dockerfile create mode 100644 lab/fortress_example.py create mode 100644 lib/Bastion/Movers/__init__.py create mode 100644 lib/Bastion/Movers/sCURL.py diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..24b7464 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,48 @@ +FROM ubuntu:22.04 + +#-- Set the specific version of bastion to install. +ENV BASTION bastion-q24A034X + +RUN apt update && apt upgrade -y + +RUN mkdir /SCRATCH && \ + mkdir /DATA && \ + mkdir /LOG && \ + mkdir /BFD && \ + mkdir /CONF && \ + mkdir /root/.private && \ + mkdir /root/.bastion && \ + mkdir /root/.ssh && \ + mkdir /etc/bastion + +#-- Install the RCAC supplied hsi package +#-- Note, that the RCAC .deb package has an undeclared dependency on libedit2 +RUN apt install -y libedit2 +COPY hsi-htar-9.3.0-rcac1-amd64.deb . +RUN dpkg --install hsi-htar-9.3.0-rcac1-amd64.deb +RUN rm hsi-htar-9.3.0-rcac1-amd64.deb + +#-- Python dependencies for bastion +RUN apt install -y \ + python3 \ + python3-yaml \ + python3-ruamel.yaml \ + python-is-python3 + +#-- Install bastion from local tar file. +WORKDIR /opt +COPY ${BASTION}.tar . +RUN tar xvf ${BASTION}.tar +RUN ln -s /opt/${BASTION} /opt/bastion +RUN rm ${BASTION}.tar + +WORKDIR / +COPY CONF/conf-000-dockerized.yaml /etc/bastion +COPY CONF/ssh_config /root/.ssh/config + +#-- Useful stuff for debugging, but can probably be taken out of distribution. +RUN apt install -y \ + nano \ + git + + diff --git a/lab/fortress_example.py b/lab/fortress_example.py new file mode 100644 index 0000000..8e98a93 --- /dev/null +++ b/lab/fortress_example.py @@ -0,0 +1,19 @@ +import pathlib + +from Bastion.Movers.sCURL import SCURLer + +USER = 'ndenny' +KEYPATH = pathlib.Path("~/.ssh/bastion_bot").expanduser() + +LOCAL = pathlib.Path("/mnt/BFD/bastion/bank") +REMOTE = pathlib.Path("/home/{}/bastion".format(USER)) + +SITE = "rusina" +ZONE = "soundscapes" +ASSET = "HackathonData" +BLONDE = "3AQXEGFS024A03CMFZMT.tar" + +fortress = SCURLer(USER, 'sftp.fortress.rcac.purdue.edu', keyfile = KEYPATH, silent = False, verbose = True) +home = fortress['/home/{}'.format(USER)] +bastion = fortress['/home/{}/bastion'.format(USER)] + diff --git a/lib/Bastion/Movers/__init__.py b/lib/Bastion/Movers/__init__.py new file mode 100644 index 0000000..c755547 --- /dev/null +++ b/lib/Bastion/Movers/__init__.py @@ -0,0 +1,5 @@ +""" +Bastion.Movers + +I am the collection of codes for the transfer ("mover") agents +""" diff --git a/lib/Bastion/Movers/sCURL.py b/lib/Bastion/Movers/sCURL.py new file mode 100644 index 0000000..fa72e52 --- /dev/null +++ b/lib/Bastion/Movers/sCURL.py @@ -0,0 +1,356 @@ +""" +Bastion.Movers.sCURL + +I am a library providing a scriptable interface to SFTP via CURL. +""" +import subprocess +import pathlib +import datetime +import sys + +import logging + +logger = logging.getLogger(__name__) +logging.basicConfig(level = logging.DEBUG) + +nMONTH = { + 'Jan': 1, + 'Feb': 2, + 'Mar': 3, + 'Apr': 4, + 'May': 5, + 'Jun': 8, + 'Jul': 7, + 'Aug': 8, + 'Sep': 9, + 'Oct': 10, + 'Nov': 11, + 'Dec': 12 +} + + +class Alien: + """ + I am an accessor to a remotely hosted object (i.e. a resource path on a remote host). + """ + XATTRS = ('permits', 'size', 'mdate') + + def __init__(self, scurler, rpath, **kwargs): + self.scurler = scurler + self.rpath = pathlib.PurePosixPath(rpath) + self.permits = None + self.size = None + self.mdate = None + + #-- look for kwargs to set permits, size, and mtime + #-- all of which default to None if not explicitly given. + for k in self.XATTRS: + setattr(self, k, kwargs.get(k, None)) + + @classmethod + def fromLS(cls, scurler, parsed): + rpath = pathlib.PurePosixPath(parsed['rpath']) + + opts = { } + for k in cls.XATTRS: + opts[k] = parsed[k] + + return cls(scurler, rpath, **opts) + + def toDEX(self): + dex = { } + dex['user'] = self.scurler.user + dex['host'] = self.scurler.host + dex['path'] = self.rpath + for k in self.XATTRS: + v = getattr(self, k, None) + if v: + dex[k] = v + return dex + + def toJDN(self): + jdn = self.toDEX() + jdn['path'] = str(jdn['path']) + if 'mdate' in jdn: + jdn['mdate'] = jdn['mdate'].isoformat() + return jdn + + @property + def URL(self): + return self.scurler.URL(self.rpath) + + def __lshift__(self, lpath): + """ + put/upload operation + """ + successful = self.scurler.put(lpath, self.rpath) + if not successful: + raise Exception("failed to put local file {} to remote path {}".format(str(lpath), str(self.rpath))) + + def __rshift__(self, lpath): + """ + get/download operation + """ + successful, location = self.scurler.get(self.rpath, lpath) + if not successful: + raise Exception("failed to download remote object {} to local path {}".format(str(self.rpath), str(lpath))) + + def __iter__(self): + """ + Iterates over the contents of the folder, + each item being a pathlib.Path object. + """ + return iter(self.ls()) + + def ls(self): + """ + Given that rpath is a remote folder, + I answer a list of aliens in the remote folder. + """ + return [Alien(self.scurler, p) for p in self.scurler.ls(self.rpath)] + + def lsall(self): + return self.scurler.lsall(self.rpath) + + @property + def is_folder(self): + #-- First look at the permission bits. + #-- If we have permission bits, the first bit is the directory flag. + if self.permits: + if self.permits[0] == 'd': + return True + + #-- If we don't have permits, then try an ls operation on this path. + folderq = True + try: + self.scurler.ls(self.rpath) + except NotADirectoryError: + folderq = False + return folderq + + +def parsel(entry, rpath = None): + """ + parse ls info, like... + -rw-r--r-- 1 ndenny student 1105920 Dec 5 2023 NAPE.tar + drwxr-xr-x 2 ndenny student 2 Apr 17 14:12 Wolfram Mathematica + """ + fields = dict(enumerate(entry.split())) + print(">>>{}<<<".format(entry)) + for k in sorted(fields.keys()): + print("{} --> {}".format(k, fields[k])) + permits = fields[0] + owner = fields[2] + group = fields[3] + size = int(fields[4]) + month = fields[5] + day = int(fields[6]) + yt = fields[7] + name = fields[8] + + if ':' in yt: + #-- interpret yt field as a time stamp, implies current year. + year = datetime.date.today().year + else: + #-- interpret yt field as a year + year = int(yt) + + mdate = datetime.date(year, nMONTH[month], day) + + parsed = { + 'name': name, + 'permits': permits, + 'owner': owner, + 'group': group, + 'size': size, + 'mdate': mdate + } + + if rpath is not None: + parsed['rpath'] = rpath / name + + return parsed + +class SCURLer: + """ + I am a client for executing single SFTP operations on a remote host via CURL. + I execute all of my SFTP operations by executing CURL in a subshell. + """ + def __init__(self, user, host, **kwargs): + self.host = host + self.user = user + + #-------------------------------------------------------------- + #-- Configure while looking for additional keyword arguments. | + #-------------------------------------------------------------- + self.keypath = kwargs.get('keyfile', None) + self.silent = kwargs.get('silent', True) #-- default to silent mode + self.verbose = kwargs.get('verbose', False) #-- default to terse output mode + self.mkdirs = kwargs.get('mkdirs', True) #-- default to creating missing directories in upload paths. + self.CURL = kwargs.get('curl', "/usr/bin/curl") + + + self.lastop = None + + #------------------------------------------------------------------------- + #-- Construct the base command given my default configuration. | + #-- All SFTP operations will be built by augmenting the base command ... | + #-- ... with additional flags and arguments. | + #------------------------------------------------------------------------- + basecom = [self.CURL] + if self.keypath is not None: + basecom.extend(['--key', str(self.keypath)]) + if self.silent: + basecom.append('-s') + if self.verbose: + basecom.append('-v') + + self.basecom = tuple(basecom) + + def URL(self, rpath = None): + """ + Given an absolute path on the remote host (rpath), I construct an SFTP URL to the remote path. + """ + if rpath is None: + return "sftp://{}@{}/".format(self.user, self.host) + else: + rpath = pathlib.PurePosixPath(rpath) + if rpath.is_absolute(): + return "sftp://{}@{}{}".format(self.user, self.host, str(rpath)) + else: + raise ValueError("rpath {} must be an absolute path".format(str(rpath))) + + def ls(self, rpath): + """ + Given an absolute path on the remote host (rpath), I execute a remote "ls" operation. + """ + rpath = pathlib.PurePosixPath(rpath) + + lsurl = "{}/".format(self.URL(rpath)) + + lscom = list(self.basecom) + lscom.append('--list-only') + lscom.append(lsurl) + + logger.debug("{}".format(lscom)) + + p = subprocess.run(lscom, capture_output = True) + self.lastop = p + if p.returncode == 0: + entries = p.stdout.decode('utf-8').split('\n') + catalog = [(rpath / entry) for entry in entries] + return catalog + else: + print(p.stderr) + raise NotADirectoryError + + def lsall(self, rpath): + """ + Given an absolute path on the remote host (rpath), I execute a remote "ls" operation. + """ + rpath = pathlib.PurePosixPath(rpath) + + lsurl = "{}/".format(self.URL(rpath)) + + lscom = list(self.basecom) + lscom.append(lsurl) + + logger.debug("{}".format(lscom)) + + p = subprocess.run(lscom, capture_output = True) + self.lastop = p + if p.returncode == 0: + entries = p.stdout.decode('utf-8').split('\n') + catalog = [ ] + for entry in entries: + entry = entry.strip() + if entry: + info = parsel(entry, rpath) + if info['name'] not in ('.', '..'): + #-- We ommit the special directory references: '.' and '..', but allow all others. + catalog.append( Alien.fromLS(self, info) ) + return catalog + else: + logger.debug(p.stderr) + raise NotADirectoryError + + def put(self, lpath, rpath): + """ + Given a path to a local file (lpath) and the full path on the remote host (rpath), + I upload the local file (at lpath) to the remote location specified by rpath. + If needed, I automatically make any intermediate folders in the remote path. + """ + pturl = self.URL(rpath) + ptcom = list(self.basecom) + ptcom.append('--upload-file') + ptcom.append(str(lpath)) + ptcom.append(str(pturl)) + if self.mkdirs: + ptcom.append('--ftp-create-dirs') + logger.debug("{}".format(ptcom)) + p = subprocess.run(ptcom, capture_output = True) + self.lastop = p + if p.returncode == 0: + return True + else: + return False + + def get(self, rpath, lpath = None): + """ + Given a full path to a file on the remote host (rpath), I download the file. + By default, the file is downloaded into the cwd as a file of the same name; + however, I can also be given a full local path (lpath) as the location to download the file. + """ + rpath = pathlib.PurePosixPath(rpath) + + if lpath is None: + target = pathlib.Path.cwd() / rpath.name + else: + lpath = pathlib.Path(lpath) + if lpath.is_dir(): + target = lpath / rpath.name + else: + target = lpath + + gturl = self.URL(rpath) + gtcom = list(self.basecom) + gtcom.append(gturl) + gtcom.append('-o') + gtcom.append(str(target)) + + logger.debug("{}".format(gtcom)) + p = subprocess.run(gtcom, capture_output = True) + self.lastop = p + if p.returncode == 0: + return (True, target) + else: + return (False, None) + + def mkdir(self, rpath): + """ + I make the remote path (rpath) via SFTP's mkdir command. + NOTE: when uploading files to a remote host, folders are typically automatically created during the upload, + making the method somewhat redundant or unnecessary for upload operations. + """ + rpath = pathlib.PurePosixPath(rpath) + if not rpath.is_absolute(): + raise ValueError("remote path {} must be an absolute path to make".format(rpath)) + + mkcom = list(self.basecom) + mkcom.append(self.URL()) + mkcom.append('-Q') + mkcom.append('mkdir {}'.format(str(rpath))) + print(mkcom) + + p = subprocess.run(mkcom, capture_output = True) + self.lastop = p + if p.returncode == 0: + return True + else: + return False + + def __getitem__(self, rpath): + """ + Using array index semantics, I answer an "Alien" interface to the given remote path (rpath). + """ + return Alien(self, rpath)