From 4936578851c08ee50831635bdae19d15115e42e4 Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Wed, 30 Dec 2020 15:13:54 -0500 Subject: [PATCH 01/14] Update Python requirements to fix Flask_JWT_Extended and PyJWT conflict (cherry picked from commit 2bd362cc29bd93d07050a86210a2827ff3aa245f) --- api/requirements.txt | 36 ++++++++++++++---------------------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/api/requirements.txt b/api/requirements.txt index 1aaa908..d87bdac 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,22 +1,14 @@ -aniso8601==8.0.0 -astroid==2.4.2 -click==7.1.2 -Flask==1.1.2 -Flask-RESTful==0.3.8 -Flask-JWT-Extended==3.24.1 -gunicorn==20.0.4 -isort==4.3.21 -itsdangerous==1.1.0 -Jinja2==2.11.2 -lazy-object-proxy==1.4.3 -MarkupSafe==1.1.1 -mccabe==0.6.1 -pylint==2.5.3 -python-dateutil==2.8.1 -python-dotenv==0.15.0 -pytz==2020.1 -six==1.15.0 -toml==0.10.1 -typed-ast==1.4.1 -Werkzeug==1.0.1 -wrapt==1.12.1 +# See: https://pip.pypa.io/en/stable/reference/pip_install/#example-requirements-file + +# General Utilities +gunicorn +pipdeptree +pylint + +# API +python-dotenv +Flask-RESTful +python-dateutil +Flask-JWT-Extended +# Prevent upgrade to 2.x until Flask-JWT-Extended is updated to support it +PyJWT == 1.* \ No newline at end of file From 49106215a8a85a19acc4b3fc4fa11d264c22b1c5 Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Wed, 30 Dec 2020 17:14:23 -0500 Subject: [PATCH 02/14] Update venv-manager to allow 60 second for requirement installation (cherry picked from commit 5dd1da95824946384fa097c7040c6c7d83943151) --- utils/venv-manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/venv-manager.py b/utils/venv-manager.py index 72ecd3b..3bb15ce 100644 --- a/utils/venv-manager.py +++ b/utils/venv-manager.py @@ -79,7 +79,7 @@ def get_args() -> argparse.Namespace: return parser.parse_args() -def run_logged_subprocess(command: Union[str, list], timeout: int = 10, shell: bool = True) -> tuple: +def run_logged_subprocess(command: Union[str, list], timeout: int = 60, shell: bool = True) -> tuple: """Executes a shell command using subprocess with logging. stderr is redirected to stdout and stdout is pipped to logger. @@ -96,7 +96,7 @@ def run_logged_subprocess(command: Union[str, list], timeout: int = 10, shell: b Args: command (Union): The command to run. If shell=False, pass a list with the first item being the command and the subsequent items being arguments. If shell=True, pass a string as you would type it into a shell. - timeout (int): The number of seconds to wait for a program before killing it + timeout (int): The number of seconds to wait for a program before killing it. Defaults to 60. Returns: tuple: With the first value being the return code and second being the combined stdout+stderr From a6476393fafe991b166cdb003f52790a3faca767 Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Fri, 22 Jan 2021 18:46:29 +0000 Subject: [PATCH 03/14] Add EasyAD to project requirements --- api/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/api/requirements.txt b/api/requirements.txt index d87bdac..e6bfde6 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -11,4 +11,5 @@ Flask-RESTful python-dateutil Flask-JWT-Extended # Prevent upgrade to 2.x until Flask-JWT-Extended is updated to support it -PyJWT == 1.* \ No newline at end of file +PyJWT == 1.* +easyad \ No newline at end of file From 75e8cd480787f90d914a3542a972ecceddde8bcb Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Fri, 22 Jan 2021 19:07:18 +0000 Subject: [PATCH 04/14] Replace shared username/password with AD auth --- api/api.py | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/api/api.py b/api/api.py index 0cbd771..997e0f9 100644 --- a/api/api.py +++ b/api/api.py @@ -7,6 +7,9 @@ ) from werkzeug.security import check_password_hash import os, dotenv +from easyad import EasyAD +from ldap.filter import escape_filter_chars +from ldap import INVALID_CREDENTIALS as LDAP_INVALID_CREDENTIALS import ECNQueue # Load envrionment variables for ./.env @@ -47,6 +50,58 @@ +def user_is_valid(username: str, password: str) -> bool: + """Checks if user is valid and in webqueue2 login group. + + Args: + username (str): Career account username. + password (str): Career account passphrase. + + Returns: + bool: True if user is valid, otherwise False. + """ + + # Check for empty arguments + if (username == "" or password == ""): + return False + + # Initialize EasyAD + config = { + "AD_SERVER": "boilerad.purdue.edu", + "AD_DOMAIN": "boilerad.purdue.edu" + } + ad = EasyAD(config) + + # Prepare search critiera for Active Directory + credentials = { + "username": escape_filter_chars(username), + "password": password + } + attributes = [ 'cn', "memberOf" ] + filter_string = f'(&(objectClass=user)(|(sAMAccountName={username})))' + + # Do user search + try: + user = ad.search(credentials=credentials, attributes=attributes, filter_string=filter_string)[0] + # pylint says this is an error but it works so ¯\_(ツ)_/¯ + except LDAP_INVALID_CREDENTIALS: + return False + + # Isolate group names + # Example: + # 'CN=00000227-ECNStuds,OU=BoilerADGroups,DC=BoilerAD,DC=Purdue,DC=edu' becomes + # `00000227-ECNStuds` + user_groups = [ group.split(',')[0].split('=')[1] for group in user["memberOf"] ] + + # Check group membership + webqueue_login_group = "00000227-ECN-webqueue" + if webqueue_login_group not in user_groups: + return False + + return True + + + class Login(Resource): def post(self) -> tuple: """Validates username/password and returns both access and refresh tokens. @@ -76,10 +131,8 @@ def post(self) -> tuple: if field not in data.keys(): return ({ "message": f"{field} missing from request body"}, 422) - if data["username"] != os.environ.get("SHARED_USERNAME"): - return ({ "message": "Username invalid"}, 401) - if not check_password_hash(os.environ.get("SHARED_PASSWORD_HASH"), data["password"]): - return ({ "message": "Password invalid"}, 401) + if not user_is_valid(data["username"], data["password"]): + return ({ "message": "Username or password is invalid"}, 401) access_token = create_access_token(data["username"]) refresh_token = create_refresh_token(data["username"]) From e0686231fc01f7d7c7fc3db2b5fb9ee6c20c97a8 Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Fri, 22 Jan 2021 19:10:05 +0000 Subject: [PATCH 05/14] Update comments --- api/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/api.py b/api/api.py index 997e0f9..65e911f 100644 --- a/api/api.py +++ b/api/api.py @@ -9,6 +9,7 @@ import os, dotenv from easyad import EasyAD from ldap.filter import escape_filter_chars +# pylint says this is an error but it works so ¯\_(ツ)_/¯ from ldap import INVALID_CREDENTIALS as LDAP_INVALID_CREDENTIALS import ECNQueue @@ -83,7 +84,6 @@ def user_is_valid(username: str, password: str) -> bool: # Do user search try: user = ad.search(credentials=credentials, attributes=attributes, filter_string=filter_string)[0] - # pylint says this is an error but it works so ¯\_(ツ)_/¯ except LDAP_INVALID_CREDENTIALS: return False From 25b33a6e2a81fff61c123f5949811449c4ea9226 Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Thu, 28 Jan 2021 12:36:01 -0500 Subject: [PATCH 06/14] Update comments, change minor formatting --- utils/venv-manager.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/utils/venv-manager.py b/utils/venv-manager.py index 3bb15ce..6209901 100644 --- a/utils/venv-manager.py +++ b/utils/venv-manager.py @@ -36,21 +36,24 @@ logger = logging.getLogger(logger_name) logger.setLevel(logging.DEBUG) -# See: https://docs.python.org/3/library/logging.html#logrecord-attributes +# See Formatting Details: https://docs.python.org/3/library/logging.html#logrecord-attributes +# Example: Jan 28 2021 12:19:28 venv-manager : [INFO] Message log_message_format = "%(asctime)s %(name)s : [%(levelname)s] %(message)s" -# See: https://docs.python.org/3.6/library/time.html#time.strftime +# See Time Formatting Details: https://docs.python.org/3.6/library/time.html#time.strftime +# Example: Jan 28 2021 12:19:28 log_time_format = "%b %d %Y %H:%M:%S" log_formatter = logging.Formatter(log_message_format, log_time_format) +# Configure output to stdout stream_handler = logging.StreamHandler() stream_handler.setFormatter(log_formatter) stream_handler.setLevel(logging.INFO) +logger.addHandler(stream_handler) +# Configure out to logfile log_file_path = Path(WEBQUEUE2_DIR, "utils", f'{logger_name}.log') file_handler = logging.FileHandler(log_file_path) file_handler.setFormatter(log_formatter) - -logger.addHandler(stream_handler) logger.addHandler(file_handler) @@ -210,6 +213,7 @@ def delete_environment() -> int: logger.critical(f"Failed to delete virtual environment {VENV_NAME} at {VENV_DIR}. Exiting") return 5 + def reset_environment() -> int: """Resets a virtual environment for webqueue2 From e33eec054a4f4b619e75aff49dc67bc013dd41e0 Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Mon, 1 Feb 2021 00:26:08 -0500 Subject: [PATCH 07/14] Add pyldap build process to venv-manager --- utils/venv-manager.py | 277 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 274 insertions(+), 3 deletions(-) diff --git a/utils/venv-manager.py b/utils/venv-manager.py index 6209901..d6ddd53 100644 --- a/utils/venv-manager.py +++ b/utils/venv-manager.py @@ -12,7 +12,8 @@ """ from pathlib import Path -import os, logging, argparse, subprocess +import os, logging, argparse, subprocess, urllib.request, json, configparser +from urllib.error import HTTPError, URLError from typing import Union @@ -27,6 +28,11 @@ VENV_DIR = Path(API_DIR, VENV_NAME) +# Set virtual evironment resource paths +VENV_INTERPRETER = Path(VENV_DIR, "bin", "python3") +VENV_REQUIREMENTS_FILE = Path(API_DIR, "requirements.txt") + + # Set minimum pip major version TARGET_PIP_VERSION = 19 @@ -130,6 +136,228 @@ def run_logged_subprocess(command: Union[str, list], timeout: int = 60, shell: b finally: logger.debug(f"Exiting subprocess for '{command}'") return (logged_shell_process.returncode, process_output_stream) + + +def install_custom_pyldap(venv_interpreter: str) -> int: + """Builds python-ldap without SASL support from GitHub source. + + The version of python-ldap to be used is determined in the following order: + - Version from requirements.txt (can be beta) + - Latest non-beta version from GitHub + + Exit Codes: + 0 = Success + 1 = Virtual environment interpreter does not exist + 5 = Could not complete GitHub API call + 10 = Could not download python-ldap source + 15 = Could not extract python-ldap source + 20 = Could not open pyldap build config file + 25 = pyldap build configuration file improperly formatted + 30 = Could not write pyldap build config file + 35 = Could not build pyldap + 40 = Could not install pyldap + + Args: + venv_interpreter (str): The absolute path to the python interpreter executable for the virtual environment. + + Returns: + int: Exit code. + """ + + logger.info("Starting pyldap build process") + + # Check for valid venv interpreter + logger.debug(f"Checking for valid venv interpreter at {venv_interpreter}") + if not os.path.exists(Path(venv_interpreter)): + logger.error(f"The interpreter for virtual environment {VENV_NAME} does not exist at {venv_interpreter}. Exiting.") + return 1 + logger.debug(f"venv interpreter is valid") + + # Get list of release tags for pyldap from GitHub API + logger.debug(f"Getting pyldap tags from GitHub") + pyldap_github_tags_url = "https://api.github.com/repos/python-ldap/python-ldap/tags" + try: + with urllib.request.urlopen(pyldap_github_tags_url) as request: + pyldap_tags = json.loads(request.read().decode("utf-8")) + except HTTPError as e: + logger.error(f"Could not connect to {pyldap_github_tags_url}. Got response {e.code} {e.msg}. Exiting") + return 5 + except URLError as e: + logger.error(f"Could not connect to {pyldap_github_tags_url}. {e.reason}. Exiting") + return 5 + logger.debug(f"Got {len(pyldap_tags)} pyldap tags from GitHub") + + # Build dictionary of available pyldap releases and their source code archive urls + # Example: + # { "name": "python-ldap-3.3.1", "zipball_url": "http://github.com/download" } becomes + # { "3.3.1": "http://github.com/download" } + logger.debug("Building list of pyldap versions.") + pyldap_versions = {} + for tag in pyldap_tags: + tag_version = tag["name"].split("-")[-1] + zipball_url = f"https://github.com/python-ldap/python-ldap/archive/python-ldap-{tag_version}.zip" + pyldap_versions[tag_version] = zipball_url + logger.debug("Built list of pyldap versions.") + + # Check requirements file for pyldap version + pyldap_version_from_requirements = "" + logger.info(f"Checking for pyldap version in requirements file {VENV_REQUIREMENTS_FILE}") + try: + with open(VENV_REQUIREMENTS_FILE) as requirements_file: + for line in requirements_file: + if line.startswith("pyldap"): + pyldap_version_from_requirements = line.split(" ")[-1].strip("\n") + logger.debug(f"Found pyldap version {pyldap_version_from_requirements} in requirements file") + break + except Exception as e: + logger.warning(f"Could not read requirements file {VENV_REQUIREMENTS_FILE}. {e.strerror}. Defaulting to latest non-beta release on GitHub.") + pass + + # Set pyldap version to value from requirements file if valid and available + if pyldap_version_from_requirements and pyldap_version_from_requirements in pyldap_versions.keys(): + logger.debug(f"pyldap version {pyldap_version_from_requirements} is available from GitHub") + pyldap_version = pyldap_version_from_requirements + logger.info(f"Set pyldap version to {pyldap_version}") + # Set to latest non-beta version + else: + logger.warning(f"pyldap version not found in requirements file. Defaulting to latest non-beta release on GitHub") + for version in pyldap_versions.keys(): + is_beta_version = "b" in version + if (not is_beta_version): + pyldap_version = version + break + logger.info(f"Set pyldap version to {pyldap_version}") + + # Download pyldap soure code + logger.info(f"Downloading pyldap {pyldap_version} source from {pyldap_versions[pyldap_version]}") + + tmp_dir = "/tmp" + download_file_name = f"python-ldap-{pyldap_version}.zip" + download_file_path = Path(tmp_dir, download_file_name) + + download_pyldap_returncode, _ = run_logged_subprocess(f"wget -q -O {download_file_path} {pyldap_versions[pyldap_version]}") + if download_pyldap_returncode == 0: + logger.debug(f"pyldap source downloaded to {download_file_path}") + else: + logger.error(f"Could not download pyldap source. Exiting.") + return 10 + + # Extract source code + logger.info("Extracing pyldap source") + + # The archive from GitHub has a root folder formatted 'user-repo-version'. + # Because the pyldap source is user 'python-ldap' and repo 'python-ldap' + # the build folder MUST be the following format: + BUILD_DIR_NAME = f"python-ldap-python-ldap-{pyldap_version}" + BUILD_DIR_PATH = Path(tmp_dir, BUILD_DIR_NAME) + + extract_source_returncode, _ = run_logged_subprocess(f"unzip -q -o -d {tmp_dir} {download_file_path}") + if extract_source_returncode == 0: + logger.debug(f"pyldap source extracted to {BUILD_DIR_PATH}") + else: + logger.error(f"Could not extract pyldap source. Exiting") + return 15 + + # Start the build process + logger.info(f"Building pyldap {pyldap_version}") + + # Read the pyldap build config file + pyldap_config_file_name = "setup.cfg" + pyldap_config_file_path = Path(BUILD_DIR_PATH, pyldap_config_file_name) + + logger.debug(f"Reading pyldap build config file {pyldap_config_file_path}") + pyldap_config = configparser.ConfigParser() + try: + with open(pyldap_config_file_path) as pyldap_config_file: + pyldap_config.read_file(pyldap_config_file) + logger.debug("Read pyldap build config file") + except Exception as e: + logger.error(f"Could not read pyldap build config file {pyldap_config_file_path}. {e.strerror}. Exiting") + return 20 + + # Check for SASL requirement in pyldap build config file + logger.debug("Checking for '_ldap' section") + if not pyldap_config.has_section("_ldap"): + logger.error("Can't find '_ldap' section in pyldap build config file. Unable to build pyldap. Exiting") + return 25 + else: + logger.debug("'_ldap' section found") + + logger.debug("Checking for 'defines' option") + if not pyldap_config.has_option("_ldap", "defines"): + logging.error("Can't find 'defines' option in pyldap build config file. Unable to build pyldap. Exiting") + else: + logger.debug("'defines' option found") + + # Remove SASL requirement if present + logger.debug("Removing SASL requirement") + + defines_options = pyldap_config['_ldap']['defines'].split(' ') + build_config_updated = False + try: + defines_options.remove('HAVE_SASL') + pyldap_config['_ldap']['defines'] = " ".join(defines_options) + logger.debug("SASL requirement removed") + build_config_updated = True + except ValueError as e: + logger.warning("SASL requirement not found in pyldap build config file. Build config file will not be modified") + pass + + # Write new build config + logger.debug("Writing new pyldap build config") + + if build_config_updated: + try: + with open(pyldap_config_file_path, 'w') as pyldap_config_file: + pyldap_config.write(pyldap_config_file) + logger.debug("Wrote new pyldap build config") + except Exception as e: + logger.error(f"Could not write pyldap build config file {pyldap_config_file_path}. {e.strerror}. Exiting") + return 30 + + # Build pyldap + logger.debug(f"Building pyldap {pyldap_version}") + build_pyldap_returncode, _ = run_logged_subprocess(f"cd {BUILD_DIR_PATH} && {VENV_DIR}/bin/python3 setup.py build") + if build_pyldap_returncode == 0: + logger.info(f"Built pyldap {pyldap_version}") + else: + logger.error(f"Could not build pyldap. Exiting.") + return 35 + + # Install pyldap + logger.debug(f"Installing pyldap {pyldap_version}") + install_pyldap_returncode, _ = run_logged_subprocess(f"cd {BUILD_DIR_PATH} && {VENV_DIR}/bin/python3 setup.py install") + if install_pyldap_returncode == 0: + logger.info(f"Installed pyldap {pyldap_version} in virtual environment {VENV_NAME} at {VENV_DIR}") + else: + logger.error(f"Could not install pyldap. Exiting.") + return 40 + + return 0 + + +def is_valid_requirement(line: str) -> bool: + """Determines if line is a valid requirement + + Args: + line (str): Line to check. + + Returns: + bool: True if line is valid requirement. False if line is not valid requirement. + """ + # Line is blank + if line == "\n" or line == "": + return False + + # Line is comment + if line.startswith("#"): + return False + + # Line is for pyldap + if line.startswith("pyldap"): + return False + + return True def create_environment() -> int: @@ -139,6 +367,8 @@ def create_environment() -> int: 0 = Success 5 = VENV_DIR already exists 10 = Could not create VENV_DIR + 11 = Could not install pyldap + 12 = Could not read requirements file 15 = Could not install requirements Returns: @@ -172,6 +402,7 @@ def create_environment() -> int: pip_version_full = check_pip_output.split()[1] logger.debug(f"pip version is {pip_version_full}") + # Update pip pip_version_major = pip_version_full.split(".")[0] if int(pip_version_major) < 19: logger.info(f"pip verion is {pip_version_major}.x (pip >= {TARGET_PIP_VERSION}.x needed.) Upgrading pip") @@ -181,10 +412,50 @@ def create_environment() -> int: logger.info(update_pip_output.split("\n")[-2]) else: logger.warning("Failed to update pip. Virtual environment dependencies may not install") - + # Install requirements logger.info("Installing requirements") - install_requirements_returncode, _ = run_logged_subprocess(f"{VENV_DIR}/bin/pip install -r {API_DIR}/requirements.txt") + + # Install python-ldap + install_pyldap_returncode = install_custom_pyldap(VENV_INTERPRETER) + if install_pyldap_returncode != 0: + logger.error("Could not install pyldap. Exiting.") + return 11 + + # Install the rest of the requirements from the requirements file + logger.debug(f"Checking for venv requirements file {VENV_REQUIREMENTS_FILE}") + if not os.path.exists(VENV_REQUIREMENTS_FILE): + logger.warning(f"Could not find requirements file {VENV_REQUIREMENTS_FILE}. No requirements will be installed.") + return 0 + logger.debug("Found requirements file") + + # Get raw requirements from requirements file + logger.debug("Reading raw requirements from requirements file") + try: + with open(VENV_REQUIREMENTS_FILE) as requirements_file: + raw_requirements = requirements_file.readlines() + except Exception as e: + logger.warning(f"Could not read requirements file {VENV_REQUIREMENTS_FILE}. {e.strerror}. Exiting.") + return 12 + logger.debug("Read raw requirements from requirements file") + + # Filter and clean requirements + logger.debug("Validating requirements") + valid_requirements = [] + for requirement in raw_requirements: + if is_valid_requirement(requirement): + valid_requirements.append(requirement.strip()) + logger.debug("Validated requirements") + + # Generate requirements string + logger.debug("Generating requirements string") + requirements_string = "" + for requirement in valid_requirements: + requirements_string += f"'{requirement}' " + logger.debug(f"Generated requirements string {requirements_string}") + + # Install requirements + install_requirements_returncode, _ = run_logged_subprocess(f"{VENV_DIR}/bin/pip install {requirements_string}") if install_requirements_returncode == 0: logger.info("Successfully installed requirements") return 0 From 469d4bd8adabd0533b28645a4f050e42d069da22 Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Mon, 1 Feb 2021 00:26:26 -0500 Subject: [PATCH 08/14] Update requirements.txt to include pyldap and update comments --- api/requirements.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/api/requirements.txt b/api/requirements.txt index e6bfde6..f7f9b7a 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,3 +1,4 @@ +# The Python virtual environment should be managed via the venv-manager utility, not directly by pip. # See: https://pip.pypa.io/en/stable/reference/pip_install/#example-requirements-file # General Utilities @@ -10,6 +11,10 @@ python-dotenv Flask-RESTful python-dateutil Flask-JWT-Extended -# Prevent upgrade to 2.x until Flask-JWT-Extended is updated to support it +# Flask-JWT-Extended doesn't support PyJWT 2.x as of 3.25.0 +# Prevent upgrade to PyJWT 2.x until Flask-JWT-Extended is updated to support it. +# Check: https://github.com/vimalloc/flask-jwt-extended/tags PyJWT == 1.* +# Specify pyldap version for custom build. This is not installed by pip. +pyldap == 3.3.1 easyad \ No newline at end of file From b91f0d9fdf5403eb5679c6588d16b911213e274c Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Tue, 2 Feb 2021 19:02:01 -0500 Subject: [PATCH 09/14] Add global exit codes with docs --- utils/venv-manager.py | 215 ++++++++++++++++++++---------------------- 1 file changed, 102 insertions(+), 113 deletions(-) diff --git a/utils/venv-manager.py b/utils/venv-manager.py index d6ddd53..051f7eb 100644 --- a/utils/venv-manager.py +++ b/utils/venv-manager.py @@ -9,6 +9,23 @@ Reset a virtual environment: $ venv-manager.py reset + +Exit Codes: + 0 = Success + 5 = Failed to make directory VENV_DIR. It already exists. + 10 = Failed to make VENV_DIR. Incorrect permissions. + 15 = Failed to create virtual environment. See LOG FILE. + 20 = Failed read requirements file VENV_REQUIREMENTS_FILE. + 25 = Failed to install requirements. See LOG FILE. + 30 = VENV_INTERPRETER does not exist. + 35 = Failed to get pyldap release info from GitHub. + 40 = Failed to download pyldap source. See LOG FILE. + 45 = Failed to extract pyldap source. See LOG FILE. + 50 = Failed to read pyldap build config file. + 55 = Failed to write pyldap build config file. + 60 = Failed to build pyldap VERSION. See LOG FILE. + 65 = Failed to install pyldap VERSION. See LOG FILE. + 70 = Failed to delete VENV_DIR """ from pathlib import Path @@ -145,32 +162,19 @@ def install_custom_pyldap(venv_interpreter: str) -> int: - Version from requirements.txt (can be beta) - Latest non-beta version from GitHub - Exit Codes: - 0 = Success - 1 = Virtual environment interpreter does not exist - 5 = Could not complete GitHub API call - 10 = Could not download python-ldap source - 15 = Could not extract python-ldap source - 20 = Could not open pyldap build config file - 25 = pyldap build configuration file improperly formatted - 30 = Could not write pyldap build config file - 35 = Could not build pyldap - 40 = Could not install pyldap - Args: venv_interpreter (str): The absolute path to the python interpreter executable for the virtual environment. Returns: int: Exit code. """ - logger.info("Starting pyldap build process") # Check for valid venv interpreter logger.debug(f"Checking for valid venv interpreter at {venv_interpreter}") if not os.path.exists(Path(venv_interpreter)): - logger.error(f"The interpreter for virtual environment {VENV_NAME} does not exist at {venv_interpreter}. Exiting.") - return 1 + logger.error(f"venv interpreter does not exist. Exiting") + exit(30) logger.debug(f"venv interpreter is valid") # Get list of release tags for pyldap from GitHub API @@ -180,11 +184,11 @@ def install_custom_pyldap(venv_interpreter: str) -> int: with urllib.request.urlopen(pyldap_github_tags_url) as request: pyldap_tags = json.loads(request.read().decode("utf-8")) except HTTPError as e: - logger.error(f"Could not connect to {pyldap_github_tags_url}. Got response {e.code} {e.msg}. Exiting") - return 5 + logger.error(f"Failed to connect to {pyldap_github_tags_url}. Got response {e.code} {e.msg}. Exiting") + exit(35) except URLError as e: logger.error(f"Could not connect to {pyldap_github_tags_url}. {e.reason}. Exiting") - return 5 + exit(35) logger.debug(f"Got {len(pyldap_tags)} pyldap tags from GitHub") # Build dictionary of available pyldap releases and their source code archive urls @@ -197,11 +201,11 @@ def install_custom_pyldap(venv_interpreter: str) -> int: tag_version = tag["name"].split("-")[-1] zipball_url = f"https://github.com/python-ldap/python-ldap/archive/python-ldap-{tag_version}.zip" pyldap_versions[tag_version] = zipball_url - logger.debug("Built list of pyldap versions.") + logger.debug(f"Built list of {len(pyldap_versions)} pyldap versions.") # Check requirements file for pyldap version pyldap_version_from_requirements = "" - logger.info(f"Checking for pyldap version in requirements file {VENV_REQUIREMENTS_FILE}") + logger.debug(f"Checking for pyldap version in requirements file {VENV_REQUIREMENTS_FILE}") try: with open(VENV_REQUIREMENTS_FILE) as requirements_file: for line in requirements_file: @@ -217,7 +221,7 @@ def install_custom_pyldap(venv_interpreter: str) -> int: if pyldap_version_from_requirements and pyldap_version_from_requirements in pyldap_versions.keys(): logger.debug(f"pyldap version {pyldap_version_from_requirements} is available from GitHub") pyldap_version = pyldap_version_from_requirements - logger.info(f"Set pyldap version to {pyldap_version}") + logger.info(f"Set pyldap version to {pyldap_version} (from requirements file)") # Set to latest non-beta version else: logger.warning(f"pyldap version not found in requirements file. Defaulting to latest non-beta release on GitHub") @@ -226,7 +230,7 @@ def install_custom_pyldap(venv_interpreter: str) -> int: if (not is_beta_version): pyldap_version = version break - logger.info(f"Set pyldap version to {pyldap_version}") + logger.info(f"Set pyldap version to {pyldap_version} (from GitHub releases)") # Download pyldap soure code logger.info(f"Downloading pyldap {pyldap_version} source from {pyldap_versions[pyldap_version]}") @@ -237,13 +241,12 @@ def install_custom_pyldap(venv_interpreter: str) -> int: download_pyldap_returncode, _ = run_logged_subprocess(f"wget -q -O {download_file_path} {pyldap_versions[pyldap_version]}") if download_pyldap_returncode == 0: - logger.debug(f"pyldap source downloaded to {download_file_path}") + logger.debug(f"Downloaded pyldap {pyldap_version} source to {download_file_path}") else: - logger.error(f"Could not download pyldap source. Exiting.") - return 10 + logger.error(f"Failed to download pyldap source. See {log_file_path}. Exiting") + exit(40) # Extract source code - logger.info("Extracing pyldap source") # The archive from GitHub has a root folder formatted 'user-repo-version'. # Because the pyldap source is user 'python-ldap' and repo 'python-ldap' @@ -251,12 +254,13 @@ def install_custom_pyldap(venv_interpreter: str) -> int: BUILD_DIR_NAME = f"python-ldap-python-ldap-{pyldap_version}" BUILD_DIR_PATH = Path(tmp_dir, BUILD_DIR_NAME) + logger.info(f"Extracing pyldap {pyldap_version} source to {BUILD_DIR_PATH}") extract_source_returncode, _ = run_logged_subprocess(f"unzip -q -o -d {tmp_dir} {download_file_path}") if extract_source_returncode == 0: - logger.debug(f"pyldap source extracted to {BUILD_DIR_PATH}") + logger.debug(f"Extracted pyldap source to {BUILD_DIR_PATH}") else: - logger.error(f"Could not extract pyldap source. Exiting") - return 15 + logger.error(f"Failed to extract pyldap source. See {log_file_path}. Exiting") + exit(45) # Start the build process logger.info(f"Building pyldap {pyldap_version}") @@ -264,6 +268,7 @@ def install_custom_pyldap(venv_interpreter: str) -> int: # Read the pyldap build config file pyldap_config_file_name = "setup.cfg" pyldap_config_file_path = Path(BUILD_DIR_PATH, pyldap_config_file_name) + pyldap_version_from_needs_updated = True logger.debug(f"Reading pyldap build config file {pyldap_config_file_path}") pyldap_config = configparser.ConfigParser() @@ -272,67 +277,70 @@ def install_custom_pyldap(venv_interpreter: str) -> int: pyldap_config.read_file(pyldap_config_file) logger.debug("Read pyldap build config file") except Exception as e: - logger.error(f"Could not read pyldap build config file {pyldap_config_file_path}. {e.strerror}. Exiting") - return 20 + logger.error(f"Failed to read pyldap build config file {pyldap_config_file_path}. {e}. Exiting") + exit(50) # Check for SASL requirement in pyldap build config file logger.debug("Checking for '_ldap' section") if not pyldap_config.has_section("_ldap"): - logger.error("Can't find '_ldap' section in pyldap build config file. Unable to build pyldap. Exiting") - return 25 + logger.warning("Failed to find '_ldap' section in pyldap build config file. pyldap may fail to build") + pyldap_version_from_needs_updated = False + pass else: logger.debug("'_ldap' section found") logger.debug("Checking for 'defines' option") if not pyldap_config.has_option("_ldap", "defines"): - logging.error("Can't find 'defines' option in pyldap build config file. Unable to build pyldap. Exiting") + logging.warning("Failed to find 'defines' option in pyldap build config file. pyldap may fail to build") + pyldap_version_from_needs_updated = False else: logger.debug("'defines' option found") # Remove SASL requirement if present - logger.debug("Removing SASL requirement") + if pyldap_version_from_needs_updated: + logger.debug("Removing SASL requirement") - defines_options = pyldap_config['_ldap']['defines'].split(' ') - build_config_updated = False - try: - defines_options.remove('HAVE_SASL') - pyldap_config['_ldap']['defines'] = " ".join(defines_options) - logger.debug("SASL requirement removed") - build_config_updated = True - except ValueError as e: - logger.warning("SASL requirement not found in pyldap build config file. Build config file will not be modified") - pass + defines_options = pyldap_config['_ldap']['defines'].split(' ') + build_config_updated = False + try: + defines_options.remove('HAVE_SASL') + pyldap_config['_ldap']['defines'] = " ".join(defines_options) + logger.debug("SASL requirement removed") + build_config_updated = True + except ValueError as e: + logger.warning("SASL requirement not found in pyldap build config file. Build config file will not be modified") + pass # Write new build config logger.debug("Writing new pyldap build config") - if build_config_updated: try: with open(pyldap_config_file_path, 'w') as pyldap_config_file: pyldap_config.write(pyldap_config_file) logger.debug("Wrote new pyldap build config") except Exception as e: - logger.error(f"Could not write pyldap build config file {pyldap_config_file_path}. {e.strerror}. Exiting") - return 30 + logger.error(f"Failed to write pyldap build config file {pyldap_config_file_path}. {e}. Exiting") + exit(55) # Build pyldap logger.debug(f"Building pyldap {pyldap_version}") build_pyldap_returncode, _ = run_logged_subprocess(f"cd {BUILD_DIR_PATH} && {VENV_DIR}/bin/python3 setup.py build") if build_pyldap_returncode == 0: - logger.info(f"Built pyldap {pyldap_version}") + logger.debug(f"Built pyldap {pyldap_version}") else: - logger.error(f"Could not build pyldap. Exiting.") - return 35 + logger.error(f"Failed to build pyldap {pyldap_version}. See {log_file_path}. Exiting") + exit(60) # Install pyldap - logger.debug(f"Installing pyldap {pyldap_version}") + logger.debug(f"Installing pyldap {pyldap_version} in virtual environment {VENV_NAME} at {VENV_DIR}") install_pyldap_returncode, _ = run_logged_subprocess(f"cd {BUILD_DIR_PATH} && {VENV_DIR}/bin/python3 setup.py install") if install_pyldap_returncode == 0: - logger.info(f"Installed pyldap {pyldap_version} in virtual environment {VENV_NAME} at {VENV_DIR}") + logger.debug(f"Installed pyldap {pyldap_version}") else: - logger.error(f"Could not install pyldap. Exiting.") - return 40 + logger.error(f"Failed to install pyldap {pyldap_version}. See {log_file_path}. Exiting") + exit(65) + logger.info(f"Finshed installing pyldap {pyldap_version}") return 0 @@ -363,14 +371,6 @@ def is_valid_requirement(line: str) -> bool: def create_environment() -> int: """Creates a virtual environment for webqueue2 - Exit Codes: - 0 = Success - 5 = VENV_DIR already exists - 10 = Could not create VENV_DIR - 11 = Could not install pyldap - 12 = Could not read requirements file - 15 = Could not install requirements - Returns: int: Exit code """ @@ -378,65 +378,68 @@ def create_environment() -> int: logger.info(f"Creating virtual environment {VENV_NAME} at {VENV_DIR}") # Check for an existing virtual environment + logger.debug(f"Creating virtual environment directory at {VENV_DIR}") try: os.mkdir(VENV_DIR) except FileExistsError: - logger.warning(f"The directory {VENV_DIR} already exists. Exiting") - return 5 + logger.error(f"Failed to make directory {VENV_DIR}. It already exists. Exiting") + exit(5) + except PermissionError: + logger.error(f"Failed to make directory {VENV_DIR}. Incorrect permissions. Exiting") + exit(10) + logger.debug(f"Created virtual environment directory") + - # Create virtual environmentc + # Create virtual environment + logger.debug(f"Creating virtual environment {VENV_NAME} at {VENV_DIR}") create_env_returncode, _ = run_logged_subprocess(f"cd {API_DIR} && python3 -m venv {VENV_NAME}", shell=True) if create_env_returncode == 0: - logger.info(f"Virtual environment {VENV_NAME} created at {VENV_DIR}") + logger.info(f"Successfully created virtual environment {VENV_NAME} created at {VENV_DIR}") else: - logger.critical(f"Could not create virtual environment {VENV_NAME} at {VENV_DIR}. Exiting") - return 10 + logger.error(f"Failed to create virtual environment. See {log_file_path}. Exiting") + exit(15) # Check pip version logger.debug("Checking pip version") check_pip_returncode, check_pip_output = run_logged_subprocess(f"{VENV_DIR}/bin/pip --version") - - if check_pip_returncode != 0: - logger.warning("Could not check pip version. Virtual environment dependencies may not install") - - pip_version_full = check_pip_output.split()[1] - logger.debug(f"pip version is {pip_version_full}") + if check_pip_returncode == 0: + pip_version_full = check_pip_output.split()[1] + logger.debug(f"pip version is {pip_version_full}") + else: + logger.warning("Failed to check pip version. Virtual environment dependencies may not install") - # Update pip + # Update pip if needed pip_version_major = pip_version_full.split(".")[0] if int(pip_version_major) < 19: - logger.info(f"pip verion is {pip_version_major}.x (pip >= {TARGET_PIP_VERSION}.x needed.) Upgrading pip") + logger.debug(f"pip verion is {pip_version_major}.x (pip >= {TARGET_PIP_VERSION}.x needed.) Upgrading pip") update_pip_returncode, update_pip_output = run_logged_subprocess(f"{VENV_DIR}/bin/pip install --upgrade pip") - if update_pip_returncode == 0: - logger.info(update_pip_output.split("\n")[-2]) + pip_install_message = update_pip_output.split("\n")[-2] + logger.debug(pip_install_message) else: logger.warning("Failed to update pip. Virtual environment dependencies may not install") # Install requirements - logger.info("Installing requirements") + logger.info("Installing virtual environment requirements") # Install python-ldap - install_pyldap_returncode = install_custom_pyldap(VENV_INTERPRETER) - if install_pyldap_returncode != 0: - logger.error("Could not install pyldap. Exiting.") - return 11 + install_custom_pyldap(VENV_INTERPRETER) # Install the rest of the requirements from the requirements file logger.debug(f"Checking for venv requirements file {VENV_REQUIREMENTS_FILE}") if not os.path.exists(VENV_REQUIREMENTS_FILE): - logger.warning(f"Could not find requirements file {VENV_REQUIREMENTS_FILE}. No requirements will be installed.") + logger.warning(f"Could not find requirements file {VENV_REQUIREMENTS_FILE}. No requirements will be installed") return 0 logger.debug("Found requirements file") # Get raw requirements from requirements file - logger.debug("Reading raw requirements from requirements file") + logger.debug(f"Reading raw requirements from requirements file {VENV_REQUIREMENTS_FILE}") try: with open(VENV_REQUIREMENTS_FILE) as requirements_file: raw_requirements = requirements_file.readlines() except Exception as e: - logger.warning(f"Could not read requirements file {VENV_REQUIREMENTS_FILE}. {e.strerror}. Exiting.") - return 12 + logger.warning(f"Failed read requirements file {VENV_REQUIREMENTS_FILE}. {e}. Exiting") + exit(20) logger.debug("Read raw requirements from requirements file") # Filter and clean requirements @@ -445,7 +448,7 @@ def create_environment() -> int: for requirement in raw_requirements: if is_valid_requirement(requirement): valid_requirements.append(requirement.strip()) - logger.debug("Validated requirements") + logger.debug(f"Validated {len(valid_requirements)} requirements") # Generate requirements string logger.debug("Generating requirements string") @@ -454,23 +457,22 @@ def create_environment() -> int: requirements_string += f"'{requirement}' " logger.debug(f"Generated requirements string {requirements_string}") - # Install requirements + # Install requirements + logger.debug("Installing requirements") install_requirements_returncode, _ = run_logged_subprocess(f"{VENV_DIR}/bin/pip install {requirements_string}") if install_requirements_returncode == 0: logger.info("Successfully installed requirements") - return 0 else: - logger.critical("Failed to install requirements. Exiting") - return 15 + logger.error(f"Failed to install requirements. See {log_file_path}. Exiting") + exit(25) + + logger.info("Finished creating virtual environment") + return 0 def delete_environment() -> int: """Deletes a virtual environment for webqueue2 - Exit Codes: - 0 = Success - 5 = Could not delete VENV_DIR - Returns: int: Exit code """ @@ -481,34 +483,21 @@ def delete_environment() -> int: logger.info(f"Successfully deleted virtual environment {VENV_NAME} at {VENV_DIR}") return 0 else: - logger.critical(f"Failed to delete virtual environment {VENV_NAME} at {VENV_DIR}. Exiting") - return 5 + logger.error(f∆"Failed to delete virtual environment {VENV_NAME} at {VENV_DIR}. Exiting") + exit(70) def reset_environment() -> int: """Resets a virtual environment for webqueue2 - Exit Codes: - 0 = Success - 5 = Could not delete VENV_DIR - 10 = Could not create VENV_DIR - Returns: int: Exit code """ logger.info(f"Resetting virtual environment {VENV_NAME} at {VENV_DIR}") - delete_returncode = delete_environment() - if delete_returncode != 0: - logger.critical(f"Failed to reset virtual environment {VENV_NAME} at {VENV_DIR}. Exiting") - return 5 - create_returncode = create_environment() - if create_returncode != 0: - logger.critical(f"Failed to reset virtual environment {VENV_NAME} at {VENV_DIR}. Exiting") - return 10 - logger.info(f"Successfully reset virtual environment {VENV_NAME} at {VENV_DIR}. Exiting") + return 0 if __name__ == "__main__": From ec5680cf4d86ae4c67dd545e249bd7ed463104ee Mon Sep 17 00:00:00 2001 From: "Campbell, Justin" Date: Thu, 4 Feb 2021 14:20:02 -0500 Subject: [PATCH 10/14] Remove breaking character --- utils/venv-manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/venv-manager.py b/utils/venv-manager.py index 051f7eb..95e6eee 100644 --- a/utils/venv-manager.py +++ b/utils/venv-manager.py @@ -483,7 +483,7 @@ def delete_environment() -> int: logger.info(f"Successfully deleted virtual environment {VENV_NAME} at {VENV_DIR}") return 0 else: - logger.error(f∆"Failed to delete virtual environment {VENV_NAME} at {VENV_DIR}. Exiting") + logger.error(f"Failed to delete virtual environment {VENV_NAME} at {VENV_DIR}. Exiting") exit(70) @@ -516,4 +516,4 @@ def reset_environment() -> int: elif action == "reset": exit(reset_environment()) else: - logger.critical(f'Invalid argument {action}') \ No newline at end of file + logger.critical(f'Invalid argument {action}') From c7e264f079251e7d6dc436f815a367f88f965c9a Mon Sep 17 00:00:00 2001 From: benne238 Date: Thu, 11 Feb 2021 16:19:29 -0500 Subject: [PATCH 11/14] removal of wekzeug.security import from api --- api/api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/api/api.py b/api/api.py index 65e911f..73128f6 100644 --- a/api/api.py +++ b/api/api.py @@ -5,7 +5,6 @@ jwt_required, get_jwt_identity, jwt_refresh_token_required, set_refresh_cookies, unset_refresh_cookies ) -from werkzeug.security import check_password_hash import os, dotenv from easyad import EasyAD from ldap.filter import escape_filter_chars From b56e22549e249bbfd4c79e7fd0a92f544df70431 Mon Sep 17 00:00:00 2001 From: benne238 Date: Thu, 11 Feb 2021 16:21:27 -0500 Subject: [PATCH 12/14] update requirements.txt --- api/requirements.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/api/requirements.txt b/api/requirements.txt index f7f9b7a..fcd08ee 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -17,4 +17,9 @@ Flask-JWT-Extended PyJWT == 1.* # Specify pyldap version for custom build. This is not installed by pip. pyldap == 3.3.1 -easyad \ No newline at end of file +easyad + +# API Documentation +mkdocs +mkdocs-material +mkautodoc \ No newline at end of file From 3139fb663534c06304e2d3a2ee30e9d8abe77ad5 Mon Sep 17 00:00:00 2001 From: benne238 Date: Thu, 11 Feb 2021 16:25:39 -0500 Subject: [PATCH 13/14] Removed duplicate columns in ItemTable (cherry picked from commit ecc31059304ff85d9089c81927dd74a017900e37) --- src/components/ItemTable/ItemTable.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/components/ItemTable/ItemTable.js b/src/components/ItemTable/ItemTable.js index 6c4df69..05e4280 100644 --- a/src/components/ItemTable/ItemTable.js +++ b/src/components/ItemTable/ItemTable.js @@ -47,10 +47,11 @@ export default function ItemTable({ data, rowCanBeSelected }) { { Header: 'Subject', accessor: 'subject' }, { Header: 'Status', accessor: 'status', }, { Header: 'Priority', accessor: 'priority' }, - { Header: 'Last Updated', accessor: 'lastUpdated', Cell: ({ value }) => }, + { Header: 'Last Updated', accessor: 'lastUpdated', sortInverted: true, Cell: ({ value }) => }, { Header: 'Department', accessor: 'department' }, { Header: 'Building', accessor: 'building' }, - { Header: 'Date Received', accessor: 'dateReceived', Cell: ({ value }) => }, + { Header: 'Date Received', accessor: 'dateReceived', sortInverted: true, Cell: ({ value }) => }, + ], []); const tableInstance = useTable( From f48de176215b35381029fc3bdf0819094c30524e Mon Sep 17 00:00:00 2001 From: benne238 Date: Thu, 11 Feb 2021 16:59:11 -0500 Subject: [PATCH 14/14] update gitignore for .egg files --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e238c91..ac3f93c 100644 --- a/.gitignore +++ b/.gitignore @@ -30,4 +30,5 @@ yarn-error.log* /api/venv __pycache__/ venv-manager.log -/api/.env \ No newline at end of file +/api/.env +*.egg* \ No newline at end of file