diff --git a/utils/venv-manager.py b/utils/venv-manager.py index 6209901..d6ddd53 100644 --- a/utils/venv-manager.py +++ b/utils/venv-manager.py @@ -12,7 +12,8 @@ """ from pathlib import Path -import os, logging, argparse, subprocess +import os, logging, argparse, subprocess, urllib.request, json, configparser +from urllib.error import HTTPError, URLError from typing import Union @@ -27,6 +28,11 @@ VENV_DIR = Path(API_DIR, VENV_NAME) +# Set virtual evironment resource paths +VENV_INTERPRETER = Path(VENV_DIR, "bin", "python3") +VENV_REQUIREMENTS_FILE = Path(API_DIR, "requirements.txt") + + # Set minimum pip major version TARGET_PIP_VERSION = 19 @@ -130,6 +136,228 @@ def run_logged_subprocess(command: Union[str, list], timeout: int = 60, shell: b finally: logger.debug(f"Exiting subprocess for '{command}'") return (logged_shell_process.returncode, process_output_stream) + + +def install_custom_pyldap(venv_interpreter: str) -> int: + """Builds python-ldap without SASL support from GitHub source. + + The version of python-ldap to be used is determined in the following order: + - Version from requirements.txt (can be beta) + - Latest non-beta version from GitHub + + Exit Codes: + 0 = Success + 1 = Virtual environment interpreter does not exist + 5 = Could not complete GitHub API call + 10 = Could not download python-ldap source + 15 = Could not extract python-ldap source + 20 = Could not open pyldap build config file + 25 = pyldap build configuration file improperly formatted + 30 = Could not write pyldap build config file + 35 = Could not build pyldap + 40 = Could not install pyldap + + Args: + venv_interpreter (str): The absolute path to the python interpreter executable for the virtual environment. + + Returns: + int: Exit code. + """ + + logger.info("Starting pyldap build process") + + # Check for valid venv interpreter + logger.debug(f"Checking for valid venv interpreter at {venv_interpreter}") + if not os.path.exists(Path(venv_interpreter)): + logger.error(f"The interpreter for virtual environment {VENV_NAME} does not exist at {venv_interpreter}. Exiting.") + return 1 + logger.debug(f"venv interpreter is valid") + + # Get list of release tags for pyldap from GitHub API + logger.debug(f"Getting pyldap tags from GitHub") + pyldap_github_tags_url = "https://api.github.com/repos/python-ldap/python-ldap/tags" + try: + with urllib.request.urlopen(pyldap_github_tags_url) as request: + pyldap_tags = json.loads(request.read().decode("utf-8")) + except HTTPError as e: + logger.error(f"Could not connect to {pyldap_github_tags_url}. Got response {e.code} {e.msg}. Exiting") + return 5 + except URLError as e: + logger.error(f"Could not connect to {pyldap_github_tags_url}. {e.reason}. Exiting") + return 5 + logger.debug(f"Got {len(pyldap_tags)} pyldap tags from GitHub") + + # Build dictionary of available pyldap releases and their source code archive urls + # Example: + # { "name": "python-ldap-3.3.1", "zipball_url": "http://github.com/download" } becomes + # { "3.3.1": "http://github.com/download" } + logger.debug("Building list of pyldap versions.") + pyldap_versions = {} + for tag in pyldap_tags: + tag_version = tag["name"].split("-")[-1] + zipball_url = f"https://github.com/python-ldap/python-ldap/archive/python-ldap-{tag_version}.zip" + pyldap_versions[tag_version] = zipball_url + logger.debug("Built list of pyldap versions.") + + # Check requirements file for pyldap version + pyldap_version_from_requirements = "" + logger.info(f"Checking for pyldap version in requirements file {VENV_REQUIREMENTS_FILE}") + try: + with open(VENV_REQUIREMENTS_FILE) as requirements_file: + for line in requirements_file: + if line.startswith("pyldap"): + pyldap_version_from_requirements = line.split(" ")[-1].strip("\n") + logger.debug(f"Found pyldap version {pyldap_version_from_requirements} in requirements file") + break + except Exception as e: + logger.warning(f"Could not read requirements file {VENV_REQUIREMENTS_FILE}. {e.strerror}. Defaulting to latest non-beta release on GitHub.") + pass + + # Set pyldap version to value from requirements file if valid and available + if pyldap_version_from_requirements and pyldap_version_from_requirements in pyldap_versions.keys(): + logger.debug(f"pyldap version {pyldap_version_from_requirements} is available from GitHub") + pyldap_version = pyldap_version_from_requirements + logger.info(f"Set pyldap version to {pyldap_version}") + # Set to latest non-beta version + else: + logger.warning(f"pyldap version not found in requirements file. Defaulting to latest non-beta release on GitHub") + for version in pyldap_versions.keys(): + is_beta_version = "b" in version + if (not is_beta_version): + pyldap_version = version + break + logger.info(f"Set pyldap version to {pyldap_version}") + + # Download pyldap soure code + logger.info(f"Downloading pyldap {pyldap_version} source from {pyldap_versions[pyldap_version]}") + + tmp_dir = "/tmp" + download_file_name = f"python-ldap-{pyldap_version}.zip" + download_file_path = Path(tmp_dir, download_file_name) + + download_pyldap_returncode, _ = run_logged_subprocess(f"wget -q -O {download_file_path} {pyldap_versions[pyldap_version]}") + if download_pyldap_returncode == 0: + logger.debug(f"pyldap source downloaded to {download_file_path}") + else: + logger.error(f"Could not download pyldap source. Exiting.") + return 10 + + # Extract source code + logger.info("Extracing pyldap source") + + # The archive from GitHub has a root folder formatted 'user-repo-version'. + # Because the pyldap source is user 'python-ldap' and repo 'python-ldap' + # the build folder MUST be the following format: + BUILD_DIR_NAME = f"python-ldap-python-ldap-{pyldap_version}" + BUILD_DIR_PATH = Path(tmp_dir, BUILD_DIR_NAME) + + extract_source_returncode, _ = run_logged_subprocess(f"unzip -q -o -d {tmp_dir} {download_file_path}") + if extract_source_returncode == 0: + logger.debug(f"pyldap source extracted to {BUILD_DIR_PATH}") + else: + logger.error(f"Could not extract pyldap source. Exiting") + return 15 + + # Start the build process + logger.info(f"Building pyldap {pyldap_version}") + + # Read the pyldap build config file + pyldap_config_file_name = "setup.cfg" + pyldap_config_file_path = Path(BUILD_DIR_PATH, pyldap_config_file_name) + + logger.debug(f"Reading pyldap build config file {pyldap_config_file_path}") + pyldap_config = configparser.ConfigParser() + try: + with open(pyldap_config_file_path) as pyldap_config_file: + pyldap_config.read_file(pyldap_config_file) + logger.debug("Read pyldap build config file") + except Exception as e: + logger.error(f"Could not read pyldap build config file {pyldap_config_file_path}. {e.strerror}. Exiting") + return 20 + + # Check for SASL requirement in pyldap build config file + logger.debug("Checking for '_ldap' section") + if not pyldap_config.has_section("_ldap"): + logger.error("Can't find '_ldap' section in pyldap build config file. Unable to build pyldap. Exiting") + return 25 + else: + logger.debug("'_ldap' section found") + + logger.debug("Checking for 'defines' option") + if not pyldap_config.has_option("_ldap", "defines"): + logging.error("Can't find 'defines' option in pyldap build config file. Unable to build pyldap. Exiting") + else: + logger.debug("'defines' option found") + + # Remove SASL requirement if present + logger.debug("Removing SASL requirement") + + defines_options = pyldap_config['_ldap']['defines'].split(' ') + build_config_updated = False + try: + defines_options.remove('HAVE_SASL') + pyldap_config['_ldap']['defines'] = " ".join(defines_options) + logger.debug("SASL requirement removed") + build_config_updated = True + except ValueError as e: + logger.warning("SASL requirement not found in pyldap build config file. Build config file will not be modified") + pass + + # Write new build config + logger.debug("Writing new pyldap build config") + + if build_config_updated: + try: + with open(pyldap_config_file_path, 'w') as pyldap_config_file: + pyldap_config.write(pyldap_config_file) + logger.debug("Wrote new pyldap build config") + except Exception as e: + logger.error(f"Could not write pyldap build config file {pyldap_config_file_path}. {e.strerror}. Exiting") + return 30 + + # Build pyldap + logger.debug(f"Building pyldap {pyldap_version}") + build_pyldap_returncode, _ = run_logged_subprocess(f"cd {BUILD_DIR_PATH} && {VENV_DIR}/bin/python3 setup.py build") + if build_pyldap_returncode == 0: + logger.info(f"Built pyldap {pyldap_version}") + else: + logger.error(f"Could not build pyldap. Exiting.") + return 35 + + # Install pyldap + logger.debug(f"Installing pyldap {pyldap_version}") + install_pyldap_returncode, _ = run_logged_subprocess(f"cd {BUILD_DIR_PATH} && {VENV_DIR}/bin/python3 setup.py install") + if install_pyldap_returncode == 0: + logger.info(f"Installed pyldap {pyldap_version} in virtual environment {VENV_NAME} at {VENV_DIR}") + else: + logger.error(f"Could not install pyldap. Exiting.") + return 40 + + return 0 + + +def is_valid_requirement(line: str) -> bool: + """Determines if line is a valid requirement + + Args: + line (str): Line to check. + + Returns: + bool: True if line is valid requirement. False if line is not valid requirement. + """ + # Line is blank + if line == "\n" or line == "": + return False + + # Line is comment + if line.startswith("#"): + return False + + # Line is for pyldap + if line.startswith("pyldap"): + return False + + return True def create_environment() -> int: @@ -139,6 +367,8 @@ def create_environment() -> int: 0 = Success 5 = VENV_DIR already exists 10 = Could not create VENV_DIR + 11 = Could not install pyldap + 12 = Could not read requirements file 15 = Could not install requirements Returns: @@ -172,6 +402,7 @@ def create_environment() -> int: pip_version_full = check_pip_output.split()[1] logger.debug(f"pip version is {pip_version_full}") + # Update pip pip_version_major = pip_version_full.split(".")[0] if int(pip_version_major) < 19: logger.info(f"pip verion is {pip_version_major}.x (pip >= {TARGET_PIP_VERSION}.x needed.) Upgrading pip") @@ -181,10 +412,50 @@ def create_environment() -> int: logger.info(update_pip_output.split("\n")[-2]) else: logger.warning("Failed to update pip. Virtual environment dependencies may not install") - + # Install requirements logger.info("Installing requirements") - install_requirements_returncode, _ = run_logged_subprocess(f"{VENV_DIR}/bin/pip install -r {API_DIR}/requirements.txt") + + # Install python-ldap + install_pyldap_returncode = install_custom_pyldap(VENV_INTERPRETER) + if install_pyldap_returncode != 0: + logger.error("Could not install pyldap. Exiting.") + return 11 + + # Install the rest of the requirements from the requirements file + logger.debug(f"Checking for venv requirements file {VENV_REQUIREMENTS_FILE}") + if not os.path.exists(VENV_REQUIREMENTS_FILE): + logger.warning(f"Could not find requirements file {VENV_REQUIREMENTS_FILE}. No requirements will be installed.") + return 0 + logger.debug("Found requirements file") + + # Get raw requirements from requirements file + logger.debug("Reading raw requirements from requirements file") + try: + with open(VENV_REQUIREMENTS_FILE) as requirements_file: + raw_requirements = requirements_file.readlines() + except Exception as e: + logger.warning(f"Could not read requirements file {VENV_REQUIREMENTS_FILE}. {e.strerror}. Exiting.") + return 12 + logger.debug("Read raw requirements from requirements file") + + # Filter and clean requirements + logger.debug("Validating requirements") + valid_requirements = [] + for requirement in raw_requirements: + if is_valid_requirement(requirement): + valid_requirements.append(requirement.strip()) + logger.debug("Validated requirements") + + # Generate requirements string + logger.debug("Generating requirements string") + requirements_string = "" + for requirement in valid_requirements: + requirements_string += f"'{requirement}' " + logger.debug(f"Generated requirements string {requirements_string}") + + # Install requirements + install_requirements_returncode, _ = run_logged_subprocess(f"{VENV_DIR}/bin/pip install {requirements_string}") if install_requirements_returncode == 0: logger.info("Successfully installed requirements") return 0