Add pyldap build process to venv-manager

utils/venv-manager.py

@@ -12,7 +12,8 @@
 """
 
 from pathlib import Path
-import os, logging, argparse, subprocess
+import os, logging, argparse, subprocess, urllib.request, json, configparser
+from urllib.error import HTTPError, URLError
 from typing import Union
 
 
@@ -27,6 +28,11 @@
 VENV_DIR = Path(API_DIR, VENV_NAME)
 
 
+# Set virtual evironment resource paths
+VENV_INTERPRETER = Path(VENV_DIR, "bin", "python3")
+VENV_REQUIREMENTS_FILE = Path(API_DIR, "requirements.txt")
+
+
 # Set minimum pip major version
 TARGET_PIP_VERSION = 19
 
@@ -130,6 +136,228 @@ def run_logged_subprocess(command: Union[str, list], timeout: int = 60, shell: b
         finally:
             logger.debug(f"Exiting subprocess for '{command}'")
             return (logged_shell_process.returncode, process_output_stream)
+
+
+def install_custom_pyldap(venv_interpreter: str) -> int:
+    """Builds python-ldap without SASL support from GitHub source.
+
+    The version of python-ldap to be used is determined in the following order:
+        - Version from requirements.txt (can be beta)
+        - Latest non-beta version from GitHub
+
+    Exit Codes:
+        0 = Success
+        1 = Virtual environment interpreter does not exist
+        5 = Could not complete GitHub API call
+        10 = Could not download python-ldap source
+        15 = Could not extract python-ldap source
+        20 = Could not open pyldap build config file
+        25 = pyldap build configuration file improperly formatted
+        30 = Could not write pyldap build config file
+        35 = Could not build pyldap
+        40 = Could not install pyldap
+
+    Args:
+        venv_interpreter (str): The absolute path to the python interpreter executable for the virtual environment.
+
+    Returns:
+        int: Exit code.
+    """
+
+    logger.info("Starting pyldap build process")
+
+    # Check for valid venv interpreter
+    logger.debug(f"Checking for valid venv interpreter at {venv_interpreter}")
+    if not os.path.exists(Path(venv_interpreter)):
+        logger.error(f"The interpreter for virtual environment {VENV_NAME} does not exist at {venv_interpreter}. Exiting.")
+        return 1
+    logger.debug(f"venv interpreter is valid")
+
+    # Get list of release tags for pyldap from GitHub API
+    logger.debug(f"Getting pyldap tags from GitHub")
+    pyldap_github_tags_url = "https://api.github.com/repos/python-ldap/python-ldap/tags"
+    try:
+        with urllib.request.urlopen(pyldap_github_tags_url) as request:
+            pyldap_tags = json.loads(request.read().decode("utf-8"))
+    except HTTPError as e:
+        logger.error(f"Could not connect to {pyldap_github_tags_url}. Got response {e.code} {e.msg}. Exiting")
+        return 5
+    except URLError as e:
+        logger.error(f"Could not connect to {pyldap_github_tags_url}. {e.reason}. Exiting")
+        return 5
+    logger.debug(f"Got {len(pyldap_tags)} pyldap tags from GitHub")
+
+    # Build dictionary of available pyldap releases and their source code archive urls
+    # Example:
+    #   { "name": "python-ldap-3.3.1", "zipball_url": "http://github.com/download" } becomes
+    #   { "3.3.1": "http://github.com/download" }
+    logger.debug("Building list of pyldap versions.")
+    pyldap_versions = {}
+    for tag in pyldap_tags:
+        tag_version = tag["name"].split("-")[-1]
+        zipball_url = f"https://github.com/python-ldap/python-ldap/archive/python-ldap-{tag_version}.zip"
+        pyldap_versions[tag_version] = zipball_url
+    logger.debug("Built list of pyldap versions.")
+
+    # Check requirements file for pyldap version
+    pyldap_version_from_requirements = ""
+    logger.info(f"Checking for pyldap version in requirements file {VENV_REQUIREMENTS_FILE}")
+    try:
+        with open(VENV_REQUIREMENTS_FILE) as requirements_file:
+            for line in requirements_file:
+                if line.startswith("pyldap"):
+                    pyldap_version_from_requirements = line.split(" ")[-1].strip("\n")
+                    logger.debug(f"Found pyldap version {pyldap_version_from_requirements} in requirements file")
+                    break
+    except Exception as e:
+        logger.warning(f"Could not read requirements file {VENV_REQUIREMENTS_FILE}. {e.strerror}. Defaulting to latest non-beta release on GitHub.")
+        pass
+
+    # Set pyldap version to value from requirements file if valid and available
+    if pyldap_version_from_requirements and pyldap_version_from_requirements in pyldap_versions.keys():
+        logger.debug(f"pyldap version {pyldap_version_from_requirements} is available from GitHub")
+        pyldap_version = pyldap_version_from_requirements
+        logger.info(f"Set pyldap version to {pyldap_version}")
+    # Set to latest non-beta version
+    else:
+        logger.warning(f"pyldap version not found in requirements file. Defaulting to latest non-beta release on GitHub")
+        for version in pyldap_versions.keys():
+            is_beta_version = "b" in version
+            if (not is_beta_version):
+                pyldap_version = version
+                break
+        logger.info(f"Set pyldap version to {pyldap_version}")
+
+    # Download pyldap soure code
+    logger.info(f"Downloading pyldap {pyldap_version} source from {pyldap_versions[pyldap_version]}")
+
+    tmp_dir = "/tmp"
+    download_file_name = f"python-ldap-{pyldap_version}.zip"
+    download_file_path = Path(tmp_dir, download_file_name)
+
+    download_pyldap_returncode, _ = run_logged_subprocess(f"wget -q -O {download_file_path} {pyldap_versions[pyldap_version]}")
+    if download_pyldap_returncode == 0:
+        logger.debug(f"pyldap source downloaded to {download_file_path}")
+    else:
+        logger.error(f"Could not download pyldap source. Exiting.")
+        return 10
+
+    # Extract source code
+    logger.info("Extracing pyldap source")
+
+    # The archive from GitHub has a root folder formatted 'user-repo-version'. 
+    # Because the pyldap source is user 'python-ldap' and repo 'python-ldap' 
+    # the build folder MUST be the following format:
+    BUILD_DIR_NAME = f"python-ldap-python-ldap-{pyldap_version}"
+    BUILD_DIR_PATH = Path(tmp_dir, BUILD_DIR_NAME)
+
+    extract_source_returncode, _ = run_logged_subprocess(f"unzip -q -o -d {tmp_dir} {download_file_path}")
+    if extract_source_returncode == 0:
+        logger.debug(f"pyldap source extracted to {BUILD_DIR_PATH}")
+    else:
+        logger.error(f"Could not extract pyldap source. Exiting")
+        return 15
+
+    # Start the build process
+    logger.info(f"Building pyldap {pyldap_version}")
+
+    # Read the pyldap build config file
+    pyldap_config_file_name = "setup.cfg"
+    pyldap_config_file_path = Path(BUILD_DIR_PATH, pyldap_config_file_name)
+
+    logger.debug(f"Reading pyldap build config file {pyldap_config_file_path}")
+    pyldap_config = configparser.ConfigParser()
+    try:
+        with open(pyldap_config_file_path) as pyldap_config_file:
+            pyldap_config.read_file(pyldap_config_file)
+        logger.debug("Read pyldap build config file")
+    except Exception as e:
+        logger.error(f"Could not read pyldap build config file {pyldap_config_file_path}. {e.strerror}. Exiting")
+        return 20
+
+    # Check for SASL requirement in pyldap build config file
+    logger.debug("Checking for '_ldap' section")
+    if not pyldap_config.has_section("_ldap"):
+        logger.error("Can't find '_ldap' section in pyldap build config file. Unable to build pyldap. Exiting")
+        return 25
+    else:
+        logger.debug("'_ldap' section found")
+
+    logger.debug("Checking for 'defines' option")
+    if not pyldap_config.has_option("_ldap", "defines"):
+        logging.error("Can't find 'defines' option in pyldap build config file. Unable to build pyldap. Exiting")
+    else:
+        logger.debug("'defines' option found")
+
+    # Remove SASL requirement if present
+    logger.debug("Removing SASL requirement")
+
+    defines_options = pyldap_config['_ldap']['defines'].split(' ')
+    build_config_updated = False
+    try:
+        defines_options.remove('HAVE_SASL')
+        pyldap_config['_ldap']['defines'] = " ".join(defines_options)
+        logger.debug("SASL requirement removed")
+        build_config_updated = True
+    except ValueError as e:
+        logger.warning("SASL requirement not found in pyldap build config file. Build config file will not be modified")
+        pass
+
+    # Write new build config
+    logger.debug("Writing new pyldap build config")
+
+    if build_config_updated:
+        try:
+            with open(pyldap_config_file_path, 'w') as pyldap_config_file:
+                pyldap_config.write(pyldap_config_file)
+            logger.debug("Wrote new pyldap build config")
+        except Exception as e:
+            logger.error(f"Could not write pyldap build config file {pyldap_config_file_path}. {e.strerror}. Exiting")
+            return 30
+
+    # Build pyldap
+    logger.debug(f"Building pyldap {pyldap_version}")
+    build_pyldap_returncode, _  = run_logged_subprocess(f"cd {BUILD_DIR_PATH} && {VENV_DIR}/bin/python3 setup.py build")
+    if build_pyldap_returncode == 0:
+        logger.info(f"Built pyldap {pyldap_version}")
+    else:
+        logger.error(f"Could not build pyldap. Exiting.")
+        return 35
+
+    # Install pyldap
+    logger.debug(f"Installing pyldap {pyldap_version}")
+    install_pyldap_returncode, _  = run_logged_subprocess(f"cd {BUILD_DIR_PATH} && {VENV_DIR}/bin/python3 setup.py install")
+    if install_pyldap_returncode == 0:
+        logger.info(f"Installed pyldap {pyldap_version} in virtual environment {VENV_NAME} at {VENV_DIR}")
+    else:
+        logger.error(f"Could not install pyldap. Exiting.")
+        return 40
+
+    return 0
+
+
+def is_valid_requirement(line: str) -> bool:
+        """Determines if line is a valid requirement
+
+        Args:
+            line (str): Line to check.
+
+        Returns:
+            bool: True if line is valid requirement. False if line is not valid requirement.
+        """
+        # Line is blank
+        if line == "\n" or line == "":
+            return False
+
+        # Line is comment
+        if line.startswith("#"):
+            return False
+
+        # Line is for pyldap
+        if line.startswith("pyldap"):
+            return False
+
+        return True
 
 
 def create_environment() -> int:
@@ -139,6 +367,8 @@ def create_environment() -> int:
         0 = Success
         5 = VENV_DIR already exists
         10 = Could not create VENV_DIR
+        11 = Could not install pyldap
+        12 = Could not read requirements file
         15 = Could not install requirements
 
     Returns:
@@ -172,6 +402,7 @@ def create_environment() -> int:
     pip_version_full = check_pip_output.split()[1]
     logger.debug(f"pip version is {pip_version_full}")
 
+    # Update pip
     pip_version_major = pip_version_full.split(".")[0]
     if int(pip_version_major) < 19:
         logger.info(f"pip verion is {pip_version_major}.x (pip >= {TARGET_PIP_VERSION}.x needed.) Upgrading pip")
@@ -181,10 +412,50 @@ def create_environment() -> int:
             logger.info(update_pip_output.split("\n")[-2])
         else:
             logger.warning("Failed to update pip. Virtual environment dependencies may not install")
-    
+
     # Install requirements
     logger.info("Installing requirements")
-    install_requirements_returncode, _ = run_logged_subprocess(f"{VENV_DIR}/bin/pip install -r {API_DIR}/requirements.txt")
+
+    # Install python-ldap
+    install_pyldap_returncode = install_custom_pyldap(VENV_INTERPRETER)
+    if install_pyldap_returncode != 0:
+        logger.error("Could not install pyldap. Exiting.")
+        return 11
+
+    # Install the rest of the requirements from the requirements file
+    logger.debug(f"Checking for venv requirements file {VENV_REQUIREMENTS_FILE}")
+    if not os.path.exists(VENV_REQUIREMENTS_FILE):
+        logger.warning(f"Could not find requirements file {VENV_REQUIREMENTS_FILE}. No requirements will be installed.")
+        return 0
+    logger.debug("Found requirements file")
+
+    # Get raw requirements from requirements file
+    logger.debug("Reading raw requirements from requirements file")
+    try:
+        with open(VENV_REQUIREMENTS_FILE) as requirements_file:
+            raw_requirements = requirements_file.readlines()
+    except Exception as e:
+        logger.warning(f"Could not read requirements file {VENV_REQUIREMENTS_FILE}. {e.strerror}. Exiting.")
+        return 12
+    logger.debug("Read raw requirements from requirements file")
+
+    # Filter and clean requirements
+    logger.debug("Validating requirements")
+    valid_requirements = []
+    for requirement in raw_requirements:
+        if is_valid_requirement(requirement):
+            valid_requirements.append(requirement.strip())
+    logger.debug("Validated requirements")
+
+    # Generate requirements string
+    logger.debug("Generating requirements string")
+    requirements_string = ""
+    for requirement in valid_requirements:
+        requirements_string += f"'{requirement}' "
+    logger.debug(f"Generated requirements string {requirements_string}")
+
+    # Install requirements 
+    install_requirements_returncode, _ = run_logged_subprocess(f"{VENV_DIR}/bin/pip install {requirements_string}")
     if install_requirements_returncode == 0:
         logger.info("Successfully installed requirements")
         return 0