Add custom python-ldap build step to installation

python_ldap_manager.py

@@ -0,0 +1,264 @@
+"""Builds python-ldap from source without SASL dependencies.
+
+Exit Codes:
+    30 = VENV_INTERPRETER does not exist.
+    35 = Failed to get pyldap release info from GitHub.
+    40 = Failed to download pyldap source.
+    45 = Failed to extract pyldap source.
+    50 = Failed to read pyldap build config file.
+    55 = Failed to write pyldap build config file.
+    60 = Failed to build pyldap VERSION.
+    65 = Failed to install pyldap VERSION.
+"""
+
+from pathlib import Path
+import os, logging, subprocess, urllib.request, json, configparser
+from urllib.error import HTTPError, URLError
+from typing import Union
+
+
+################################################################################
+# Configuration
+################################################################################
+
+# Configure the logger
+logger_name = "python-ldap-manager"
+logger = logging.getLogger(logger_name)
+logger.setLevel(logging.DEBUG)
+
+# See Formatting Details: https://docs.python.org/3/library/logging.html#logrecord-attributes
+# Example: Jan 28 2021 12:19:28 venv-manager : [INFO] Message
+log_message_format = "%(asctime)s %(name)s : [%(levelname)s] %(message)s"
+# See Time Formatting Details: https://docs.python.org/3.6/library/time.html#time.strftime
+# Example: Jan 28 2021 12:19:28
+log_time_format = "%b %d %Y %H:%M:%S"
+log_formatter = logging.Formatter(log_message_format, log_time_format)
+
+# Configure output to stdout
+stream_handler = logging.StreamHandler()
+stream_handler.setFormatter(log_formatter)
+logger.addHandler(stream_handler)
+
+
+################################################################################
+# Functions
+################################################################################
+
+def run_logged_subprocess(command: Union[str, list], timeout: int = 60, shell: bool = True) -> tuple:
+    """Executes a shell command using subprocess with logging.
+
+    stderr is redirected to stdout and stdout is pipped to logger.
+    If the subprocess raises an exception, the exception is logged as critical.
+
+    Example:
+        Running a successful command:
+            run_logged_subprocess(command=["git", "commit", "-m", "'Commit message.'"])
+            Returns: (0, "")
+
+        Running an unsuccessful shell command with a 20 second timeout:
+            run_logged_subprocess(command="cd test/", timeout=20, shell=True)
+            Returns: (1, "cd: test: No such file or directory\n")
+
+    Args:
+        command (Union): The command to run. If shell=False, pass a list with the first item being the command and the subsequent items being arguments.  If shell=True, pass a string as you would type it into a shell.
+        timeout (int): The number of seconds to wait for a program before killing it. Defaults to 60.
+
+    Returns:
+        tuple: With the first value being the return code and second being the combined stdout+stderr
+    """
+    logger.debug(f"Entering subprocess for '{command}'")
+    with subprocess.Popen(command,\
+        stderr=subprocess.STDOUT, stdout=subprocess.PIPE, shell=shell, universal_newlines=True)\
+        as logged_shell_process:
+
+        subprocess_log_prefix = f"(PID: {logged_shell_process.pid})"
+
+        try:
+            # Convert combined stdout and stderr stream to list of strings
+            process_output_stream, _ =  logged_shell_process.communicate(timeout=timeout)
+            process_output_lines = process_output_stream.split("\n")
+            # Remove last entry in process_output_lines because it is always empty
+            process_output_lines.pop(-1)
+
+            for line in process_output_lines:
+                logger.debug(f"{subprocess_log_prefix}: {line}")
+        except Exception as exception:
+            logger.critical(str(exception))
+        else:
+            if logged_shell_process.returncode != 0:
+                logger.debug(f"Something went wrong. '{command}' exited with return code {logged_shell_process.returncode}")
+            elif logged_shell_process.returncode == 0:
+                logger.debug(f"Subprocess for '{command}' completed successfuly")
+        finally:
+            logger.debug(f"Exiting subprocess for '{command}'")
+            return (logged_shell_process.returncode, process_output_stream)
+
+
+def install_custom_pyldap(venv_interpreter: str, pyldap_version: str = None) -> int:
+    """Builds python-ldap without SASL support from GitHub source.
+
+    Args:
+        venv_interpreter (str): The absolute path to the python interpreter executable for the virtual environment.
+        pyldap_version (str): The version of python-ldap to install. Must be a valid python-ldap GitHub tag. If not provided, the latest non-beta version of python-ldap is used. (Default: None)
+
+    Returns:
+        int: Exit code.
+    """
+    logger.info("Starting pyldap build process")
+
+    # Check for valid venv interpreter
+    logger.debug(f"Checking for valid venv interpreter at {venv_interpreter}")
+    if not os.path.exists(Path(venv_interpreter)):
+        logger.error(f"venv interpreter does not exist. Exiting")
+        exit(30)
+    logger.debug(f"venv interpreter is valid")
+
+    # Get list of release tags for pyldap from GitHub API
+    logger.debug(f"Getting pyldap tags from GitHub")
+    pyldap_github_tags_url = "https://api.github.com/repos/python-ldap/python-ldap/tags"
+    try:
+        with urllib.request.urlopen(pyldap_github_tags_url) as request:
+            pyldap_tags = json.loads(request.read().decode("utf-8"))
+    except HTTPError as e:
+        logger.error(f"Failed to connect to {pyldap_github_tags_url}. Got response {e.code} {e.msg}. Exiting")
+        exit(35)
+    except URLError as e:
+        logger.error(f"Could not connect to {pyldap_github_tags_url}. {e.reason}. Exiting")
+        exit(35)
+    logger.debug(f"Got {len(pyldap_tags)} pyldap tags from GitHub")
+
+    # Build dictionary of available pyldap releases and their source code archive urls
+    # Example:
+    #   { "name": "python-ldap-3.3.1", "zipball_url": "http://github.com/download" } becomes
+    #   { "3.3.1": "http://github.com/download" }
+    logger.debug("Building list of pyldap versions.")
+    pyldap_versions = {}
+    for tag in pyldap_tags:
+        tag_version = tag["name"].split("-")[-1]
+        zipball_url = f"https://github.com/python-ldap/python-ldap/archive/python-ldap-{tag_version}.zip"
+        pyldap_versions[tag_version] = zipball_url
+    logger.debug(f"Built list of {len(pyldap_versions)} pyldap versions.")
+
+    # Set pyldap version to value from from argument if valid and available
+    if pyldap_version and pyldap_version in pyldap_versions.keys():
+        logger.debug(f"pyldap version {pyldap_version} is available from GitHub")
+    # Set to latest non-beta version
+    else:
+        logger.warning(f"pyldap version not found in arguments file. Defaulting to latest non-beta release on GitHub")
+        for version in pyldap_versions.keys():
+            is_beta_version = "b" in version
+            if (not is_beta_version):
+                pyldap_version = version
+                break
+        logger.info(f"Set pyldap version to {pyldap_version} (from GitHub releases)")
+
+    # Download pyldap soure code
+    logger.info(f"Downloading pyldap {pyldap_version} source from {pyldap_versions[pyldap_version]}")
+
+    tmp_dir = "/tmp"
+    download_file_name = f"python-ldap-{pyldap_version}.zip"
+    download_file_path = Path(tmp_dir, download_file_name)
+
+    download_pyldap_returncode, _ = run_logged_subprocess(f"wget -q -O {download_file_path} {pyldap_versions[pyldap_version]}")
+    if download_pyldap_returncode == 0:
+        logger.debug(f"Downloaded pyldap {pyldap_version} source to {download_file_path}")
+    else:
+        logger.error(f"Failed to download pyldap source.")
+        exit(40)
+
+    # Extract source code
+
+    # The archive from GitHub has a root folder formatted 'user-repo-version'. 
+    # Because the pyldap source is user 'python-ldap' and repo 'python-ldap' 
+    # the build folder MUST be the following format:
+    BUILD_DIR_NAME = f"python-ldap-python-ldap-{pyldap_version}"
+    BUILD_DIR_PATH = Path(tmp_dir, BUILD_DIR_NAME)
+
+    logger.info(f"Extracing pyldap {pyldap_version} source to {BUILD_DIR_PATH}")
+    extract_source_returncode, _ = run_logged_subprocess(f"unzip -q -o -d {tmp_dir} {download_file_path}")
+    if extract_source_returncode == 0:
+        logger.debug(f"Extracted pyldap source to {BUILD_DIR_PATH}")
+    else:
+        logger.error(f"Failed to extract pyldap source. Exiting")
+        exit(45)
+
+    # Start the build process
+    logger.info(f"Building pyldap {pyldap_version}")
+
+    # Read the pyldap build config file
+    pyldap_config_file_name = "setup.cfg"
+    pyldap_config_file_path = Path(BUILD_DIR_PATH, pyldap_config_file_name)
+    pyldap_version_from_needs_updated = True
+
+    logger.debug(f"Reading pyldap build config file {pyldap_config_file_path}")
+    pyldap_config = configparser.ConfigParser()
+    try:
+        with open(pyldap_config_file_path) as pyldap_config_file:
+            pyldap_config.read_file(pyldap_config_file)
+        logger.debug("Read pyldap build config file")
+    except Exception as e:
+        logger.error(f"Failed to read pyldap build config file {pyldap_config_file_path}. {e}. Exiting")
+        exit(50)
+
+    # Check for SASL requirement in pyldap build config file
+    logger.debug("Checking for '_ldap' section")
+    if not pyldap_config.has_section("_ldap"):
+        logger.warning("Failed to find '_ldap' section in pyldap build config file. pyldap may fail to build")
+        pyldap_version_from_needs_updated = False
+        pass
+    else:
+        logger.debug("'_ldap' section found")
+
+    logger.debug("Checking for 'defines' option")
+    if not pyldap_config.has_option("_ldap", "defines"):
+        logging.warning("Failed to find 'defines' option in pyldap build config file. pyldap may fail to build")
+        pyldap_version_from_needs_updated = False
+    else:
+        logger.debug("'defines' option found")
+
+    # Remove SASL requirement if present
+    if pyldap_version_from_needs_updated:
+        logger.debug("Removing SASL requirement")
+
+        defines_options = pyldap_config['_ldap']['defines'].split(' ')
+        build_config_updated = False
+        try:
+            defines_options.remove('HAVE_SASL')
+            pyldap_config['_ldap']['defines'] = " ".join(defines_options)
+            logger.debug("SASL requirement removed")
+            build_config_updated = True
+        except ValueError as e:
+            logger.warning("SASL requirement not found in pyldap build config file. Build config file will not be modified")
+            pass
+
+    # Write new build config
+    logger.debug("Writing new pyldap build config")
+    if build_config_updated:
+        try:
+            with open(pyldap_config_file_path, 'w') as pyldap_config_file:
+                pyldap_config.write(pyldap_config_file)
+            logger.debug("Wrote new pyldap build config")
+        except Exception as e:
+            logger.error(f"Failed to write pyldap build config file {pyldap_config_file_path}. {e}. Exiting")
+            exit(55)
+
+    # Build pyldap
+    logger.debug(f"Building pyldap {pyldap_version}")
+    build_pyldap_returncode, _  = run_logged_subprocess(f"cd {BUILD_DIR_PATH} && {venv_interpreter} setup.py build")
+    if build_pyldap_returncode == 0:
+        logger.debug(f"Built pyldap {pyldap_version}")
+    else:
+        logger.error(f"Failed to build pyldap {pyldap_version}. Exiting")
+        exit(60)
+
+    # Install pyldap
+    logger.debug(f"Installing pyldap {pyldap_version}.")
+    install_pyldap_returncode, _  = run_logged_subprocess(f"cd {BUILD_DIR_PATH} && {venv_interpreter} setup.py install")
+    if install_pyldap_returncode == 0:
+        logger.debug(f"Installed pyldap {pyldap_version}")
+    else:
+        logger.error(f"Failed to install pyldap {pyldap_version}. Exiting")
+        exit(65)
+
+    logger.info(f"Finshed installing pyldap {pyldap_version}")
+    return 0

setup.py

@@ -1,6 +1,12 @@
-import setuptools
+import setuptools, sys
+from pathlib import Path
+import python_ldap_manager
 
 VERSION = "2.1.0"
+python_ldap_version = "3.3.1"
+
+# Build and install python-ldap without SASL requirements
+python_ldap_manager.install_custom_pyldap(sys.executable, python_ldap_version=python_ldap_version)
 
 setuptools.setup(
     name="webqueue-api",