From b91f0d9fdf5403eb5679c6588d16b911213e274c Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Tue, 2 Feb 2021 19:02:01 -0500 Subject: [PATCH] Add global exit codes with docs --- utils/venv-manager.py | 215 ++++++++++++++++++++---------------------- 1 file changed, 102 insertions(+), 113 deletions(-) diff --git a/utils/venv-manager.py b/utils/venv-manager.py index d6ddd53..051f7eb 100644 --- a/utils/venv-manager.py +++ b/utils/venv-manager.py @@ -9,6 +9,23 @@ Reset a virtual environment: $ venv-manager.py reset + +Exit Codes: + 0 = Success + 5 = Failed to make directory VENV_DIR. It already exists. + 10 = Failed to make VENV_DIR. Incorrect permissions. + 15 = Failed to create virtual environment. See LOG FILE. + 20 = Failed read requirements file VENV_REQUIREMENTS_FILE. + 25 = Failed to install requirements. See LOG FILE. + 30 = VENV_INTERPRETER does not exist. + 35 = Failed to get pyldap release info from GitHub. + 40 = Failed to download pyldap source. See LOG FILE. + 45 = Failed to extract pyldap source. See LOG FILE. + 50 = Failed to read pyldap build config file. + 55 = Failed to write pyldap build config file. + 60 = Failed to build pyldap VERSION. See LOG FILE. + 65 = Failed to install pyldap VERSION. See LOG FILE. + 70 = Failed to delete VENV_DIR """ from pathlib import Path @@ -145,32 +162,19 @@ def install_custom_pyldap(venv_interpreter: str) -> int: - Version from requirements.txt (can be beta) - Latest non-beta version from GitHub - Exit Codes: - 0 = Success - 1 = Virtual environment interpreter does not exist - 5 = Could not complete GitHub API call - 10 = Could not download python-ldap source - 15 = Could not extract python-ldap source - 20 = Could not open pyldap build config file - 25 = pyldap build configuration file improperly formatted - 30 = Could not write pyldap build config file - 35 = Could not build pyldap - 40 = Could not install pyldap - Args: venv_interpreter (str): The absolute path to the python interpreter executable for the virtual environment. Returns: int: Exit code. """ - logger.info("Starting pyldap build process") # Check for valid venv interpreter logger.debug(f"Checking for valid venv interpreter at {venv_interpreter}") if not os.path.exists(Path(venv_interpreter)): - logger.error(f"The interpreter for virtual environment {VENV_NAME} does not exist at {venv_interpreter}. Exiting.") - return 1 + logger.error(f"venv interpreter does not exist. Exiting") + exit(30) logger.debug(f"venv interpreter is valid") # Get list of release tags for pyldap from GitHub API @@ -180,11 +184,11 @@ def install_custom_pyldap(venv_interpreter: str) -> int: with urllib.request.urlopen(pyldap_github_tags_url) as request: pyldap_tags = json.loads(request.read().decode("utf-8")) except HTTPError as e: - logger.error(f"Could not connect to {pyldap_github_tags_url}. Got response {e.code} {e.msg}. Exiting") - return 5 + logger.error(f"Failed to connect to {pyldap_github_tags_url}. Got response {e.code} {e.msg}. Exiting") + exit(35) except URLError as e: logger.error(f"Could not connect to {pyldap_github_tags_url}. {e.reason}. Exiting") - return 5 + exit(35) logger.debug(f"Got {len(pyldap_tags)} pyldap tags from GitHub") # Build dictionary of available pyldap releases and their source code archive urls @@ -197,11 +201,11 @@ def install_custom_pyldap(venv_interpreter: str) -> int: tag_version = tag["name"].split("-")[-1] zipball_url = f"https://github.com/python-ldap/python-ldap/archive/python-ldap-{tag_version}.zip" pyldap_versions[tag_version] = zipball_url - logger.debug("Built list of pyldap versions.") + logger.debug(f"Built list of {len(pyldap_versions)} pyldap versions.") # Check requirements file for pyldap version pyldap_version_from_requirements = "" - logger.info(f"Checking for pyldap version in requirements file {VENV_REQUIREMENTS_FILE}") + logger.debug(f"Checking for pyldap version in requirements file {VENV_REQUIREMENTS_FILE}") try: with open(VENV_REQUIREMENTS_FILE) as requirements_file: for line in requirements_file: @@ -217,7 +221,7 @@ def install_custom_pyldap(venv_interpreter: str) -> int: if pyldap_version_from_requirements and pyldap_version_from_requirements in pyldap_versions.keys(): logger.debug(f"pyldap version {pyldap_version_from_requirements} is available from GitHub") pyldap_version = pyldap_version_from_requirements - logger.info(f"Set pyldap version to {pyldap_version}") + logger.info(f"Set pyldap version to {pyldap_version} (from requirements file)") # Set to latest non-beta version else: logger.warning(f"pyldap version not found in requirements file. Defaulting to latest non-beta release on GitHub") @@ -226,7 +230,7 @@ def install_custom_pyldap(venv_interpreter: str) -> int: if (not is_beta_version): pyldap_version = version break - logger.info(f"Set pyldap version to {pyldap_version}") + logger.info(f"Set pyldap version to {pyldap_version} (from GitHub releases)") # Download pyldap soure code logger.info(f"Downloading pyldap {pyldap_version} source from {pyldap_versions[pyldap_version]}") @@ -237,13 +241,12 @@ def install_custom_pyldap(venv_interpreter: str) -> int: download_pyldap_returncode, _ = run_logged_subprocess(f"wget -q -O {download_file_path} {pyldap_versions[pyldap_version]}") if download_pyldap_returncode == 0: - logger.debug(f"pyldap source downloaded to {download_file_path}") + logger.debug(f"Downloaded pyldap {pyldap_version} source to {download_file_path}") else: - logger.error(f"Could not download pyldap source. Exiting.") - return 10 + logger.error(f"Failed to download pyldap source. See {log_file_path}. Exiting") + exit(40) # Extract source code - logger.info("Extracing pyldap source") # The archive from GitHub has a root folder formatted 'user-repo-version'. # Because the pyldap source is user 'python-ldap' and repo 'python-ldap' @@ -251,12 +254,13 @@ def install_custom_pyldap(venv_interpreter: str) -> int: BUILD_DIR_NAME = f"python-ldap-python-ldap-{pyldap_version}" BUILD_DIR_PATH = Path(tmp_dir, BUILD_DIR_NAME) + logger.info(f"Extracing pyldap {pyldap_version} source to {BUILD_DIR_PATH}") extract_source_returncode, _ = run_logged_subprocess(f"unzip -q -o -d {tmp_dir} {download_file_path}") if extract_source_returncode == 0: - logger.debug(f"pyldap source extracted to {BUILD_DIR_PATH}") + logger.debug(f"Extracted pyldap source to {BUILD_DIR_PATH}") else: - logger.error(f"Could not extract pyldap source. Exiting") - return 15 + logger.error(f"Failed to extract pyldap source. See {log_file_path}. Exiting") + exit(45) # Start the build process logger.info(f"Building pyldap {pyldap_version}") @@ -264,6 +268,7 @@ def install_custom_pyldap(venv_interpreter: str) -> int: # Read the pyldap build config file pyldap_config_file_name = "setup.cfg" pyldap_config_file_path = Path(BUILD_DIR_PATH, pyldap_config_file_name) + pyldap_version_from_needs_updated = True logger.debug(f"Reading pyldap build config file {pyldap_config_file_path}") pyldap_config = configparser.ConfigParser() @@ -272,67 +277,70 @@ def install_custom_pyldap(venv_interpreter: str) -> int: pyldap_config.read_file(pyldap_config_file) logger.debug("Read pyldap build config file") except Exception as e: - logger.error(f"Could not read pyldap build config file {pyldap_config_file_path}. {e.strerror}. Exiting") - return 20 + logger.error(f"Failed to read pyldap build config file {pyldap_config_file_path}. {e}. Exiting") + exit(50) # Check for SASL requirement in pyldap build config file logger.debug("Checking for '_ldap' section") if not pyldap_config.has_section("_ldap"): - logger.error("Can't find '_ldap' section in pyldap build config file. Unable to build pyldap. Exiting") - return 25 + logger.warning("Failed to find '_ldap' section in pyldap build config file. pyldap may fail to build") + pyldap_version_from_needs_updated = False + pass else: logger.debug("'_ldap' section found") logger.debug("Checking for 'defines' option") if not pyldap_config.has_option("_ldap", "defines"): - logging.error("Can't find 'defines' option in pyldap build config file. Unable to build pyldap. Exiting") + logging.warning("Failed to find 'defines' option in pyldap build config file. pyldap may fail to build") + pyldap_version_from_needs_updated = False else: logger.debug("'defines' option found") # Remove SASL requirement if present - logger.debug("Removing SASL requirement") + if pyldap_version_from_needs_updated: + logger.debug("Removing SASL requirement") - defines_options = pyldap_config['_ldap']['defines'].split(' ') - build_config_updated = False - try: - defines_options.remove('HAVE_SASL') - pyldap_config['_ldap']['defines'] = " ".join(defines_options) - logger.debug("SASL requirement removed") - build_config_updated = True - except ValueError as e: - logger.warning("SASL requirement not found in pyldap build config file. Build config file will not be modified") - pass + defines_options = pyldap_config['_ldap']['defines'].split(' ') + build_config_updated = False + try: + defines_options.remove('HAVE_SASL') + pyldap_config['_ldap']['defines'] = " ".join(defines_options) + logger.debug("SASL requirement removed") + build_config_updated = True + except ValueError as e: + logger.warning("SASL requirement not found in pyldap build config file. Build config file will not be modified") + pass # Write new build config logger.debug("Writing new pyldap build config") - if build_config_updated: try: with open(pyldap_config_file_path, 'w') as pyldap_config_file: pyldap_config.write(pyldap_config_file) logger.debug("Wrote new pyldap build config") except Exception as e: - logger.error(f"Could not write pyldap build config file {pyldap_config_file_path}. {e.strerror}. Exiting") - return 30 + logger.error(f"Failed to write pyldap build config file {pyldap_config_file_path}. {e}. Exiting") + exit(55) # Build pyldap logger.debug(f"Building pyldap {pyldap_version}") build_pyldap_returncode, _ = run_logged_subprocess(f"cd {BUILD_DIR_PATH} && {VENV_DIR}/bin/python3 setup.py build") if build_pyldap_returncode == 0: - logger.info(f"Built pyldap {pyldap_version}") + logger.debug(f"Built pyldap {pyldap_version}") else: - logger.error(f"Could not build pyldap. Exiting.") - return 35 + logger.error(f"Failed to build pyldap {pyldap_version}. See {log_file_path}. Exiting") + exit(60) # Install pyldap - logger.debug(f"Installing pyldap {pyldap_version}") + logger.debug(f"Installing pyldap {pyldap_version} in virtual environment {VENV_NAME} at {VENV_DIR}") install_pyldap_returncode, _ = run_logged_subprocess(f"cd {BUILD_DIR_PATH} && {VENV_DIR}/bin/python3 setup.py install") if install_pyldap_returncode == 0: - logger.info(f"Installed pyldap {pyldap_version} in virtual environment {VENV_NAME} at {VENV_DIR}") + logger.debug(f"Installed pyldap {pyldap_version}") else: - logger.error(f"Could not install pyldap. Exiting.") - return 40 + logger.error(f"Failed to install pyldap {pyldap_version}. See {log_file_path}. Exiting") + exit(65) + logger.info(f"Finshed installing pyldap {pyldap_version}") return 0 @@ -363,14 +371,6 @@ def is_valid_requirement(line: str) -> bool: def create_environment() -> int: """Creates a virtual environment for webqueue2 - Exit Codes: - 0 = Success - 5 = VENV_DIR already exists - 10 = Could not create VENV_DIR - 11 = Could not install pyldap - 12 = Could not read requirements file - 15 = Could not install requirements - Returns: int: Exit code """ @@ -378,65 +378,68 @@ def create_environment() -> int: logger.info(f"Creating virtual environment {VENV_NAME} at {VENV_DIR}") # Check for an existing virtual environment + logger.debug(f"Creating virtual environment directory at {VENV_DIR}") try: os.mkdir(VENV_DIR) except FileExistsError: - logger.warning(f"The directory {VENV_DIR} already exists. Exiting") - return 5 + logger.error(f"Failed to make directory {VENV_DIR}. It already exists. Exiting") + exit(5) + except PermissionError: + logger.error(f"Failed to make directory {VENV_DIR}. Incorrect permissions. Exiting") + exit(10) + logger.debug(f"Created virtual environment directory") + - # Create virtual environmentc + # Create virtual environment + logger.debug(f"Creating virtual environment {VENV_NAME} at {VENV_DIR}") create_env_returncode, _ = run_logged_subprocess(f"cd {API_DIR} && python3 -m venv {VENV_NAME}", shell=True) if create_env_returncode == 0: - logger.info(f"Virtual environment {VENV_NAME} created at {VENV_DIR}") + logger.info(f"Successfully created virtual environment {VENV_NAME} created at {VENV_DIR}") else: - logger.critical(f"Could not create virtual environment {VENV_NAME} at {VENV_DIR}. Exiting") - return 10 + logger.error(f"Failed to create virtual environment. See {log_file_path}. Exiting") + exit(15) # Check pip version logger.debug("Checking pip version") check_pip_returncode, check_pip_output = run_logged_subprocess(f"{VENV_DIR}/bin/pip --version") - - if check_pip_returncode != 0: - logger.warning("Could not check pip version. Virtual environment dependencies may not install") - - pip_version_full = check_pip_output.split()[1] - logger.debug(f"pip version is {pip_version_full}") + if check_pip_returncode == 0: + pip_version_full = check_pip_output.split()[1] + logger.debug(f"pip version is {pip_version_full}") + else: + logger.warning("Failed to check pip version. Virtual environment dependencies may not install") - # Update pip + # Update pip if needed pip_version_major = pip_version_full.split(".")[0] if int(pip_version_major) < 19: - logger.info(f"pip verion is {pip_version_major}.x (pip >= {TARGET_PIP_VERSION}.x needed.) Upgrading pip") + logger.debug(f"pip verion is {pip_version_major}.x (pip >= {TARGET_PIP_VERSION}.x needed.) Upgrading pip") update_pip_returncode, update_pip_output = run_logged_subprocess(f"{VENV_DIR}/bin/pip install --upgrade pip") - if update_pip_returncode == 0: - logger.info(update_pip_output.split("\n")[-2]) + pip_install_message = update_pip_output.split("\n")[-2] + logger.debug(pip_install_message) else: logger.warning("Failed to update pip. Virtual environment dependencies may not install") # Install requirements - logger.info("Installing requirements") + logger.info("Installing virtual environment requirements") # Install python-ldap - install_pyldap_returncode = install_custom_pyldap(VENV_INTERPRETER) - if install_pyldap_returncode != 0: - logger.error("Could not install pyldap. Exiting.") - return 11 + install_custom_pyldap(VENV_INTERPRETER) # Install the rest of the requirements from the requirements file logger.debug(f"Checking for venv requirements file {VENV_REQUIREMENTS_FILE}") if not os.path.exists(VENV_REQUIREMENTS_FILE): - logger.warning(f"Could not find requirements file {VENV_REQUIREMENTS_FILE}. No requirements will be installed.") + logger.warning(f"Could not find requirements file {VENV_REQUIREMENTS_FILE}. No requirements will be installed") return 0 logger.debug("Found requirements file") # Get raw requirements from requirements file - logger.debug("Reading raw requirements from requirements file") + logger.debug(f"Reading raw requirements from requirements file {VENV_REQUIREMENTS_FILE}") try: with open(VENV_REQUIREMENTS_FILE) as requirements_file: raw_requirements = requirements_file.readlines() except Exception as e: - logger.warning(f"Could not read requirements file {VENV_REQUIREMENTS_FILE}. {e.strerror}. Exiting.") - return 12 + logger.warning(f"Failed read requirements file {VENV_REQUIREMENTS_FILE}. {e}. Exiting") + exit(20) logger.debug("Read raw requirements from requirements file") # Filter and clean requirements @@ -445,7 +448,7 @@ def create_environment() -> int: for requirement in raw_requirements: if is_valid_requirement(requirement): valid_requirements.append(requirement.strip()) - logger.debug("Validated requirements") + logger.debug(f"Validated {len(valid_requirements)} requirements") # Generate requirements string logger.debug("Generating requirements string") @@ -454,23 +457,22 @@ def create_environment() -> int: requirements_string += f"'{requirement}' " logger.debug(f"Generated requirements string {requirements_string}") - # Install requirements + # Install requirements + logger.debug("Installing requirements") install_requirements_returncode, _ = run_logged_subprocess(f"{VENV_DIR}/bin/pip install {requirements_string}") if install_requirements_returncode == 0: logger.info("Successfully installed requirements") - return 0 else: - logger.critical("Failed to install requirements. Exiting") - return 15 + logger.error(f"Failed to install requirements. See {log_file_path}. Exiting") + exit(25) + + logger.info("Finished creating virtual environment") + return 0 def delete_environment() -> int: """Deletes a virtual environment for webqueue2 - Exit Codes: - 0 = Success - 5 = Could not delete VENV_DIR - Returns: int: Exit code """ @@ -481,34 +483,21 @@ def delete_environment() -> int: logger.info(f"Successfully deleted virtual environment {VENV_NAME} at {VENV_DIR}") return 0 else: - logger.critical(f"Failed to delete virtual environment {VENV_NAME} at {VENV_DIR}. Exiting") - return 5 + logger.error(f∆"Failed to delete virtual environment {VENV_NAME} at {VENV_DIR}. Exiting") + exit(70) def reset_environment() -> int: """Resets a virtual environment for webqueue2 - Exit Codes: - 0 = Success - 5 = Could not delete VENV_DIR - 10 = Could not create VENV_DIR - Returns: int: Exit code """ logger.info(f"Resetting virtual environment {VENV_NAME} at {VENV_DIR}") - delete_returncode = delete_environment() - if delete_returncode != 0: - logger.critical(f"Failed to reset virtual environment {VENV_NAME} at {VENV_DIR}. Exiting") - return 5 - create_returncode = create_environment() - if create_returncode != 0: - logger.critical(f"Failed to reset virtual environment {VENV_NAME} at {VENV_DIR}. Exiting") - return 10 - logger.info(f"Successfully reset virtual environment {VENV_NAME} at {VENV_DIR}. Exiting") + return 0 if __name__ == "__main__":