From e634be14d9f3f915983b6536955b2ae0718ec655 Mon Sep 17 00:00:00 2001 From: benne238 Date: Mon, 28 Jun 2021 12:45:26 -0400 Subject: [PATCH 01/56] Remove old ECNQueue module --- webqueue2_api/ECNQueue.py | 1499 ------------------------------------- webqueue2_api/__init__.py | 1 - webqueue2_api/api.py | 272 ------- 3 files changed, 1772 deletions(-) delete mode 100644 webqueue2_api/ECNQueue.py delete mode 100644 webqueue2_api/__init__.py delete mode 100644 webqueue2_api/api.py diff --git a/webqueue2_api/ECNQueue.py b/webqueue2_api/ECNQueue.py deleted file mode 100644 index 4fbcd65..0000000 --- a/webqueue2_api/ECNQueue.py +++ /dev/null @@ -1,1499 +0,0 @@ -"""A library for interacting with Purdue ECN's ticketing system. - -This library allows interacting with queue Items (called Items) and collections -of items (called Queues). - -Example: - # Create a single Item (ce100) - >>> item = Item("ce", 100) - # Get the sender's email address from an Item - >>> item = Item("ce", 100) - >>> item.userEmail - - # Create an entire Queue (ce) - >>> queue = Queue("ce") - # Get the number of items in a Queue - >>> queue = Queue("ce") - >>> numItems = len(queue) - - # Get all queues (and their items) - >>> queues = getQueues() - -Attributes: - queueDirectory: The directory to load queues from. - queuesToIgnore: Queues that will not be loaded when running getQueues() - -Raises: - # TODO: Add description(s) of when a ValueError is raised. - ValueError: [description] -""" - -#------------------------------------------------------------------------------# -# Imports -#------------------------------------------------------------------------------# -import os -import time -import email -import re -import datetime -from dateutil.parser import parse -from dateutil import tz -from typing import Union -import json - - -#------------------------------------------------------------------------------# -# Configuration -#------------------------------------------------------------------------------# - -# The directory where queue items are -currentFilePath = __file__ -currentFileDirectory = os.path.dirname(currentFilePath) -currentFileDirectoryParent = os.path.dirname(currentFileDirectory) -queueDirectory = os.path.join(currentFileDirectoryParent, "q-snapshot") - -# Queues to not load in getQueues() -queuesToIgnore = ["archives", "drafts", "inbox", "coral"] - - - -#------------------------------------------------------------------------------# -# Utilities -#------------------------------------------------------------------------------# - -def isValidItemName(name: str) -> bool: - """Returns true if file name is a valid item name - - Example: - isValidItemName("21") -> true - isValidItemName("twentyone") -> false - - Args: - name (str): The name to test. - - Returns: - bool: Name is valid item name. - """ - itemPattern = re.compile("^[0123456789]{1,3}$") - return True if itemPattern.match(name) else False - - - -#------------------------------------------------------------------------------# -# Classes -#------------------------------------------------------------------------------# -class Item: - """A single issue. - - **Example:** - - ``` - # Create an Item (ce100) - >>> item = Item("ce", 100) - ``` - - **Attributes:** - - ``` - lastUpdated: An ISO 8601 formatted time string showing the last time the file was updated according to the filesystem. - headers: A list of dictionaries containing header keys and values. - content: A list of section dictionaries. - isLocked: A boolean showing whether or not a lockfile for the item is present. - userEmail: The email address of the person who this item is from. - userName: The real name of the person who this item is from. - userAlias: The Purdue career account alias of the person this item is from. - assignedTo: The Purdue career account alias of the person this item is assigned to - subject: The subject of the original message for this item. - status: The most recent status update for the item. - priority: The most recent priority for this item. - department: The most recent department for this item. - dateReceived: The date this item was created. - jsonData: A JSON serializable representation of the Item. - ``` - """ - - def __init__(self, queue: str, number: int) -> None: - self.queue = queue - try: - self.number = int(number) - except ValueError: - raise ValueError(" Could not convert \"" + - number + "\" to an integer") - - self.__path = "/".join([queueDirectory, self.queue, str(self.number)]) - self.lastUpdated = self.__getLastUpdated() - self.__rawItem = self.__getRawItem() - self.headers = self.__parseHeaders() - self.content = self.__parseSections() - self.isLocked = self.__isLocked() - self.userEmail = self.__parseFromData(data="userEmail") - self.userName = self.__parseFromData(data="userName") - self.userAlias = self.__getUserAlias() - self.assignedTo = self.__getMostRecentHeaderByType("Assigned-To") - self.subject = self.__getMostRecentHeaderByType("Subject") - self.status = self.__getMostRecentHeaderByType("Status") - self.priority = self.__getMostRecentHeaderByType("Priority") - self.department = self.__getMostRecentHeaderByType("Department") - self.building = self.__getMostRecentHeaderByType("Building") - self.dateReceived = self.__getFormattedDate( - self.__getMostRecentHeaderByType("Date")) - - # TODO: Autopopulate jsonData w/ __dir__() command. Exclude `^_` and `jsonData`. - self.jsonData = { - "queue": self.queue, - "number": self.number, - "lastUpdated": self.lastUpdated, - "headers": self.headers, - "content": self.content, - "isLocked": self.isLocked, - "userEmail": self.userEmail, - "userName": self.userName, - "userAlias": self.userAlias, - "assignedTo": self.assignedTo, - "subject": self.subject, - "status": self.status, - "priority": self.priority, - "department": self.department, - "building": self.building, - "dateReceived": self.dateReceived - } - - def __getLastUpdated(self) -> str: - """Returns last modified time of item reported by the filesystem in mm-dd-yy hh:mm am/pm format. - - **Example:** - ``` - 07-23-20 10:34 AM - ``` - - **Returns:** - ``` - str: last modified time of item reported by the filesystem in mm-dd-yy hh:mm am/pm format. - ``` - """ - # TODO: Simplify this code block by allowing __getFormattedDate to accept milliseconds since the epoch. - unixTime = os.path.getmtime(self.__path) - formattedTime = time.strftime( - '%m-%d-%y %I:%M %p', time.localtime(unixTime)) - return self.__getFormattedDate(formattedTime) - - def __getRawItem(self) -> list: - """Returns a list of all lines in the item file - - **Returns:** - ``` - list: List of all the lines in the item file - ``` - """ - with open(self.__path, errors="replace") as file: - return file.readlines() - - def __getHeaderBoundary(self) -> int: - """Returns the 0 based line number where the Item headers stop. - - **Example:** - ``` - The header end would be on line 13 - 12: X-ECN-Queue-Original-URL: - 13: - 14: I need help. - ``` - - **Returns:** - ``` - int: line number where the Item headers end - ``` - """ - for lineNumber, line in enumerate(self.__rawItem): - if line == "\n": - return lineNumber - - def __parseHeaders(self) -> list: - """Returns a list containing dictionaries of header type and data. - Removes queue prefixes and whitespace. - - **Examples:** - ``` - "[ce] QStatus: Dont Delete\\nFrom: Justin Campbell \\n" - becomes - [ - {"QStatus": "Don't Delete"}, - {"From": "Justin Campbell "} - ] - ``` - - **Returns:** - ``` - list: Header dicts - ``` - """ - headerString = "" - - # Remove '[queue] ' prefixes: - # Example: - # [ce] QTime-Updated-By: campb303 becomes - # QTime-Updated-By: campb303 - queuePrefixPattern = re.compile(r"\[.*?\] {1}") - for lineNumber in range(self.__getHeaderBoundary()): - line = self.__rawItem[lineNumber] - lineHasQueuePrefix = queuePrefixPattern.match(line) - - if lineHasQueuePrefix: - queuePrefix = line[lineHasQueuePrefix.regs[0] - [0]: lineHasQueuePrefix.regs[0][1]] - line = line.replace(queuePrefix, "") - - headerString += line - - # message = email.message_from_string(headerString + "".join(self.__getContent())) - message = email.message_from_string(headerString) - - headers = [] - dateHeaders=[ - "QStatus-Updated-Time", - "Status-Updated-Time", - "Edited-Time", - "QTime-Updated-Time", - "Merged-Time", - "Time-Updated-Time", - "Replied-Time", - "Assigned-To-Updated-Time", - "QAssigned-To-Updated-Time", - "Date", - "Sent" - ] - - for key in message.keys(): - headers.append({"type": key, "content": self.__getFormattedDate(message[key]) if key in dateHeaders else message[key]}) - - return headers - - # TODO: Implement attachment parsing - - def __parseSections(self) -> list: - # List of all item events - sections = [] - - contentStart = self.__getHeaderBoundary() + 1 - contentEnd = len(self.__rawItem) - 1 - - # List of assignments for the item - assignementLsit = self.__assignmentParsing(contentStart) - - # Appends each assignment individually to sections - for assignment in assignementLsit: - sections.append(assignment) - - # Checks for empty content within an item and returns and - if contentEnd <= contentStart: - blankInitialMessage = self.__initialMessageParsing([""]) - sections.append(blankInitialMessage) - return sections - - # Checks for Directory Identifiers - if self.__rawItem[contentStart] == "\n" and self.__rawItem[contentStart + 1].startswith("\t"): - - directoryStartLine = contentStart + 1 - - # Parses the directory information and returns a dictionary of directory values - directoryInfo = self.__directoryParsing(directoryStartLine) - - # Appends Directory Information into the sections array - sections.append(directoryInfo) - - # Sets the initial message start to the next line after all directory lines and newlines - contentStart = contentStart + len(directoryInfo) + 1 - - # The start line, type, and end line for item events - sectionBoundaries = [] - - # Delimiter info - delimiters = [ - {"name": "edit", "pattern": "*** Edited"}, - {"name": "status", "pattern": "*** Status"}, - {"name": "replyToUser", "pattern": "*** Replied"}, - {"name": "replyFromUser", "pattern": "=== "}, - ] - - # Signifies that there is an initial message to parse - initialMessageSection = True - - # Parses the entire contents of the message, stores everything before any delimiter as the initial message - # and the line number of any delimiters as well as the type - for lineNumber in range(contentStart, contentEnd + 1): - - line = self.__rawItem[lineNumber] - - # Looks for a starting delimiter and explicity excludes the reply-from-user ending delimiter - if (line.startswith("*** Edited by: ") or - line.startswith("*** Replied by: ") or - line.startswith("*** Status updated by: ") or - line == "=== Additional information supplied by user ===\n" and not - line == "===============================================\n" - ): - - # Sets the delimiter type based on the pattern within the delimiters list - for delimiter in delimiters: - - if line.startswith(delimiter["pattern"]): - sectionBoundaries.append( - {"start": lineNumber, "type": delimiter["name"]}) - break - - # If a starting delimiter was encountered, then there is no initial message - if initialMessageSection: - initialMessageSection = False - - elif initialMessageSection == True: - # Delimiter not encountered yet, so append initial message starting line as the current lin number - sectionBoundaries.append( - {"start": lineNumber, "type": "initial_message"}) - initialMessageSection = False - - # Used to set the end line of the last delimiter - sectionBoundaries.append({"start": contentEnd + 1}) - - # Sets the end of the section boundary to the begining of the next section boundary - for boundaryIndex in range(0, len(sectionBoundaries) - 1): - - sectionBoundaries[boundaryIndex]["end"] = sectionBoundaries[boundaryIndex + 1]["start"] - - # Remove End of File boundary since the line number has been assigned to the last delimiter - del sectionBoundaries[-1] - - # Parses through all the boundaries in section boundaries - for boundary in sectionBoundaries: - - # Sets line to the first line of the boundary (which is always the delimiter) - line = self.__rawItem[boundary["start"]] - - # Returns all of the lines within the current section - sectionContent = self.__rawItem[boundary["start"]: boundary["end"]] - - # Appends an initial message dictionary to sections - if boundary["type"] == "initial_message": - initialMessageDictionary = self.__initialMessageParsing( - sectionContent) - sections.append(initialMessageDictionary) - - elif boundary["type"] == "edit": - # Returns a dictionary with edit information - editInfo = self.__editParsing( - sectionContent, boundary["start"]) - - # Checks for a parse error and appends it, returning the sections list which stops the parsing - if editInfo["type"] == "parse_error": - sections.append(editInfo) - return self.__getSortedSections(sections) - - # Appends the edit dictionary to sections - sections.append(editInfo) - - elif boundary["type"] == "replyToUser": - # Returns a dictionary with reply-to information - replyToInfo = self.__replyToParsing( - sectionContent, boundary["start"]) - - # Checks for a parse error and appends it, returning the sections list which stops the parsing - if replyToInfo["type"] == "parse_error": - sections.append(replyToInfo) - return self.__getSortedSections(sections) - - # Appends the reply-to to sections - sections.append(replyToInfo) - - elif boundary["type"] == "status": - # Returns a dictionary with status information - statusInfo = self.__statusParsing( - sectionContent, boundary["start"]) - - if statusInfo["type"] == "parse_error": - sections.append(statusInfo) - return self.__getSortedSections(sections) - - # Appends the status to sections - sections.append(statusInfo) - - elif boundary["type"] == "replyFromUser": - # Returns a dictionary with userReply information - replyFromInfo = self.__userReplyParsing( - sectionContent, boundary["start"]) - - if replyFromInfo["type"] == "parse_error": - sections.append(replyFromInfo) - return self.__getSortedSections(sections) - - # Appends the replyFrom to sections - sections.append(replyFromInfo) - - sortedSections = self.__getSortedSections(sections) - - return sortedSections - # return sections - - def __directoryParsing(self, directoryStartLine: int) -> dict: - """Returns a dictionary with directory information - - **Example:** - - ``` - Name: Nestor Fabian Rodriguez Buitrago - Login: rodri563 - Computer: ce-205-38 (128.46.205.67) - Location: HAMP G230 - Email: rodri563@purdue.edu - Phone: 7654766893 - Office: HAMP G230 - UNIX Dir: /home/bridge/b/rodri563 - Zero Dir: U=\\\\bridge.ecn.purdue.edu\\rodri563 - User ECNDB: http://eng.purdue.edu/jump/2e8399a - Host ECNDB: http://eng.purdue.edu/jump/2e83999 - Subject: Autocad installation - ``` - - **Args:** - - `directoryStartLine (int)`: line number within the item that the directory starts on - - **Returns:** - ``` - dict: dictionary that splits each line within the directory into a key and a value - ``` - """ - directoryInformation = {"type": "directory_information"} - - directoryPossibleKeys = [ - "Name", - "Login", - "Computer", - "Location", - "Email", - "Phone", - "Office", - "UNIX Dir", - "Zero Dir", - "User ECNDB", - "Host ECNDB", - "Subject" - ] - # Executies until the directory start line is greater than the directory ending line - while True: - - # Returns the line number at directory start line - info = self.__rawItem[directoryStartLine] - - # Breaks the loop if it encountrs a newline, signifying the end of the directory information - if info == "\n": - - break - - else: - - # Removes white including space, newlines, and tabs from the directory info line - strippedInfo = info.strip() - - # Attempts to find ": " but will accept ":", denoting a blank entry for a directory item - if ": " in strippedInfo: - - # Seperates the directory info line into two variables, the first variable being the key, the second being the value - # swt1 - key, value = strippedInfo.split(": ", 1) - - if key in directoryPossibleKeys: - # Adds the key value pair to the directory info dictionary - directoryInformation[key] = value - else: - # Casts the list type on to a dictionary - dictionaryList = list(directoryInformation) - # Length of dictionary list - lenDictionaryList = len(dictionaryList) - # The last key appended to the directory dictionary - lastKeyAppended = dictionaryList[lenDictionaryList - 1] - - directoryInformation[lastKeyAppended] = directoryInformation[lastKeyAppended] + \ - " " + strippedInfo - - elif ":" in strippedInfo: - - # Seperates the directory info line into two variables, the first variable being the key, the second being the value - key, value = strippedInfo.split(":", 1) - - if key in directoryPossibleKeys: - # Adds the key value pair to the directory info dictionary - directoryInformation[key] = value - else: - # Casts the list type on to a dictionary - dictionaryList = list(directoryInformation) - # Length of dictionary list - lenDictionaryList = len(dictionaryList) - # The last key appended to the directory dictionary - lastKeyAppended = dictionaryList[lenDictionaryList - 1] - - directoryInformation[lastKeyAppended] = directoryInformation[lastKeyAppended] + \ - " " + strippedInfo - - # Signifies that this line belongs to the most previous line - elif ": " not in strippedInfo and ":" not in strippedInfo: - # Casts the list type on to a dictionary - dictionaryList = list(directoryInformation) - # Length of dictionary list - lenDictionaryList = len(dictionaryList) - # The last key appended to the directory dictionary - lastKeyAppended = dictionaryList[lenDictionaryList - 1] - - directoryInformation[lastKeyAppended] = directoryInformation[lastKeyAppended] + \ - " " + strippedInfo - # Counter to denote the end of the directory - directoryStartLine = directoryStartLine + 1 - - # Returns the directory information dictionary - return directoryInformation - - def __assignmentParsing(self, contentStart: int) -> list: - """Returns a list with assignment information dictionaries - - **Example:** - - ``` - Assigned-To: campb303 - Assigned-To-Updated-Time: Tue, 23 Jun 2020 13:27:00 EDT - Assigned-To-Updated-By: campb303 - ``` - - **Args:** - - `contentStart (int)`: line number where the content starts - - **Returns:** - - ``` - list: [ - {"type": "assignment", - "datetime": datetime of the assignment, - "by": user who initiated the assignment, - "to": user who was assigned - }, - ] - ``` - """ - assignmentList = [] - - # Assignment Information - assignedBy = "" - assignedDateTime = "" - assignedTo = "" - - # Parses the header looking for assignment delimeters and stores info into their respective variables - for headerContent in range(0, contentStart): - - line = self.__rawItem[headerContent] - - # Gets who the Item was assigned to - if line.startswith("Assigned-To: "): - - assignedTo = ( - re.search("(?<=Assigned-To: )(.*)", line)).group() - - # Gets the date the Item was assigned - elif line.startswith("Assigned-To-Updated-Time: "): - - dateFromLine = ( - re.search("(?<=Assigned-To-Updated-Time: )(.*)", line)).group() - - assignedDateTime = self.__getFormattedDate(dateFromLine) - - # Gets who assigned the Item - elif line.startswith("Assigned-To-Updated-By: "): - - assignedBy = ( - re.search("(?<=Assigned-To-Updated-By: )(.*)", line)).group() - - # Appends the assignment to the sections list - assignmentList.append( - {"type": "assignment", - "datetime": assignedDateTime, - "by": assignedBy, - "to": assignedTo} - ) - - return assignmentList - - def __initialMessageParsing(self, content: list) -> dict: - """Returns a dictionary with initial message information - - **Example:** - ``` - \n - Testtest\n - \n - ``` - - **Args:** - `content (list)`: content of the initial message - - **Returns:** - ``` - dict: - "type": "initial_message", - "datetime": datetime the initial message was sent, - "from_name": from_name, - "from_email": user_email, - "to": [{email, name}], - "cc": [{email, name}], - "subject": initial message subject - "content": content of the initial message - ``` - """ - initialMessageDictionary = {} - - initialMessageDictionary["type"] = "initial_message" - - # Gets the initial message date from the header - rawMessageDateStr = self.__getMostRecentHeaderByType("Date") - - # Sets datetime in the intialMessage dictionary to UTC formatted date - initialMessageDictionary["datetime"] = self.__getFormattedDate( - rawMessageDateStr) - - initialMessageDictionary["from_name"] = self.__parseFromData( - data="userName") - - initialMessageDictionary["from_email"] = self.__parseFromData( - data="userEmail") - - # Stores list of dictionaries for the recipients of the initial message - initialMessageDictionary["to"] = [] - - # Parses the header looking for recipients of the initial message and stores it in a list of tuples - rawMessageRecipientsList = email.utils.getaddresses( - [self.__getMostRecentHeaderByType("To")]) - - # Parses the CC list and stores the cc recipient information in a list of dictionaries - for recipients in rawMessageRecipientsList: - - initialMessageDictionary["to"].append( - {"name": recipients[0], - "email": recipients[1]} - ) - - # Stores list of dictionaries for CC information - initialMessageDictionary["cc"] = [] - - # Parses the header looking for CC recipients of the initial message and stores it in a list of tuples - rawMessageCCList = email.utils.getaddresses( - [self.__getMostRecentHeaderByType("CC")]) - - # Parses the CC list and stores the cc recipient information in a list of dictionaries - for ccRecipients in rawMessageCCList: - - initialMessageDictionary["cc"].append( - {"name": ccRecipients[0], - "email": ccRecipients[1]} - ) - - initialMessageDictionary["subject"] = self.__getMostRecentHeaderByType( - "Subject") - - # Removes unecessary newlines from the begining and the end of the initial message - initialMessageDictionary["content"] = self.__getFormattedSectionContent( - content) - - return initialMessageDictionary - - def __editParsing(self, content: list, lineNum: int) -> dict: - """Returns a dictionary with edit information - - **Example:** - ``` - \*\*\* Edited by: campb303 at: 06/23/20 13:27:56 \*\*\* - This be an edit my boy - - - ``` - **Args:** - - `content (list)`: content of an edit - - `lineNum (int)`: line number of an edit within an item - - **Returns:** - ``` - dict: a dictionary with these keys: - "type": "edit", - "by": initiator of the edit, - "datetime": datetime of the edit, - "content": content of the edit - ``` - """ - - # Edit Info dictionary - editInfo = {} - - for count, line in enumerate(content): - if line == "===============================================\n": - errorMessage = "Reply-from-user ending delimter encountered without Reply-from-user starting delimter" - return self.__errorParsing(line, lineNum + count + 1, errorMessage) - - editInfo["type"] = "edit" - - delimiterLine = content[0] - # Parses for the author of the edit, which is located between the "*** Edited by: " and " at:" substrings - try: - editInfo["by"] = ( - re.search("(?<=\*{3} Edited by: )(.*)(?= at:)", delimiterLine)).group() - except: - errorMessage = "*** Edited by: [username] at: [date and time] ***\n" - return self.__errorParsing(delimiterLine, lineNum, errorMessage) - - try: - # Parses for the date and time of the edit, which is located between the " at: " and "***\n" substrings - dateTimeString = ( - re.search("(?<= at: )(.*)(?= \*\*\*\n)", delimiterLine)).group() - except: - # Returns an error message if there is no space after "at:" - errorMessage = "*** Edited by: [username] at: [date and time] ***\n" - return self.__errorParsing(delimiterLine, lineNum, errorMessage) - - # Attempts to format the date and time into utc format - editInfo["datetime"] = self.__getFormattedDate(dateTimeString) - - # Remove the delimiter String and unecessary newlines - editInfo["content"] = self.__getFormattedSectionContent(content) - - return editInfo - - def __replyToParsing(self, content: list, lineNum: int) -> dict: - """Returns a dictionary with reply to user information - - **Example:** - ``` - \*\*\* Replied by: campb303 at: 06/23/20 13:28:18 \*\*\* - - This be a reply my son - - Justin - ECN - ``` - - **Args:** - - `content (list)`: content of a reply to user - - `lineNum (int)`: line number of a reply to user in an item - - **Returns:** - ``` - dict: a dictionary with these keys, - "type": "reply_to_user", - "by": initiator of the reply to user, - "datetime": datetime of the reply to user, - "content": content of the reply to user - ``` - """ - replyInfo = {} - - replyInfo["type"] = "reply_to_user" - - delimiterLine = content[0] - - for count, line in enumerate(content): - if line == "===============================================\n": - errorMessage = "Reply-from-user ending delimter encountered without Reply-from-user starting delimter" - return self.__errorParsing(line, lineNum + count + 1, errorMessage) - - try: - # Parses for the author of the reply, which is located between the "*** Replied by: " and " at:" substrings - replyInfo["by"] = ( - re.search("(?<=\*{3} Replied by: )(.*)(?= at:)", delimiterLine)).group() - except: - errorMessage = "*** Replied by: [username] at: [date and time] ***\n" - return self.__errorParsing(delimiterLine, lineNum, errorMessage) - - # Parses for the date and time of the reply, which is located between the " at: " and "***\n" substrings - try: - dateTimeString = ( - re.search("(?<= at: )(.*)(?= \*\*\*\n)", delimiterLine)).group() - except: - errorMessage = "*** Replied by: [username] at: [date and time] ***\n" - return self.__errorParsing(delimiterLine, lineNum, errorMessage) - - # Formats date to UTC - replyInfo["datetime"] = self.__getFormattedDate(dateTimeString) - - replyInfo["content"] = self.__getFormattedSectionContent(content) - - return replyInfo - - def __statusParsing(self, content: list, lineNum: int) -> dict: - """Returns a dictionary with status information - - **Example:** - ``` - \*\*\* Status updated by: campb303 at: 6/23/2020 13:26:55 \*\*\* - Dont Delete - \n - ``` - - **Args:** - - `content (list)`: The content of a status update - - `lineNum (int)`: The line number of a status update in an item - - **Returns:** - ``` - dict: a dictionary with these keys, - "type": "status", - "by": initiator of the status update, - "datetime": datetime of the status update, - "content": content of the status update - ``` - """ - statusInfo = {} - - statusInfo["type"] = "status" - - delimiterLine = content[0] - - for count, line in enumerate(content): - if line == "===============================================\n": - errorMessage = "Reply-from-user ending delimter encountered without Reply-from-user starting delimter" - return self.__errorParsing(line, lineNum + count + 1, errorMessage) - - # Parses for the author of the status change, which is located between the "*** Status updated by: " and " at:" substrings - try: - statusInfo["by"] = ( - re.search("(?<=\*{3} Status updated by: )(.*)(?= at:)", delimiterLine)).group() - except: - errorMessage = "*** Status updated by: [username] at: [date and time] ***\n" - - return self.__errorParsing(delimiterLine, lineNum, errorMessage) - - # Parses for the date and time of the status change, which is located between the " at: " and "***\n" substrings - try: - dateTimeString = re.search( - "(?<= at: )(.*)(?= \*\*\*\n)", delimiterLine).group() - except: - errorMessage = "*** Status updated by: [username] at: [date and time] ***\n" - - return self.__errorParsing(delimiterLine, lineNum, errorMessage) - - # Formats the date to UTC - statusInfo["datetime"] = self.__getFormattedDate(dateTimeString) - - # Remove the delimiter String and unecessary newlines - statusInfo["content"] = self.__getFormattedSectionContent(content) - - return statusInfo - - def __userReplyParsing(self, replyContent: list, lineNumber: int) -> dict: - """Returns a dictionary with user reply information - - **Example:** - ``` - === Additional information supplied by user === - - Subject: Re: Beepboop\n - From: Justin Campbell \n - Date: Tue, 23 Jun 2020 13:30:45 -0400\n - X-ECN-Queue-Original-Path: /home/pier/e/queue/Attachments/inbox/2020-06-23/212-original.txt\n - X-ECN-Queue-Original-URL: https://engineering.purdue.edu/webqueue/Attachments/inbox/2020-06-23/212-original.txt\n - - Huzzah! - - =============================================== - \n - ``` - **Args:** - - `replyContent (list)`: The entire section of a reply-from-user - - `lineNumber (int)`: The line number of the begining of a reply-from-user section within and item - - Returns: - ``` - dict: a dictionary with these keys, - "type": "reply_from_user", - "from_name": name of the user that sent the reply, - "from_email": email of the user that sent the reply, - "subject": subject of the reply, - "datetime": the datetime of the reply, - "cc": [ - {"name": name of the carbon copied recipient, - "email": email of the carbon copied recipient - }, - ] - "content": content of the reply - "headers": [ - {"type": headerType, - "content": content - }, - ] - ``` - """ - replyFromInfo = {} - - replyFromInfo["type"] = "reply_from_user" - - replyFromHeaders = [] - newLineCounter = 0 - endingDelimiterCount = 0 - - # Delimiter information line numbers to remove from reply from user - linesToRemove = [] - - # Parses the section content looking for any line that starts with a metadata, also tracks the line - # number with the enumerate function - for lineNum, line in enumerate(replyContent): - - if endingDelimiterCount == 0 and lineNum == len(replyContent) - 1: - errorMessage = "Did not encounter a reply-from-user ending delimiter" - return self.__errorParsing(line, lineNumber + lineNum + 1, errorMessage) - - if newLineCounter == 1 and line != "\n": - - try: - # Append header information for each headr line - headerType, content = line.split(": ", 1) - replyFromHeaders.append( - {"type": headerType, - "content": content - } - ) - except: - lenReplyFromHeaders = len(replyFromHeaders) - if lenReplyFromHeaders == 0: - errorMessage = ("Expected reply-from-user header information:\n" + - "=== Additional information supplied by user ===\n" + - "\n" + - "[Header Type]: [Header Value]\n" + - "\n" - ) - return self.__errorParsing(line, lineNumber + lineNum + 1, errorMessage) - - else: - replyFromHeaders[lenReplyFromHeaders - - 1]["content"] = replyFromHeaders[lenReplyFromHeaders - 1]["content"] + " " + line - - linesToRemove.append(lineNum) - # Checks for a newline and breaks for loop on second occurance of a newline - if line == "\n": - newLineCounter = newLineCounter + 1 - - if newLineCounter == 2 and "datetime" not in replyFromInfo.keys(): - errorMessage = "Expected \"Date: [datetime]\" in the header info" - return self.__errorParsing(line, lineNumber + lineNum + 1, errorMessage) - - elif line == "===============================================\n": - endingDelimiterCount = endingDelimiterCount + 1 - - elif line.startswith("From: ") and newLineCounter == 1: - # Returns a list of one tuples with a name stored in the first index of the tuple and an email stored in the second index of the tuple - emailList = email.utils.getaddresses([line]) - replyFromInfo["from_name"] = emailList[0][0] - replyFromInfo["from_email"] = emailList[0][1] - - elif line.startswith("Subject: ") and newLineCounter == 1: - # Matches everything after "Subject: " - try: - subjectStr = ( - re.search("(?<=Subject: )(.*)", line)).group() - except: - errorMessage = "Expeted syntax of \"Subject: [subject]\"" - return self.__errorParsing(line, lineNumber + lineNum + 1, errorMessage) - - # Formatts the date to UTC - replyFromInfo["subject"] = subjectStr - - elif line.startswith("Date: ") and newLineCounter == 1: - # Matches everything after "Date: " - try: - dateStr = (re.search("(?<=Date: )(.*)", line)).group() - except: - errorMessage = "\"Date: [datetime]\"" - return self.__errorParsing(line, lineNumber + lineNum + 1, errorMessage) - - # Formatts the date to UTC - replyFromInfo["datetime"] = self.__getFormattedDate(dateStr) - - elif line.startswith("Cc: ") and newLineCounter == 1: - - replyFromInfo["cc"] = [] - - # Returns a list of tuples with email information - recipientsList = email.utils.getaddresses([line]) - - # Parses through the cc tuple list - for cc in recipientsList: - # Stores the cc information in a dictionary and appends it to the ccRecipientsList - replyFromInfo["cc"].append( - {"name": cc[0], - "email": cc[1]} - ) - - # Deletes reduntant lines from the message content in reverse order - for lineNum in sorted(linesToRemove, reverse=True): - replyContent.pop(lineNum) - - # Strips any unnecessary newlines or any delimiters frm the message content - replyFromInfo["content"] = self.__getFormattedSectionContent( - replyContent) - - replyFromInfo["headers"] = replyFromHeaders - - return replyFromInfo - - def __getFormattedSectionContent(self, sectionContent: list) -> list: - """Returns a list with message content that is stripped of unnecessary newlines and begining delimiters - - **Example:** - ``` - \*\*\* Edited by: mph at: 02/21/20 10:27:16 \*\*\* - - Still need to rename machines - but the networking issue now seems to \n - be resolved via another ticket. - \n - ``` - - **Args:** - - `sectionContent (list)`: The section content of a parsed section - - **Returns:** - ``` - list: the section content of a parsed section without any delimiters and unnecessary newlines - ``` - """ - # Continually removes the first line of sectionContent if it is a newline or delimiter in each iteration - while len(sectionContent) > 1: - if (sectionContent[0] == "\n" or - sectionContent[0].startswith("*** Edited by: ") or - sectionContent[0].startswith("*** Replied by: ") or - sectionContent[0].startswith("*** Status updated by: ") or - sectionContent[0] == "=== Additional information supplied by user ===\n" or - sectionContent[0] == "===============================================\n" - ): - sectionContent.pop(0) - else: - # Breaks the loop if the first line isn't a newline or delimiter - break - - # Continually removes the last line of sectionContent if it is a newline or delimiter in each iteration - while len(sectionContent) > 1: - # Initializes the Length of sectionContent each iteration of the loop - sectionContentLength = len(sectionContent) - - if (sectionContent[sectionContentLength - 1] == "\n" or - sectionContent[sectionContentLength - - 1] == "===============================================\n" - ): - sectionContent.pop(sectionContentLength - 1) - else: - # Breaks the loop if the last line isn't a newline or delimiter - break - - return sectionContent - - def __errorParsing(self, line: str, lineNum: int, expectedSyntax: str) -> dict: - """Returns a dictionary with error parse information when a line is malformed - - **Example:** - ``` - \*\*\* Status updated by: ewhile at: 5/7/2020 10:59:11 \*\*\* - ``` - - **Args:** - - `line (str)`: line of that threw error - - `lineNum (int)`: line number in the item that threw error - - `expectedSyntax (str)`: a message stating the syntax the line should follow - - **Returns:** - ``` - dict: a dictionary with these keys, - "type": "parse_error", - "datetime": time the error was encountered, - "file_path": path of the item with erroneos line, - "expected": expectedSyntax, - "got": line, - "line_num": lineNum - ``` - """ - errorDictionary = {} - - # Type - errorDictionary["type"] = "parse_error" - - # Dateime of the parse error - errorDictionary["datetime"] = self.__getFormattedDate( - str(datetime.datetime.now())) - - # Item filepath - errorDictionary["file_path"] = self.__path - - # Expected value - errorDictionary["expected"] = expectedSyntax - - # line that threw error - errorDictionary["got"] = line - - # line number that threw error - errorDictionary["line_num"] = lineNum - - # returns the error dictionary - return errorDictionary - - def __getSortedSections(self, sectionsList: list) -> list: - """Sorts the sections chronologically by datetime - - **Example:** - ``` - [example] need to do - ``` - - **Args:** - - `sections (list)`: the list of sections to be sorted - - **Returns:** - ``` - list: a list of sections sorted by datetime - ``` - """ - sectionsLength = len(sectionsList) - sortedSections = [] - oldestSection = {} - - while len(sortedSections) < sectionsLength: - - for iteration, currentSection in enumerate(sectionsList): - - if currentSection["type"] == "directory_information": - sortedSections.append(currentSection) - sectionsList.remove(currentSection) - break - - if iteration == 0: - oldestSection = currentSection - - #datetime.datetime.strptime(date_time_str, '%Y-%m-%d %H:%M:%S.%f') - - elif parse(currentSection["datetime"]) < parse(oldestSection["datetime"]): - oldestSection = currentSection - - if iteration == len(sectionsList) - 1: - sortedSections.append(oldestSection) - sectionsList.remove(oldestSection) - - return sortedSections - - def __isLocked(self) -> Union[str, bool]: - """Returns a string info about the lock if true and a bool False if false - - **Example:** - ``` - A file is locked - "CE 100 is locked by campb303 using qvi" - ``` - - **Example:** - ``` - a file is not locked - False - ``` - - **Returns:** - - `Union[str, bool]`: String with info about lock if true, bool False if false - """ - lockFile = self.__path + ".lck" - if os.path.exists(lockFile): - with open(lockFile) as file: - lockInfo = file.readline().split(" ") - lockedBy = lockInfo[4] - lockedUsing = lockInfo[1] - return "{queue} {number} is locked by {lockedBy} using {lockedUsing}".format(queue=self.queue, number=self.number, lockedBy=lockedBy, lockedUsing=lockedUsing) - else: - return False - - def __getMostRecentHeaderByType(self, headerType: str) -> str: - """Return the data of most recent header of the given type. - If no header of that type exists, return an empty string. - - **Example:** - ``` - Requesting a Status header that does exist - __getMostRecentHeaderByType("Status") - becomes "Waiting for Reply" - ``` - - **Example:** - ``` - Requesting a Status header that doesn't exist - __getMostRecentHeaderByType("Status") - becomes "" - ``` - - **Args:** - - `headerType (str)`: Type of header to return. - - **Returns:** - ``` - str: data of most recent header of the given type or empty string. - ``` - """ - for header in self.headers: - if header["type"] == headerType: - return header["content"] - return "" - - def __parseFromData(self, data: str) -> str: - """Parse From header and return requested data. - Returns empty string if requested data is unavailable. - - **Examples:** - ``` - From data is "From: Campbell, Justin " - __parseFromData(data="userName") returns "Campbell, Justin" - __parseFromData(data="userEmail") returns "campb303@purdue.edu" - ``` - - **Args:** - - `data (str)`: The data desired; can be "userName" or "userEmail". - - **Returns:** - ``` - str: userName, userEmail or empty string. - ``` - """ - fromHeader = self.__getMostRecentHeaderByType("From") - userName, userEmail = email.utils.parseaddr(fromHeader) - - if data == "userName": - return userName - elif data == "userEmail": - return userEmail - else: - raise ValueError( - "data='" + str(data) + "' is not a valid option. data must be \"userName\" or \"userEmail\".") - - def __getUserAlias(self) -> str: - """Returns user's Career Account alias if present. - If Career Account alias isn't present, returns empty string. - - **Example:** - ``` - Email from campb303@purdue.edu - userAlias = "campb303" - ``` - - **Example:** - ``` - Email from spam@spammer.net - userAlias = "" - ``` - - **Returns:** - ``` - str: User's Career Account alias if present or empty string - ``` - """ - - - try: - emailUser, emailDomain = self.userEmail.split("@") - - # Returns an error parse if the self.useremail doesn't contain exactally one "@" symbol - except ValueError: - # Parses through the self.headers list to find the "From" header and its line number - for lineNum, header in enumerate(self.headers): - if header["type"] == "From": - headerString = header["type"] + ": " + header["content"] - return self.__errorParsing(headerString, lineNum + 1, "Expected valid email Address") - - return emailUser if emailDomain.endswith("purdue.edu") else "" - - def __getFormattedDate(self, date: str) -> str: - """Returns the date/time formatted as RFC 8601 YYYY-MM-DDTHH:MM:SS+00:00. - Returns empty string if the string argument passed to the function is not a datetime. - See: https://en.wikipedia.org/wiki/ISO_8601 - - **Returns:** - ``` - str: Properly formatted date/time recieved or empty string. - ``` - """ - try: - # This date is never meant to be used. The default attribute is just to set timezone. - parsedDate = parse(date, default=datetime.datetime( - 1970, 1, 1, tzinfo=tz.gettz('EDT'))) - except: - return "" - - parsedDateString = parsedDate.strftime("%Y-%m-%dT%H:%M:%S%z") - - return parsedDateString - - def toJson(self) -> dict: - """Returns a JSON safe representation of the item. - - **Returns:** - ``` - dict: JSON safe representation of the item. - ``` - """ - return self.jsonData - - def __repr__(self) -> str: - return self.queue + str(self.number) - -# TODO: Make Queue iterable using __iter__. See: https://thispointer.com/python-how-to-make-a-class-iterable-create-iterator-class-for-it/ -class Queue: - """A collection of items. - - **Example:** - - ``` - # Create a queue (ce) - >>> queue = Queue("ce") - ``` - - **Attributes:** - - ``` - name: The name of the queue. - items: A list of Items in the queue. - jsonData: A JSON serializable representation of the Queue. - ``` - """ - - def __init__(self, name: str) -> None: - self.name = name - self.__directory = queueDirectory + "/" + self.name + "/" - self.items = self.__getItems() - - self.jsonData = { - "name": self.name, - "length": len(self) - } - - def __getItems(self) -> list: - """Returns a list of items for this Queue - - **Returns:** - ``` - list: a list of items for this Queue - ``` - """ - items = [] - - for item in os.listdir(self.__directory): - itemPath = self.__directory + "/" + item - - isFile = True if os.path.isfile(itemPath) else False - - if isFile and isValidItemName(item): - items.append(Item(self.name, item)) - - return items - - def toJson(self) -> dict: - """Return JSON safe representation of the Queue - - The JSON representation of every item in the Queue is added to the - Queue's JSON data then the Queue's JSON data is returned. - - **Returns:** - ``` - dict: JSON safe representation of the Queue - ``` - """ - items = [] - for item in self.items: - items.append(item.toJson()) - self.jsonData["items"] = items - - return self.jsonData - - def __len__(self) -> int: - return len(self.items) - - def __repr__(self) -> str: - return f'{self.name}_queue' - -def getValidQueues() -> list: - """Returns a list of queues on the filesystem excluding ignored queues. - - **Example:** - ``` - ["bidc", "me", "ce"] - ``` - - **Returns:** - ``` - list: Valid queues - ``` - """ - queues = [] - - for file in os.listdir(queueDirectory): - currentFile = queueDirectory + "/" + file - isDirectory = os.path.isdir(currentFile) - isValid = file not in queuesToIgnore - - if isDirectory and isValid: - queues.append(file) - - return queues - -def getQueueCounts() -> list: - """Returns a list of dictionaries with the number of items in each queue. - - **Example:** - ``` - [ - { - name: "me", - number_of_items: 42 - }, - { - name: "bidc", - number_of_items: 3 - } - ] - ``` - **Returns:** - ``` - list: Dictionaries with the number of items in each queue. - ``` - """ - queueInfo = [] - for queue in getValidQueues(): - possibleItems = os.listdir(queueDirectory + "/" + queue) - validItems = [isValidItemName for file in possibleItems] - queueInfo.append( {"name": queue, "number_of_items": len(validItems)} ) - - # Sorts list of queue info alphabetically - sortedQueueInfo = sorted(queueInfo, key = lambda queueInfoList: queueInfoList['name']) - - return sortedQueueInfo - - -def loadQueues() -> list: - """Return a list of Queues for each queue. - - **Returns:** - ``` - list: list of Queues for each queue. - ``` - """ - queues = [] - - for queue in getValidQueues(): - queues.append(Queue(queue)) - - return queues \ No newline at end of file diff --git a/webqueue2_api/__init__.py b/webqueue2_api/__init__.py deleted file mode 100644 index 3bec1e8..0000000 --- a/webqueue2_api/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from . import api, ECNQueue \ No newline at end of file diff --git a/webqueue2_api/api.py b/webqueue2_api/api.py deleted file mode 100644 index e8a0618..0000000 --- a/webqueue2_api/api.py +++ /dev/null @@ -1,272 +0,0 @@ -from flask import Flask, request, after_this_request -from flask_restful import Api, Resource -from flask_jwt_extended import ( - JWTManager, create_access_token, create_refresh_token, - jwt_required, get_jwt_identity, jwt_refresh_token_required, - set_refresh_cookies, unset_refresh_cookies -) -import os, dotenv -from easyad import EasyAD -from ldap.filter import escape_filter_chars -# pylint says this is an error but it works so ¯\_(ツ)_/¯ -from ldap import INVALID_CREDENTIALS as LDAP_INVALID_CREDENTIALS -from . import ECNQueue - -# Load envrionment variables for ./.env -dotenv.load_dotenv() - -# Create Flask App -app = Flask(__name__) - -# Create API Interface -api = Api(app) - - -################################################################################ -# Configure Flask-JWT-Extended -################################################################################ - -# Set JWT secret key and create JWT manager -app.config["JWT_SECRET_KEY"] = os.environ.get("JWT_SECRET_KEY") -# Set identity claim field key to sub for JWT RFC complience -# Flask-JWT-Extended uses 'identity' by default for compatibility reasons -app.config["JWT_IDENTITY_CLAIM"] = "sub" -# Set the key for error messages generated by Flask-JWT-Extended -app.config["JWT_ERROR_MESSAGE_KEY"] = "message" - -# Look for JWTs in headers (for access) then cookies (for refresh) -app.config["JWT_TOKEN_LOCATION"] = ["headers", "cookies"] -# Restrict cookies to HTTPS in prod, allow HTTP in dev -app.config["JWT_COOKIE_SECURE"] = False if os.environ.get("ENVIRONMENT") == "dev" else True -# Restrict cookies using SameSite=strict flag -app.config["JWT_COOKIE_SAMESITE"] = "strict" -# Set the cookie key for CRSF validation string -# This is the default value. Adding it for easy reference -app.config["JWT_REFRESH_CSRF_HEADER_NAME"] = "X-CSRF-TOKEN" - -tokenManager = JWTManager(app) - - - -def user_is_valid(username: str, password: str) -> bool: - """Checks if user is valid and in webqueue2 login group. - - Args: - username (str): Career account username. - password (str): Career account passphrase. - - Returns: - bool: True if user is valid, otherwise False. - """ - - # Check for empty arguments - if (username == "" or password == ""): - return False - - # Check for adm account - if username.endswith("adm"): - return False; - - # Initialize EasyAD - config = { - "AD_SERVER": "boilerad.purdue.edu", - "AD_DOMAIN": "boilerad.purdue.edu" - } - ad = EasyAD(config) - - # Prepare search critiera for Active Directory - credentials = { - "username": escape_filter_chars(username), - "password": password - } - attributes = [ 'cn', "memberOf" ] - filter_string = f'(&(objectClass=user)(|(sAMAccountName={username})))' - - # Do user search - try: - user = ad.search(credentials=credentials, attributes=attributes, filter_string=filter_string)[0] - except LDAP_INVALID_CREDENTIALS: - return False - - # Isolate group names - # Example: - # 'CN=00000227-ECNStuds,OU=BoilerADGroups,DC=BoilerAD,DC=Purdue,DC=edu' becomes - # `00000227-ECNStuds` - user_groups = [ group.split(',')[0].split('=')[1] for group in user["memberOf"] ] - - # Check group membership - webqueue_login_group = "00000227-ECN-webqueue" - if webqueue_login_group not in user_groups: - return False - - return True - - - -class Login(Resource): - def post(self) -> tuple: - """Validates username/password and returns both access and refresh tokens. - - **Return Codes:** - ``` - 200 (OK): On success. - 401 (Unauthroized): When username or password are incorrect. - 422 (Unprocessable Entitiy): When the username or password can't be parsed. - ``` - **Example:** - ``` - curl -X POST - -H "Content-Type: application/json" - -d '{"username": "bob", "password": "super_secret"}' - - { "access_token": fjr09hfp09h932jp9ruj3.3r8ihf8h0w8hr08ifhj804h8i.8h48ith08ity409hip0t4 } - ``` - **Returns:** - ``` - tuple: Response containing tokens and HTTP response code. - ``` - """ - if not request.is_json: - return ({ "message": "JSON missing from request body"}, 422) - - data = request.json - - fields_to_check = ["username", "password"] - for field in fields_to_check: - if field not in data.keys(): - return ({ "message": f"{field} missing from request body"}, 422) - - if not user_is_valid(data["username"], data["password"]): - return ({ "message": "Username or password is invalid"}, 401) - - access_token = create_access_token(data["username"]) - refresh_token = create_refresh_token(data["username"]) - - # This decorator is needed because Flask-RESTful's 'resourceful routing` - # doesn't allow for direct modification to the Flask response object. - # See: https://flask-restful.readthedocs.io/en/latest/quickstart.html#resourceful-routing - @after_this_request - def _does_this_work(response): - set_refresh_cookies(response, refresh_token) - return response - - return ({ "access_token": access_token }, 200) - -class RefreshAccessToken(Resource): - @jwt_refresh_token_required - def post(self): - username = get_jwt_identity() - access_token = create_access_token(username) - return ({"access_token": access_token}, 200) - -class Item(Resource): - @jwt_required - def get(self, queue: str, number: int) -> tuple: - """Returns the JSON representation of the item requested. - - **Return Codes:** - ``` - 200 (OK): On success. - ``` - - **Example:** - ``` - /api/ce/100 returns: - { - "lastUpdated": "07-23-20 10:11 PM", - "headers": [...], - "content": [...], - "isLocked": "ce 100 is locked by knewell using qvi", - "userEmail": "campb303@purdue.edu", - "userName": "Justin Campbell", - "userAlias": "campb303", - "assignedTo": "campb303", - "subject": "Beepboop", - "status": "Dont Delete", - "priority": "", - "deparment": "", - "building": "", - "dateReceived": "Tue, 23 Jun 2020 13:25:51 -0400" - } - ``` - **Args:** - ``` - queue (str): The queue of the item requested. - item (int): The number of the item requested. - ``` - - **Returns:** - ``` - tuple: Item as JSON and HTTP response code. - ``` - """ - return (ECNQueue.Item(queue, number).toJson(), 200) - -class Queue(Resource): - @jwt_required - def get(self, queues: str) -> tuple: - """Returns the JSON representation of the queue requested. - - **Return Codes:** - ``` - 200 (OK): On success. - ``` - - **Args:** - ``` - queues (str): Plus (+) deliminited list of queues. - ``` - - **Returns:** - ``` - tuple: Queues as JSON and HTTP response code. - ``` - """ - queues_requested = queues.split("+") - - queue_list = [] - for queue in queues_requested: - queue_list.append(ECNQueue.Queue(queue).toJson()) - - return (queue_list, 200) - -class QueueList(Resource): - @jwt_required - def get(self) -> tuple: - """Returns a list of dictionaries with the number of items in each queue. - - **Return Codes:** - ``` - 200 (OK): On success. - ``` - - **Example:** - ``` - [ - { - name: "me", - number_of_items: 42 - }, - { - name: "bidc", - number_of_items: 3 - } - ] - ``` - **Returns:** - ``` - tuple: Queues and item counts as JSON and HTTP response code. - ``` - """ - return (ECNQueue.getQueueCounts(), 200) - - - -api.add_resource(Login, "/login") -api.add_resource(RefreshAccessToken, "/tokens/refresh") -api.add_resource(Item, "/api//") -api.add_resource(Queue, "/api/") -api.add_resource(QueueList, "/api/get_queues") - -if __name__ == "__main__": - app.run() From 2648c8e416f6ddb3db42debe7cf2e21c4f93e724 Mon Sep 17 00:00:00 2001 From: benne238 Date: Mon, 28 Jun 2021 12:46:53 -0400 Subject: [PATCH 02/56] Add pyparsing to requirements --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 42a29bf..61fb84a 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,8 @@ def get_all_dependencies(): # Custom version of python-ldap without SASL requirements "python-ldap @ git+https://github.itap.purdue.edu/ECN/python-ldap/@python-ldap-3.3.1", "easyad", - "dataclasses" + "dataclasses", + "pyparsing" ], extras_require={ "dev": conditional_dependencies["dev"], From 425b41ac6776451e86d8664650ec659edc9a04a0 Mon Sep 17 00:00:00 2001 From: benne238 Date: Mon, 28 Jun 2021 12:48:54 -0400 Subject: [PATCH 03/56] Add ParseError to parser module --- src/webqueue2api/parser/errors.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/webqueue2api/parser/errors.py b/src/webqueue2api/parser/errors.py index ef1ac48..9cdb754 100644 --- a/src/webqueue2api/parser/errors.py +++ b/src/webqueue2api/parser/errors.py @@ -6,4 +6,9 @@ def __init__(self, path: str): class QueueDoesNotExistError(Exception): def __init__(self, path: str): self.message = f"Directory {path} not found." + super().__init__(self.message) + +class ParseError(Exception): + def __init__(self, line_number: int, message: str = "Unable to parse item."): + self.message = f"{message} at line {line_number}" super().__init__(self.message) \ No newline at end of file From dc11658b9059bd5bcf062ec9fcad4f2e48d68f46 Mon Sep 17 00:00:00 2001 From: benne238 Date: Mon, 28 Jun 2021 13:00:29 -0400 Subject: [PATCH 04/56] Move Item.__get_formatted_date to webqueue2api.parser.utils and rename to format_date_string --- src/webqueue2api/parser/utils.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 src/webqueue2api/parser/utils.py diff --git a/src/webqueue2api/parser/utils.py b/src/webqueue2api/parser/utils.py new file mode 100644 index 0000000..945a421 --- /dev/null +++ b/src/webqueue2api/parser/utils.py @@ -0,0 +1,26 @@ +"""Shared utilities for the parser package""" + +from dateutil import parser, tz +from datetime import datetime + +def format_date_string(date: str) -> str: + """Returns the date/time formatted as RFC 8601 YYYY-MM-DDTHH:MM:SS+00:00. + Returns empty string if the string argument passed to the function is not a datetime. + See: https://en.wikipedia.org/wiki/ISO_8601 + + Returns: + str: Properly formatted date/time recieved or empty string. + """ + try: + # This date is never meant to be used. The default attribute is just to set timezone. + parsed_date = parser.parse(date, default=datetime( + 1970, 1, 1, tzinfo=tz.gettz('EDT'))) + except: + return "" + + parsed_date_string = parsed_date.strftime("%Y-%m-%dT%H:%M:%S%z") + + return parsed_date_string + +if __name__ == "__main__": + print(format_date_string("Tue, 23 Jun 2020 13:30:45 -0400")) \ No newline at end of file From 12407b08b872ee5771358dd877c902da5ace307f Mon Sep 17 00:00:00 2001 From: benne238 Date: Mon, 28 Jun 2021 13:00:55 -0400 Subject: [PATCH 05/56] Refactor to use new format_date_string function --- src/webqueue2api/parser/item.py | 40 +++++++++------------------------ 1 file changed, 11 insertions(+), 29 deletions(-) diff --git a/src/webqueue2api/parser/item.py b/src/webqueue2api/parser/item.py index 27ea817..db484af 100644 --- a/src/webqueue2api/parser/item.py +++ b/src/webqueue2api/parser/item.py @@ -9,6 +9,7 @@ from pathlib import Path from .config import config from .errors import ItemDoesNotExistError +from .utils import format_date_string @@ -76,7 +77,7 @@ def __init__(self, queue: str, number: int, headers_only: bool = False) -> None: self.priority = self.__get_most_recent_header_by_type("Priority") self.department = self.__get_most_recent_header_by_type("Department") self.building = self.__get_most_recent_header_by_type("Building") - self.date_received = self.__get_formatted_date(self.__get_most_recent_header_by_type("Date")) + self.date_received = format_date_string(self.__get_most_recent_header_by_type("Date")) self.json_data = self.__generate_json_data() def __generate_json_data(self) -> dict: @@ -134,7 +135,7 @@ def __get_time_last_updated(self) -> str: """ unix_time = os.path.getmtime(self.path) formatted_time = time.strftime('%m-%d-%y %I:%M %p', time.localtime(unix_time)) - return self.__get_formatted_date(formatted_time) + return format_date_string(formatted_time) def __get_raw_item(self) -> list: """Returns a list of all lines in the item file @@ -210,7 +211,7 @@ def __parse_headers(self) -> list: ] for key in message.keys(): - headers.append({"type": key, "content": self.__get_formatted_date(message[key]) if key in date_headers else message[key]}) + headers.append({"type": key, "content": format_date_string(message[key]) if key in date_headers else message[key]}) return headers @@ -531,7 +532,7 @@ def __assignmentParsing(self, contentStart: int) -> list: dateFromLine = ( re.search("(?<=Assigned-To-Updated-Time: )(.*)", line)).group() - assignedDateTime = self.__get_formatted_date(dateFromLine) + assignedDateTime = format_date_string(dateFromLine) # Gets who assigned the Item elif line.startswith("Assigned-To-Updated-By: "): @@ -579,7 +580,7 @@ def __initialMessageParsing(self, content: list) -> dict: rawMessageDateStr = self.__get_most_recent_header_by_type("Date") # Sets datetime in the intialMessage dictionary to UTC formatted date - initialMessageDictionary["datetime"] = self.__get_formatted_date( + initialMessageDictionary["datetime"] = format_date_string( rawMessageDateStr) initialMessageDictionary["from_name"] = self.__parse_from_data( @@ -679,7 +680,7 @@ def __editParsing(self, content: list, lineNum: int) -> dict: return self.__errorParsing(delimiterLine, lineNum, errorMessage) # Attempts to format the date and time into utc format - editInfo["datetime"] = self.__get_formatted_date(dateTimeString) + editInfo["datetime"] = format_date_string(dateTimeString) # Remove the delimiter String and unecessary newlines editInfo["content"] = self.__getFormattedSectionContent(content) @@ -737,7 +738,7 @@ def __replyToParsing(self, content: list, lineNum: int) -> dict: return self.__errorParsing(delimiterLine, lineNum, errorMessage) # Formats date to UTC - replyInfo["datetime"] = self.__get_formatted_date(dateTimeString) + replyInfo["datetime"] = format_date_string(dateTimeString) replyInfo["content"] = self.__getFormattedSectionContent(content) @@ -791,7 +792,7 @@ def __statusParsing(self, content: list, lineNum: int) -> dict: return self.__errorParsing(delimiterLine, lineNum, errorMessage) # Formats the date to UTC - statusInfo["datetime"] = self.__get_formatted_date(dateTimeString) + statusInfo["datetime"] = format_date_string(dateTimeString) # Remove the delimiter String and unecessary newlines statusInfo["content"] = self.__getFormattedSectionContent(content) @@ -920,7 +921,7 @@ def __userReplyParsing(self, replyContent: list, lineNumber: int) -> dict: return self.__errorParsing(line, lineNumber + lineNum + 1, errorMessage) # Formatts the date to UTC - replyFromInfo["datetime"] = self.__get_formatted_date(dateStr) + replyFromInfo["datetime"] = format_date_string(dateStr) elif line.startswith("Cc: ") and newLineCounter == 1: @@ -1025,7 +1026,7 @@ def __errorParsing(self, line: str, lineNum: int, expectedSyntax: str) -> dict: errorDictionary["type"] = "parse_error" # Dateime of the parse error - errorDictionary["datetime"] = self.__get_formatted_date( + errorDictionary["datetime"] = format_date_string( str(datetime.datetime.now())) # Item filepath @@ -1175,25 +1176,6 @@ def __get_user_alias(self) -> str: return email_user if email_domain.endswith("purdue.edu") else "" - def __get_formatted_date(self, date: str) -> str: - """Returns the date/time formatted as RFC 8601 YYYY-MM-DDTHH:MM:SS+00:00. - Returns empty string if the string argument passed to the function is not a datetime. - See: https://en.wikipedia.org/wiki/ISO_8601 - - Returns: - str: Properly formatted date/time recieved or empty string. - """ - try: - # This date is never meant to be used. The default attribute is just to set timezone. - parsed_date = parse(date, default=datetime.datetime( - 1970, 1, 1, tzinfo=tz.gettz('EDT'))) - except: - return "" - - parsed_date_string = parsed_date.strftime("%Y-%m-%dT%H:%M:%S%z") - - return parsed_date_string - def to_json(self) -> dict: """Returns a JSON safe representation of the item. From 7ee915f4999eee400ff5ce3ab7849f06b898aafb Mon Sep 17 00:00:00 2001 From: benne238 Date: Mon, 28 Jun 2021 13:58:26 -0400 Subject: [PATCH 06/56] replaced old parser and imported new pyparsing parser in the item class --- src/webqueue2api/parser/item.py | 837 +----------------------------- src/webqueue2api/parser/parser.py | 293 +++++++++++ 2 files changed, 302 insertions(+), 828 deletions(-) create mode 100644 src/webqueue2api/parser/parser.py diff --git a/src/webqueue2api/parser/item.py b/src/webqueue2api/parser/item.py index db484af..ede4a5f 100644 --- a/src/webqueue2api/parser/item.py +++ b/src/webqueue2api/parser/item.py @@ -10,6 +10,7 @@ from .config import config from .errors import ItemDoesNotExistError from .utils import format_date_string +from .parser import parse_item @@ -64,7 +65,7 @@ def __init__(self, queue: str, number: int, headers_only: bool = False) -> None: raise ItemDoesNotExistError(str(self.path)) self.last_updated = self.__get_time_last_updated() - self.__raw_tem = self.__get_raw_item() + self.__raw_item = self.__get_raw_item() self.headers = self.__parse_headers() if not headers_only: self.content = self.__parseSections() self.is_locked = self.__check_is_locked() @@ -157,7 +158,7 @@ def __get_header_boundary(self) -> int: Returns: int: line number where the Item headers end """ - for line_number, line in enumerate(self.__raw_tem): + for line_number, line in enumerate(self.__raw_item): if line == "\n": return line_number @@ -184,7 +185,7 @@ def __parse_headers(self) -> list: # QTime-Updated-By: campb303 queue_prefix_pattern = re.compile(r"\[.*?\] {1}") for line_number in range(self.__get_header_boundary()): - line = self.__raw_tem[line_number] + line = self.__raw_item[line_number] line_has_queue_prefix = queue_prefix_pattern.match(line) if line_has_queue_prefix: @@ -216,833 +217,13 @@ def __parse_headers(self) -> list: return headers def __parseSections(self) -> list: - # List of all item events - sections = [] + # Convert list of lines to single string + raw_item_as_string = "".join(self.__raw_item) - contentStart = self.__get_header_boundary() + 1 - contentEnd = len(self.__raw_tem) - 1 + # Parse body + body_sections = parse_item(raw_item_as_string) - # List of assignments for the item - assignementLsit = self.__assignmentParsing(contentStart) - - # Appends each assignment individually to sections - for assignment in assignementLsit: - sections.append(assignment) - - # Checks for empty content within an item and returns and - if contentEnd <= contentStart: - blankInitialMessage = self.__initialMessageParsing([""]) - sections.append(blankInitialMessage) - return sections - - # Checks for Directory Identifiers - if self.__raw_tem[contentStart] == "\n" and self.__raw_tem[contentStart + 1].startswith("\t"): - - directoryStartLine = contentStart + 1 - - # Parses the directory information and returns a dictionary of directory values - directoryInfo = self.__directoryParsing(directoryStartLine) - - # Appends Directory Information into the sections array - sections.append(directoryInfo) - - # Sets the initial message start to the next line after all directory lines and newlines - contentStart = contentStart + len(directoryInfo) + 1 - - # The start line, type, and end line for item events - sectionBoundaries = [] - - # Delimiter info - delimiters = [ - {"name": "edit", "pattern": "*** Edited"}, - {"name": "status", "pattern": "*** Status"}, - {"name": "replyToUser", "pattern": "*** Replied"}, - {"name": "replyFromUser", "pattern": "=== "}, - ] - - # Signifies that there is an initial message to parse - initialMessageSection = True - - # Parses the entire contents of the message, stores everything before any delimiter as the initial message - # and the line number of any delimiters as well as the type - for lineNumber in range(contentStart, contentEnd + 1): - - line = self.__raw_tem[lineNumber] - - # Looks for a starting delimiter and explicity excludes the reply-from-user ending delimiter - if (line.startswith("*** Edited by: ") or - line.startswith("*** Replied by: ") or - line.startswith("*** Status updated by: ") or - line == "=== Additional information supplied by user ===\n" and not - line == "===============================================\n" - ): - - # Sets the delimiter type based on the pattern within the delimiters list - for delimiter in delimiters: - - if line.startswith(delimiter["pattern"]): - sectionBoundaries.append( - {"start": lineNumber, "type": delimiter["name"]}) - break - - # If a starting delimiter was encountered, then there is no initial message - if initialMessageSection: - initialMessageSection = False - - elif initialMessageSection == True: - # Delimiter not encountered yet, so append initial message starting line as the current lin number - sectionBoundaries.append( - {"start": lineNumber, "type": "initial_message"}) - initialMessageSection = False - - # Used to set the end line of the last delimiter - sectionBoundaries.append({"start": contentEnd + 1}) - - # Sets the end of the section boundary to the begining of the next section boundary - for boundaryIndex in range(0, len(sectionBoundaries) - 1): - - sectionBoundaries[boundaryIndex]["end"] = sectionBoundaries[boundaryIndex + 1]["start"] - - # Remove End of File boundary since the line number has been assigned to the last delimiter - del sectionBoundaries[-1] - - # Parses through all the boundaries in section boundaries - for boundary in sectionBoundaries: - - # Sets line to the first line of the boundary (which is always the delimiter) - line = self.__raw_tem[boundary["start"]] - - # Returns all of the lines within the current section - sectionContent = self.__raw_tem[boundary["start"]: boundary["end"]] - - # Appends an initial message dictionary to sections - if boundary["type"] == "initial_message": - initialMessageDictionary = self.__initialMessageParsing( - sectionContent) - sections.append(initialMessageDictionary) - - elif boundary["type"] == "edit": - # Returns a dictionary with edit information - editInfo = self.__editParsing( - sectionContent, boundary["start"]) - - # Checks for a parse error and appends it, returning the sections list which stops the parsing - if editInfo["type"] == "parse_error": - sections.append(editInfo) - return self.__getSortedSections(sections) - - # Appends the edit dictionary to sections - sections.append(editInfo) - - elif boundary["type"] == "replyToUser": - # Returns a dictionary with reply-to information - replyToInfo = self.__replyToParsing( - sectionContent, boundary["start"]) - - # Checks for a parse error and appends it, returning the sections list which stops the parsing - if replyToInfo["type"] == "parse_error": - sections.append(replyToInfo) - return self.__getSortedSections(sections) - - # Appends the reply-to to sections - sections.append(replyToInfo) - - elif boundary["type"] == "status": - # Returns a dictionary with status information - statusInfo = self.__statusParsing( - sectionContent, boundary["start"]) - - if statusInfo["type"] == "parse_error": - sections.append(statusInfo) - return self.__getSortedSections(sections) - - # Appends the status to sections - sections.append(statusInfo) - - elif boundary["type"] == "replyFromUser": - # Returns a dictionary with userReply information - replyFromInfo = self.__userReplyParsing( - sectionContent, boundary["start"]) - - if replyFromInfo["type"] == "parse_error": - sections.append(replyFromInfo) - return self.__getSortedSections(sections) - - # Appends the replyFrom to sections - sections.append(replyFromInfo) - - sortedSections = self.__getSortedSections(sections) - - return sortedSections - # return sections - - def __directoryParsing(self, directoryStartLine: int) -> dict: - """Returns a dictionary with directory information - - Example: - Name: Nestor Fabian Rodriguez Buitrago - Login: rodri563 - Computer: ce-205-38 (128.46.205.67) - Location: HAMP G230 - Email: rodri563@purdue.edu - Phone: 7654766893 - Office: HAMP G230 - UNIX Dir: /home/bridge/b/rodri563 - Zero Dir: U=\\bridge.ecn.purdue.edu\rodri563 - User ECNDB: http://eng.purdue.edu/jump/2e8399a - Host ECNDB: http://eng.purdue.edu/jump/2e83999 - Subject: Autocad installation - - Args: - directoryStartLine (int): line number within the item that the directory starts on - - Returns: - dict: dictionary that splits each line within the directory into a key and a value - """ - directoryInformation = {"type": "directory_information"} - - directoryPossibleKeys = [ - "Name", - "Login", - "Computer", - "Location", - "Email", - "Phone", - "Office", - "UNIX Dir", - "Zero Dir", - "User ECNDB", - "Host ECNDB", - "Subject" - ] - # Executies until the directory start line is greater than the directory ending line - while True: - - # Returns the line number at directory start line - info = self.__raw_tem[directoryStartLine] - - # Breaks the loop if it encountrs a newline, signifying the end of the directory information - if info == "\n": - - break - - else: - - # Removes white including space, newlines, and tabs from the directory info line - strippedInfo = info.strip() - - # Attempts to find ": " but will accept ":", denoting a blank entry for a directory item - if ": " in strippedInfo: - - # Seperates the directory info line into two variables, the first variable being the key, the second being the value - # swt1 - key, value = strippedInfo.split(": ", 1) - - if key in directoryPossibleKeys: - # Adds the key value pair to the directory info dictionary - directoryInformation[key] = value - else: - # Casts the list type on to a dictionary - dictionaryList = list(directoryInformation) - # Length of dictionary list - lenDictionaryList = len(dictionaryList) - # The last key appended to the directory dictionary - lastKeyAppended = dictionaryList[lenDictionaryList - 1] - - directoryInformation[lastKeyAppended] = directoryInformation[lastKeyAppended] + \ - " " + strippedInfo - - elif ":" in strippedInfo: - - # Seperates the directory info line into two variables, the first variable being the key, the second being the value - key, value = strippedInfo.split(":", 1) - - if key in directoryPossibleKeys: - # Adds the key value pair to the directory info dictionary - directoryInformation[key] = value - else: - # Casts the list type on to a dictionary - dictionaryList = list(directoryInformation) - # Length of dictionary list - lenDictionaryList = len(dictionaryList) - # The last key appended to the directory dictionary - lastKeyAppended = dictionaryList[lenDictionaryList - 1] - - directoryInformation[lastKeyAppended] = directoryInformation[lastKeyAppended] + \ - " " + strippedInfo - - # Signifies that this line belongs to the most previous line - elif ": " not in strippedInfo and ":" not in strippedInfo: - # Casts the list type on to a dictionary - dictionaryList = list(directoryInformation) - # Length of dictionary list - lenDictionaryList = len(dictionaryList) - # The last key appended to the directory dictionary - lastKeyAppended = dictionaryList[lenDictionaryList - 1] - - directoryInformation[lastKeyAppended] = directoryInformation[lastKeyAppended] + \ - " " + strippedInfo - # Counter to denote the end of the directory - directoryStartLine = directoryStartLine + 1 - - # Returns the directory information dictionary - return directoryInformation - - def __assignmentParsing(self, contentStart: int) -> list: - """Returns a list with assignment information dictionaries - - Example: - Assigned-To: campb303 - Assigned-To-Updated-Time: Tue, 23 Jun 2020 13:27:00 EDT - Assigned-To-Updated-By: campb303 - - Args: - contentStart (int): line number where the content starts - - Returns: - list: [ - {"type": "assignment", - "datetime": datetime of the assignment, - "by": user who initiated the assignment, - "to": user who was assigned - }, - ] - """ - assignmentList = [] - - # Assignment Information - assignedBy = "" - assignedDateTime = "" - assignedTo = "" - - # Parses the header looking for assignment delimeters and stores info into their respective variables - for headerContent in range(0, contentStart): - - line = self.__raw_tem[headerContent] - - # Gets who the Item was assigned to - if line.startswith("Assigned-To: "): - - assignedTo = ( - re.search("(?<=Assigned-To: )(.*)", line)).group() - - # Gets the date the Item was assigned - elif line.startswith("Assigned-To-Updated-Time: "): - - dateFromLine = ( - re.search("(?<=Assigned-To-Updated-Time: )(.*)", line)).group() - - assignedDateTime = format_date_string(dateFromLine) - - # Gets who assigned the Item - elif line.startswith("Assigned-To-Updated-By: "): - - assignedBy = ( - re.search("(?<=Assigned-To-Updated-By: )(.*)", line)).group() - - # Appends the assignment to the sections list - assignmentList.append( - {"type": "assignment", - "datetime": assignedDateTime, - "by": assignedBy, - "to": assignedTo} - ) - - return assignmentList - - def __initialMessageParsing(self, content: list) -> dict: - """Returns a dictionary with initial message information - - Example: - \n - Testtest\n - \n - - Args: - content (list): content of the initial message - - Returns: - dict: - "type": "initial_message", - "datetime": datetime the initial message was sent, - "from_name": from_name, - "from_email": user_email, - "to": [{email, name}], - "cc": [{email, name}], - "subject": initial message subject - "content": content of the initial message - """ - initialMessageDictionary = {} - - initialMessageDictionary["type"] = "initial_message" - - # Gets the initial message date from the header - rawMessageDateStr = self.__get_most_recent_header_by_type("Date") - - # Sets datetime in the intialMessage dictionary to UTC formatted date - initialMessageDictionary["datetime"] = format_date_string( - rawMessageDateStr) - - initialMessageDictionary["from_name"] = self.__parse_from_data( - data="user_name") - - initialMessageDictionary["from_email"] = self.__parse_from_data( - data="user_email") - - # Stores list of dictionaries for the recipients of the initial message - initialMessageDictionary["to"] = [] - - # Parses the header looking for recipients of the initial message and stores it in a list of tuples - rawMessageRecipientsList = email.utils.getaddresses( - [self.__get_most_recent_header_by_type("To")]) - - # Parses the CC list and stores the cc recipient information in a list of dictionaries - for recipients in rawMessageRecipientsList: - - initialMessageDictionary["to"].append( - {"name": recipients[0], - "email": recipients[1]} - ) - - # Stores list of dictionaries for CC information - initialMessageDictionary["cc"] = [] - - # Parses the header looking for CC recipients of the initial message and stores it in a list of tuples - rawMessageCCList = email.utils.getaddresses( - [self.__get_most_recent_header_by_type("CC")]) - - # Parses the CC list and stores the cc recipient information in a list of dictionaries - for ccRecipients in rawMessageCCList: - - initialMessageDictionary["cc"].append( - {"name": ccRecipients[0], - "email": ccRecipients[1]} - ) - - initialMessageDictionary["subject"] = self.__get_most_recent_header_by_type( - "Subject") - - # Removes unecessary newlines from the begining and the end of the initial message - initialMessageDictionary["content"] = self.__getFormattedSectionContent( - content) - - return initialMessageDictionary - - def __editParsing(self, content: list, lineNum: int) -> dict: - """Returns a dictionary with edit information - - Example: - *** Edited by: campb303 at: 06/23/20 13:27:56 ***\n - \n - This be an edit my boy\n - \n - \n - \n - - Args: - content (list): content of an edit - lineNum (int): line number of an edit within an item - - Returns: - dict: a dictionary with these keys, - "type": "edi", - "by": initiator of the edit, - "datetime": datetime of the edit, - "content": content of the edit - """ - - # Edit Info dictionary - editInfo = {} - - for count, line in enumerate(content): - if line == "===============================================\n": - errorMessage = "Reply-from-user ending delimter encountered without Reply-from-user starting delimter" - return self.__errorParsing(line, lineNum + count + 1, errorMessage) - - editInfo["type"] = "edit" - - delimiterLine = content[0] - # Parses for the author of the edit, which is located between the "*** Edited by: " and " at:" substrings - try: - editInfo["by"] = ( - re.search("(?<=\*{3} Edited by: )(.*)(?= at:)", delimiterLine)).group() - except: - errorMessage = "*** Edited by: [username] at: [date and time] ***\n" - return self.__errorParsing(delimiterLine, lineNum, errorMessage) - - try: - # Parses for the date and time of the edit, which is located between the " at: " and "***\n" substrings - dateTimeString = ( - re.search("(?<= at: )(.*)(?= \*\*\*\n)", delimiterLine)).group() - except: - # Returns an error message if there is no space after "at:" - errorMessage = "*** Edited by: [username] at: [date and time] ***\n" - return self.__errorParsing(delimiterLine, lineNum, errorMessage) - - # Attempts to format the date and time into utc format - editInfo["datetime"] = format_date_string(dateTimeString) - - # Remove the delimiter String and unecessary newlines - editInfo["content"] = self.__getFormattedSectionContent(content) - - return editInfo - - def __replyToParsing(self, content: list, lineNum: int) -> dict: - """Returns a dictionary with reply to user information - - Example: - *** Replied by: campb303 at: 06/23/20 13:28:18 ***\n - \n - This be a reply my son\n - \n - Justin\n - ECN\n - \n - - Args: - content (list): content of a reply to user - lineNum (int): line number of a reply to user in an item - - Returns: - dict: a dictionary with these keys, - "type": "reply_to_user", - "by": initiator of the reply to user, - "datetime": datetime of the reply to user, - "content": content of the reply to user - """ - replyInfo = {} - - replyInfo["type"] = "reply_to_user" - - delimiterLine = content[0] - - for count, line in enumerate(content): - if line == "===============================================\n": - errorMessage = "Reply-from-user ending delimter encountered without Reply-from-user starting delimter" - return self.__errorParsing(line, lineNum + count + 1, errorMessage) - - try: - # Parses for the author of the reply, which is located between the "*** Replied by: " and " at:" substrings - replyInfo["by"] = ( - re.search("(?<=\*{3} Replied by: )(.*)(?= at:)", delimiterLine)).group() - except: - errorMessage = "*** Replied by: [username] at: [date and time] ***\n" - return self.__errorParsing(delimiterLine, lineNum, errorMessage) - - # Parses for the date and time of the reply, which is located between the " at: " and "***\n" substrings - try: - dateTimeString = ( - re.search("(?<= at: )(.*)(?= \*\*\*\n)", delimiterLine)).group() - except: - errorMessage = "*** Replied by: [username] at: [date and time] ***\n" - return self.__errorParsing(delimiterLine, lineNum, errorMessage) - - # Formats date to UTC - replyInfo["datetime"] = format_date_string(dateTimeString) - - replyInfo["content"] = self.__getFormattedSectionContent(content) - - return replyInfo - - def __statusParsing(self, content: list, lineNum: int) -> dict: - """Returns a dictionary with status information - - Example: - *** Status updated by: campb303 at: 6/23/2020 13:26:55 ***\n - Dont Delete\n - - Args: - content (list): The content of a status update - lineNum (int): The line number of a status update in an item - - Returns: - dict: a dictionary with these keys, - "type": "status", - "by": initiator of the status update, - "datetime": datetime of the status update, - "content": content of the status update - """ - statusInfo = {} - - statusInfo["type"] = "status" - - delimiterLine = content[0] - - for count, line in enumerate(content): - if line == "===============================================\n": - errorMessage = "Reply-from-user ending delimter encountered without Reply-from-user starting delimter" - return self.__errorParsing(line, lineNum + count + 1, errorMessage) - - # Parses for the author of the status change, which is located between the "*** Status updated by: " and " at:" substrings - try: - statusInfo["by"] = ( - re.search("(?<=\*{3} Status updated by: )(.*)(?= at:)", delimiterLine)).group() - except: - errorMessage = "*** Status updated by: [username] at: [date and time] ***\n" - - return self.__errorParsing(delimiterLine, lineNum, errorMessage) - - # Parses for the date and time of the status change, which is located between the " at: " and "***\n" substrings - try: - dateTimeString = re.search( - "(?<= at: )(.*)(?= \*\*\*\n)", delimiterLine).group() - except: - errorMessage = "*** Status updated by: [username] at: [date and time] ***\n" - - return self.__errorParsing(delimiterLine, lineNum, errorMessage) - - # Formats the date to UTC - statusInfo["datetime"] = format_date_string(dateTimeString) - - # Remove the delimiter String and unecessary newlines - statusInfo["content"] = self.__getFormattedSectionContent(content) - - return statusInfo - - def __userReplyParsing(self, replyContent: list, lineNumber: int) -> dict: - """Returns a dictionary with user reply information - - Example: - === Additional information supplied by user ===\n - \n - Subject: Re: Beepboop\n - From: Justin Campbell \n - Date: Tue, 23 Jun 2020 13:30:45 -0400\n - X-ECN-Queue-Original-Path: /home/pier/e/queue/Attachments/inbox/2020-06-23/212-original.txt\n - X-ECN-Queue-Original-URL: https://engineering.purdue.edu/webqueue/Attachments/inbox/2020-06-23/212-original.txt\n - \n - Huzzah!\n - \n - ===============================================\n - \n - Args: - replyContent (list): The entire section of a reply-from-user - lineNumber (int): The line number of the begining of a reply-from-user section within and item - - Returns: - dict: a dictionary with these keys, - "type": "reply_from_user", - "from_name": name of the user that sent the reply, - "from_email": email of the user that sent the reply, - "subject": subject of the reply, - "datetime": the datetime of the reply, - "cc": [ - {"name": name of the carbon copied recipient, - "email": email of the carbon copied recipient - }, - ] - "content": content of the reply - "headers": [ - {"type": headerType, - "content": content - }, - ] - """ - replyFromInfo = {} - - replyFromInfo["type"] = "reply_from_user" - - replyFromHeaders = [] - newLineCounter = 0 - endingDelimiterCount = 0 - - # Delimiter information line numbers to remove from reply from user - linesToRemove = [] - - # Parses the section content looking for any line that starts with a metadata, also tracks the line - # number with the enumerate function - for lineNum, line in enumerate(replyContent): - - if endingDelimiterCount == 0 and lineNum == len(replyContent) - 1: - errorMessage = "Did not encounter a reply-from-user ending delimiter" - return self.__errorParsing(line, lineNumber + lineNum + 1, errorMessage) - - if newLineCounter == 1 and line != "\n": - - try: - # Append header information for each headr line - headerType, content = line.split(": ", 1) - replyFromHeaders.append( - {"type": headerType, - "content": content - } - ) - except: - lenReplyFromHeaders = len(replyFromHeaders) - if lenReplyFromHeaders == 0: - errorMessage = ("Expected reply-from-user header information:\n" + - "=== Additional information supplied by user ===\n" + - "\n" + - "[Header Type]: [Header Value]\n" + - "\n" - ) - return self.__errorParsing(line, lineNumber + lineNum + 1, errorMessage) - - else: - replyFromHeaders[lenReplyFromHeaders - - 1]["content"] = replyFromHeaders[lenReplyFromHeaders - 1]["content"] + " " + line - - linesToRemove.append(lineNum) - # Checks for a newline and breaks for loop on second occurance of a newline - if line == "\n": - newLineCounter = newLineCounter + 1 - - if newLineCounter == 2 and "datetime" not in replyFromInfo.keys(): - errorMessage = "Expected \"Date: [datetime]\" in the header info" - return self.__errorParsing(line, lineNumber + lineNum + 1, errorMessage) - - elif line == "===============================================\n": - endingDelimiterCount = endingDelimiterCount + 1 - - elif line.startswith("From: ") and newLineCounter == 1: - # Returns a list of one tuples with a name stored in the first index of the tuple and an email stored in the second index of the tuple - emailList = email.utils.getaddresses([line]) - replyFromInfo["from_name"] = emailList[0][0] - replyFromInfo["from_email"] = emailList[0][1] - - elif line.startswith("Subject: ") and newLineCounter == 1: - # Matches everything after "Subject: " - try: - subjectStr = ( - re.search("(?<=Subject: )(.*)", line)).group() - except: - errorMessage = "Expeted syntax of \"Subject: [subject]\"" - return self.__errorParsing(line, lineNumber + lineNum + 1, errorMessage) - - # Formatts the date to UTC - replyFromInfo["subject"] = subjectStr - - elif line.startswith("Date: ") and newLineCounter == 1: - # Matches everything after "Date: " - try: - dateStr = (re.search("(?<=Date: )(.*)", line)).group() - except: - errorMessage = "\"Date: [datetime]\"" - return self.__errorParsing(line, lineNumber + lineNum + 1, errorMessage) - - # Formatts the date to UTC - replyFromInfo["datetime"] = format_date_string(dateStr) - - elif line.startswith("Cc: ") and newLineCounter == 1: - - replyFromInfo["cc"] = [] - - # Returns a list of tuples with email information - recipientsList = email.utils.getaddresses([line]) - - # Parses through the cc tuple list - for cc in recipientsList: - # Stores the cc information in a dictionary and appends it to the ccRecipientsList - replyFromInfo["cc"].append( - {"name": cc[0], - "email": cc[1]} - ) - - # Deletes reduntant lines from the message content in reverse order - for lineNum in sorted(linesToRemove, reverse=True): - replyContent.pop(lineNum) - - # Strips any unnecessary newlines or any delimiters frm the message content - replyFromInfo["content"] = self.__getFormattedSectionContent( - replyContent) - - replyFromInfo["headers"] = replyFromHeaders - - return replyFromInfo - - def __getFormattedSectionContent(self, sectionContent: list) -> list: - """Returns a list with message content that is stripped of unnecessary newlines and begining delimiters - - Example: - *** Edited by: mph at: 02/21/20 10:27:16 ***\n - \n - Still need to rename machines - but the networking issue now seems to \n - be resolved via another ticket.\n - \n - \n - \n - \n - \n - - Args: - sectionContent (list): The section content of a parsed section - - Returns: - list: the section content of a parsed section without any delimiters and unnecessary newlines - """ - # Continually removes the first line of sectionContent if it is a newline or delimiter in each iteration - while len(sectionContent) > 1: - if (sectionContent[0] == "\n" or - sectionContent[0].startswith("*** Edited by: ") or - sectionContent[0].startswith("*** Replied by: ") or - sectionContent[0].startswith("*** Status updated by: ") or - sectionContent[0] == "=== Additional information supplied by user ===\n" or - sectionContent[0] == "===============================================\n" - ): - sectionContent.pop(0) - else: - # Breaks the loop if the first line isn't a newline or delimiter - break - - # Continually removes the last line of sectionContent if it is a newline or delimiter in each iteration - while len(sectionContent) > 1: - # Initializes the Length of sectionContent each iteration of the loop - sectionContentLength = len(sectionContent) - - if (sectionContent[sectionContentLength - 1] == "\n" or - sectionContent[sectionContentLength - - 1] == "===============================================\n" - ): - sectionContent.pop(sectionContentLength - 1) - else: - # Breaks the loop if the last line isn't a newline or delimiter - break - - return sectionContent - - def __errorParsing(self, line: str, lineNum: int, expectedSyntax: str) -> dict: - """Returns a dictionary with error parse information when a line is malformed - - Example: - "*** Status updated by: ewhile at: 5/7/2020 10:59:11 *** sharing between\n" - - Args: - line (str): line of that threw error - lineNum (int): line number in the item that threw error - expectedSyntax (str): a message stating the syntax the line should follow - - Returns: - dict: a dictionary with these keys, - "type": "parse_error", - "datetime": time the error was encountered, - "file_path": path of the item with erroneos line, - "expected": expectedSyntax, - "got": line, - "line_num": lineNum - """ - errorDictionary = {} - - # Type - errorDictionary["type"] = "parse_error" - - # Dateime of the parse error - errorDictionary["datetime"] = format_date_string( - str(datetime.datetime.now())) - - # Item filepath - errorDictionary["file_path"] = str(self.path) - - # Expected value - errorDictionary["expected"] = expectedSyntax - - # line that threw error - errorDictionary["got"] = line - - # line number that threw error - errorDictionary["line_num"] = lineNum - - # returns the error dictionary - return errorDictionary + return body_sections def __getSortedSections(self, sectionsList: list) -> list: """Sorts the sections chronologically by datetime diff --git a/src/webqueue2api/parser/parser.py b/src/webqueue2api/parser/parser.py new file mode 100644 index 0000000..f69056d --- /dev/null +++ b/src/webqueue2api/parser/parser.py @@ -0,0 +1,293 @@ +import pyparsing as pp +import json +import string +import email +import datetime +from .utils import format_date_string +from .errors import ParseError + + + +parsed_item = [] + + + +################################################################################ +# Delimiters +################################################################################ +action_start_delimiter = "*** " +action_end_delimiter = " ***" + +edit_start_delimiter = action_start_delimiter + "Edited by: " +status_start_delimiter = action_start_delimiter + "Status updated by: " +reply_to_user_delimiter = action_start_delimiter + "Replied by: " +reply_from_user_start_delimiter = "=== Additional information supplied by user ===" +reply_from_user_end_delimiter = "===============================================\n" + + + +################################################################################ +# Parse Actions: Callbacks for Rules +################################################################################ +def parse_section_by_type(section_type): + """Returns a function to parse a section based on the section type. + + Args: + section_type (str): The type of section to parse. + Can be "reply_to_user" | "edit" | "reply_from_user" | "status" | "directory_information" | "initial_message" + """ + + def parse_section(original_string: str, match_start_index: int, tokens: pp.ParseResults) -> None: + """Parses section and adds to global section list. + + Args: + original_string (string): The original string passed to PyParsing + match_start_index (int): The character index where the match starts. + tokens (pyparsing.ParseResults): The PyParsing results. + + Raises: + ValueError, OverflowError: If a date cannot be formatted. + ParseError: If unexpected formatting is encountered. + """ + tokens_dictionary = tokens.asDict() + tokens_dictionary["type"] = section_type + + # Remove empty keys + for key in tokens_dictionary.keys(): + if key == "": del tokens_dictionary[key] + + # Parse reply-from-user headers + if section_type == "reply_from_user": + headers = email.message_from_string(tokens_dictionary["headers"]) + headers_list = [] + for key in headers.keys(): + headers_list.append({"type": key, "content": headers[key]}) + + for header in headers_list: + if header["type"] == "Date": + tokens_dictionary["datetime"] = header["content"] + elif header["type"] == "Subject": + tokens_dictionary["subject"] = header["content"] + elif header["type"] == "From": + user_name, user_email = email.utils.parseaddr(header["content"]) + tokens_dictionary["from_name"] = user_name + tokens_dictionary["from_email"] = user_email + elif header["type"].lower() == "cc": + cc_list = [ + { "name": user_name, "email" :user_email } + for user_name, user_email in email.utils.getaddresses([header["content"]]) + ] + tokens_dictionary["cc"] = cc_list + + tokens_dictionary["headers"] = headers_list + + # Format date header + if "datetime" in tokens_dictionary.keys(): + try: + formatted_date = format_date_string(tokens_dictionary["datetime"]) + except (ValueError, OverflowError): + line_number = original_string[:original_string.find(f"{key}: {headers[key]}")].count("\n") + 1 + parsed_item.append({ + "type": "parse_error", + "datetime": format_date_string(str(datetime.datetime.now())), + "expected": "ISO 8601 formatted time string.", + "got": headers[key], + "line_num": line_number + }) + raise ParseError(line_number, f"Could not format date header") + + tokens_dictionary["datetime"] = formatted_date + + # Convert content string to list of lines + if "content" in tokens_dictionary.keys(): + tokens_dictionary["content"] = tokens_dictionary["content"][0].strip() + tokens_dictionary["content"] = tokens_dictionary["content"].splitlines(keepends=True) + + parsed_item.append(tokens_dictionary) + return + + return parse_section + +def check_for_nested_action(original_string, match_start_index, tokens): + """Checks for nested action in reply_from_user. + + Args: + original_string (string): The original string passed to PyParsing + match_start_index (int): The character index where the match starts. + tokens (pyparsing.ParseResults): The PyParsing results. + + Raises: + ParseError: If nested action is found. + """ + token_string = tokens[0] + strings_that_indicate_nesting = [ + edit_start_delimiter, + status_start_delimiter, + reply_to_user_delimiter, + reply_from_user_start_delimiter + ] + + for delimiter in strings_that_indicate_nesting: + if delimiter in token_string: + line_number = 1 + original_string[:match_start_index].count("\n") + token_string[:token_string.find(delimiter)].count("\n") + parsed_item.append({ + "type": "parse_error", + "datetime": format_date_string(str(datetime.datetime.now())), + "expected": reply_from_user_end_delimiter, + "got": f"Found nested action '{delimiter}' in reply from user", + "line_num": line_number + }) + raise ParseError(line_number, f"Found nested action '{delimiter}' in reply from user") + return + +def error_handler(original_string, match_start_index, tokens): + token_string = tokens[0][0] + + parse_error = { + "type": "parse_error", + 'datetime': format_date_string(str(datetime.datetime.now())), + } + + if token_string == reply_from_user_start_delimiter: + expected_token = reply_from_user_end_delimiter + line_number = original_string.count('\n') + 1 + + parse_error["expected"] = expected_token + parse_error["got"] = "End of file" + parse_error["line_num"] = line_number + parsed_item.append(parse_error) + raise ParseError(line_number, f"No reply from user end delimiter found") + else: + expected_token = f"Action delimiter starting with {action_start_delimiter} or {reply_from_user_start_delimiter}" + line_number = (original_string[:match_start_index]).count('\n') + 1 + + parse_error["expected"] = f"Action start delimiter: '{action_start_delimiter}' or '{reply_from_user_start_delimiter}'" + parse_error["got"] = token_string + parse_error["line_num"] = line_number + parsed_item.append(parse_error) + raise ParseError(line_number, f"No action start delimiter found") + + +################################################################################ +# Rules +################################################################################ +header_rule = pp.SkipTo("\n\n").leaveWhitespace() + +directory_rule = pp.Dict( + pp.White("\n").suppress() + + pp.Optional(pp.Group("Name" + pp.Literal(":").suppress() + pp.SkipTo(pp.LineEnd()))) + + pp.Optional(pp.Group("Login" + pp.Literal(":").suppress() + pp.SkipTo(pp.LineEnd()))) + + pp.Optional(pp.Group("Computer" + pp.Literal(":").suppress() + pp.SkipTo(pp.LineEnd()))) + + pp.Optional(pp.Group("Location" + pp.Literal(":").suppress() + pp.SkipTo(pp.LineEnd()))) + + pp.Optional(pp.Group("Email" + pp.Literal(":").suppress() + pp.SkipTo(pp.LineEnd()))) + + pp.Optional(pp.Group("Phone" + pp.Literal(":").suppress() + pp.SkipTo(pp.LineEnd()))) + + pp.Optional(pp.Group("Office" + pp.Literal(":").suppress() + pp.SkipTo(pp.LineEnd()))) + + pp.Optional(pp.Group("UNIX Dir" + pp.Literal(":").suppress() + pp.SkipTo(pp.LineEnd()))) + + pp.Optional(pp.Group("Zero Dir" + pp.Literal(":").suppress() + pp.SkipTo(pp.LineEnd()))) + + pp.Optional(pp.Group("User ECNDB" + pp.Literal(":").suppress() + pp.SkipTo(pp.LineEnd()))) + + pp.Optional(pp.Group("Host ECNDB" + pp.Literal(":").suppress() + pp.SkipTo(pp.LineEnd()))) + + pp.Optional(pp.Group("Subject" + pp.Literal(":").suppress() + pp.SkipTo(pp.LineEnd()))) + + pp.White("\n\n").suppress() +).setParseAction(parse_section_by_type("directory_information")) + +initial_message_rule = pp.Group( + pp.SkipTo( + pp.Literal(reply_from_user_start_delimiter) + | pp.Literal(action_start_delimiter) + | pp.StringEnd() + ).leaveWhitespace() +).setResultsName("content").setParseAction(parse_section_by_type("initial_message")) + +reply_from_user_rule = ( + (reply_from_user_start_delimiter + pp.OneOrMore(pp.LineEnd())).suppress() + + pp.SkipTo("\n\n").setResultsName("headers") + + (pp.Group(pp.SkipTo(reply_from_user_end_delimiter + pp.LineEnd()).setParseAction(check_for_nested_action)).setResultsName("content")) + + (pp.Literal(reply_from_user_end_delimiter) + pp.LineEnd()).suppress() + + pp.ZeroOrMore(pp.LineEnd()).suppress() +).leaveWhitespace().setParseAction(parse_section_by_type("reply_from_user")) + +reply_to_user_rule = ( + pp.Literal(reply_to_user_delimiter).suppress() + + pp.Word(pp.alphanums).setResultsName("by")+ + pp.Literal(" at: ").suppress() + + pp.SkipTo(action_end_delimiter + pp.LineEnd()).setResultsName("datetime") + + (pp.Literal(action_end_delimiter) + pp.LineEnd()).suppress() + + pp.Group( + pp.SkipTo( + pp.Literal(reply_from_user_start_delimiter) + | pp.Literal(action_start_delimiter) + | pp.StringEnd() + ) + ).setResultsName("content") +).leaveWhitespace().setParseAction(parse_section_by_type("reply_to_user")) + +edit_rule = ( + pp.Literal(edit_start_delimiter).suppress() + + pp.Word(pp.alphanums).setResultsName("by") + + pp.Literal(" at: ").suppress() + + pp.SkipTo(action_end_delimiter + pp.LineEnd()).setResultsName("datetime") + + (pp.Literal(action_end_delimiter) + pp.LineEnd()).suppress() + + pp.Group( + pp.SkipTo( + pp.Literal(reply_from_user_start_delimiter) + | pp.Literal(action_start_delimiter) + | pp.stringEnd() + ) + ).setResultsName("content") +).leaveWhitespace().setParseAction(parse_section_by_type("edit")) + +status_rule = ( + pp.Literal(status_start_delimiter).suppress() + + pp.Word(pp.alphanums).setResultsName("by") + + pp.Literal(" at: ").suppress() + + pp.SkipTo(action_end_delimiter + pp.LineEnd()).setResultsName("datetime") + + (pp.Literal(action_end_delimiter) + pp.LineEnd()).suppress() + + pp.Group( + pp.SkipTo( + pp.Literal(reply_from_user_start_delimiter) + | pp.Literal(action_start_delimiter) + | pp.StringEnd() + ) + ).setResultsName("content") +).leaveWhitespace().setParseAction(parse_section_by_type("status")) + +error_rule = pp.Group( + pp.SkipTo(pp.LineEnd()) +).setParseAction(error_handler) + +item_rule = ( + header_rule + + pp.Optional(directory_rule).suppress() + + initial_message_rule + + pp.ZeroOrMore( + reply_from_user_rule + | reply_to_user_rule + | edit_rule + | status_rule + ) + pp.Optional(error_rule) +) + + + +def parse_item(item_body: string, raise_on_error: bool = False) -> list: + """Accepts string of an Item body and returns JSON serializable list of dictionary with formatted action types. + + Args: + item_body (string): The string of the item to be parsed. + raise_on_error (bool): If true, a ParseError is raised when parsing error encountered. Otherwise, a parse error dictionary is added to the return value before being returned. Defaults to False. + + Returns: + list: List of actions as ordered in the item. Does not include initial message metadata or assignments. + + Raises: + ParseError: If raise_on_error is True, raises ParseError when parsing error occurs. Otherwise adds parse_error acction to return value. + """ + if raise_on_error: + item_rule.parseString(item_body) + else: + try: + item_rule.parseString(item_body) + except ParseError: + pass + + return parsed_item \ No newline at end of file From 5738191591522d67a7a80488dd6bde04d8ddaa99 Mon Sep 17 00:00:00 2001 From: benne238 Date: Mon, 28 Jun 2021 15:11:14 -0400 Subject: [PATCH 07/56] Fixed bug where the end of the string was not matched --- src/webqueue2api/parser/parser.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/webqueue2api/parser/parser.py b/src/webqueue2api/parser/parser.py index f69056d..261e0d6 100644 --- a/src/webqueue2api/parser/parser.py +++ b/src/webqueue2api/parser/parser.py @@ -214,10 +214,9 @@ def error_handler(original_string, match_start_index, tokens): (pp.Literal(action_end_delimiter) + pp.LineEnd()).suppress() + pp.Group( pp.SkipTo( - pp.Literal(reply_from_user_start_delimiter) + pp.Literal(reply_from_user_start_delimiter) | pp.Literal(action_start_delimiter) - | pp.StringEnd() - ) + ) | pp.SkipTo(pp.StringEnd(), include=True) ).setResultsName("content") ).leaveWhitespace().setParseAction(parse_section_by_type("reply_to_user")) @@ -231,8 +230,7 @@ def error_handler(original_string, match_start_index, tokens): pp.SkipTo( pp.Literal(reply_from_user_start_delimiter) | pp.Literal(action_start_delimiter) - | pp.stringEnd() - ) + ) | pp.SkipTo(pp.StringEnd(), include=True) ).setResultsName("content") ).leaveWhitespace().setParseAction(parse_section_by_type("edit")) @@ -244,10 +242,9 @@ def error_handler(original_string, match_start_index, tokens): (pp.Literal(action_end_delimiter) + pp.LineEnd()).suppress() + pp.Group( pp.SkipTo( - pp.Literal(reply_from_user_start_delimiter) + pp.Literal(reply_from_user_start_delimiter) | pp.Literal(action_start_delimiter) - | pp.StringEnd() - ) + ) | pp.SkipTo(pp.StringEnd(), include=True) ).setResultsName("content") ).leaveWhitespace().setParseAction(parse_section_by_type("status")) From c7e0d0a26fde580647c316c004f0bfe0af4c8b26 Mon Sep 17 00:00:00 2001 From: benne238 Date: Tue, 29 Jun 2021 11:27:55 -0400 Subject: [PATCH 08/56] created function to append the initial message headers to the parsed initial message content --- src/webqueue2api/parser/item.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/webqueue2api/parser/item.py b/src/webqueue2api/parser/item.py index ede4a5f..d6f893c 100644 --- a/src/webqueue2api/parser/item.py +++ b/src/webqueue2api/parser/item.py @@ -225,6 +225,27 @@ def __parseSections(self) -> list: return body_sections + def __add_initial_message_headers(self, initial_message: dict) -> dict: + """Adds header information to the intial message + + Example: + [example] + + Args: + initial_message (dict): the intial message dictionary without any headers + + Returns: + dict: modified initial message dictionary to include headers + """ + initial_message["datetime"] = self.date_received + initial_message["from_name"] = self.user_name + initial_message["from_email"] = self.user_email + initial_message["to"] = self.__get_most_recent_header_by_type("To") + initial_message["cc"] = self.__get_most_recent_header_by_type("CC") + initial_message["subject"] = self.subject + + return initial_message + def __getSortedSections(self, sectionsList: list) -> list: """Sorts the sections chronologically by datetime From 0f69c1885cdd641f193ae68fc01ff67ca84a79ff Mon Sep 17 00:00:00 2001 From: benne238 Date: Tue, 29 Jun 2021 12:03:54 -0400 Subject: [PATCH 09/56] modifed __add_initial_message_headers to output the expected format for the "to" and "cc" keys --- src/webqueue2api/parser/item.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/webqueue2api/parser/item.py b/src/webqueue2api/parser/item.py index d6f893c..5a1952a 100644 --- a/src/webqueue2api/parser/item.py +++ b/src/webqueue2api/parser/item.py @@ -63,7 +63,7 @@ def __init__(self, queue: str, number: int, headers_only: bool = False) -> None: self.path = Path(config.queue_directory, self.queue, str(self.number)) if not self.path.exists(): raise ItemDoesNotExistError(str(self.path)) - + self.last_updated = self.__get_time_last_updated() self.__raw_item = self.__get_raw_item() self.headers = self.__parse_headers() @@ -222,7 +222,7 @@ def __parseSections(self) -> list: # Parse body body_sections = parse_item(raw_item_as_string) - + return body_sections def __add_initial_message_headers(self, initial_message: dict) -> dict: @@ -237,11 +237,20 @@ def __add_initial_message_headers(self, initial_message: dict) -> dict: Returns: dict: modified initial message dictionary to include headers """ + raw_cc = self.__get_most_recent_header_by_type("CC") + raw_to = self.__get_most_recent_header_by_type("To") + initial_message["datetime"] = self.date_received initial_message["from_name"] = self.user_name initial_message["from_email"] = self.user_email - initial_message["to"] = self.__get_most_recent_header_by_type("To") - initial_message["cc"] = self.__get_most_recent_header_by_type("CC") + initial_message["to"] = [ + { "name": user_name, "email": user_email } + for user_name, user_email in email.utils.getaddresses([raw_to]) + ] + initial_message["cc"] = [ + { "name": user_name, "email": user_email } + for user_name, user_email in email.utils.getaddresses([raw_cc]) + ] initial_message["subject"] = self.subject return initial_message From 42bb13526801b18ce67140054ecfae1d9a1731d2 Mon Sep 17 00:00:00 2001 From: benne238 Date: Tue, 29 Jun 2021 12:06:46 -0400 Subject: [PATCH 10/56] Moved the content attribute in item to be defined after all the other attributes --- src/webqueue2api/parser/item.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webqueue2api/parser/item.py b/src/webqueue2api/parser/item.py index 5a1952a..c0fd134 100644 --- a/src/webqueue2api/parser/item.py +++ b/src/webqueue2api/parser/item.py @@ -67,7 +67,6 @@ def __init__(self, queue: str, number: int, headers_only: bool = False) -> None: self.last_updated = self.__get_time_last_updated() self.__raw_item = self.__get_raw_item() self.headers = self.__parse_headers() - if not headers_only: self.content = self.__parseSections() self.is_locked = self.__check_is_locked() self.user_email = self.__parse_from_data(data="user_email") self.user_name = self.__parse_from_data(data="user_name") @@ -79,6 +78,7 @@ def __init__(self, queue: str, number: int, headers_only: bool = False) -> None: self.department = self.__get_most_recent_header_by_type("Department") self.building = self.__get_most_recent_header_by_type("Building") self.date_received = format_date_string(self.__get_most_recent_header_by_type("Date")) + if not headers_only: self.content = self.__parseSections() self.json_data = self.__generate_json_data() def __generate_json_data(self) -> dict: From e7e7687bf403c7870b2de2e978bcd4a4c2fb3f35 Mon Sep 17 00:00:00 2001 From: benne238 Date: Tue, 29 Jun 2021 12:08:10 -0400 Subject: [PATCH 11/56] add_intitial_message_headers function call from the parseSections function --- test.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 test.py diff --git a/test.py b/test.py new file mode 100644 index 0000000..1e0ef9d --- /dev/null +++ b/test.py @@ -0,0 +1,21 @@ +import webqueue2api +import webqueue2api.parser +#import webqueue2api.parser.queue as queueueu +import json +import os + +if __name__ == "__main__": + webqueue2api.parser.config.queues_to_ignore = ["archives", "drafts", "inbox", "coral", "linux"] + queues = webqueue2api.load_queues() + #if "linux" not in queues: print("No linux") + for queue in queues: + for item in queue.items: + os.system('cls||clear') + item = webqueue2api.parser.Item(item.queue, item.number, False) + print(item.queue + str(item.number)) + print(json.dumps(item.content, indent=2)) + input() + + print(queues) + #for queue in queues: + #json.dumps(webqueue2api.Item("ce", 7).content, indent=2) \ No newline at end of file From b701cdac5ea8b14524015a52cf234ca4fe0a7b44 Mon Sep 17 00:00:00 2001 From: benne238 Date: Tue, 29 Jun 2021 12:27:10 -0400 Subject: [PATCH 12/56] Revert "add_intitial_message_headers function call from the parseSections function" This reverts commit e7e7687bf403c7870b2de2e978bcd4a4c2fb3f35. --- test.py | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 test.py diff --git a/test.py b/test.py deleted file mode 100644 index 1e0ef9d..0000000 --- a/test.py +++ /dev/null @@ -1,21 +0,0 @@ -import webqueue2api -import webqueue2api.parser -#import webqueue2api.parser.queue as queueueu -import json -import os - -if __name__ == "__main__": - webqueue2api.parser.config.queues_to_ignore = ["archives", "drafts", "inbox", "coral", "linux"] - queues = webqueue2api.load_queues() - #if "linux" not in queues: print("No linux") - for queue in queues: - for item in queue.items: - os.system('cls||clear') - item = webqueue2api.parser.Item(item.queue, item.number, False) - print(item.queue + str(item.number)) - print(json.dumps(item.content, indent=2)) - input() - - print(queues) - #for queue in queues: - #json.dumps(webqueue2api.Item("ce", 7).content, indent=2) \ No newline at end of file From 3951832dcc0a65c2d9f6dd54e3205745fd3409ab Mon Sep 17 00:00:00 2001 From: benne238 Date: Tue, 29 Jun 2021 12:29:29 -0400 Subject: [PATCH 13/56] return initial message section with appropriate headers --- src/webqueue2api/parser/item.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/webqueue2api/parser/item.py b/src/webqueue2api/parser/item.py index c0fd134..3c7b166 100644 --- a/src/webqueue2api/parser/item.py +++ b/src/webqueue2api/parser/item.py @@ -223,6 +223,12 @@ def __parseSections(self) -> list: # Parse body body_sections = parse_item(raw_item_as_string) + # Add initial message headers to intial message section + for index, section in enumerate(body_sections): + if section["type"] == "initial_message": + body_sections[index] = self.__add_initial_message_headers(section) + break + return body_sections def __add_initial_message_headers(self, initial_message: dict) -> dict: From b9d49459121f9a511dd4ef2cfdb594f9d9ce1406 Mon Sep 17 00:00:00 2001 From: benne238 Date: Tue, 29 Jun 2021 12:34:26 -0400 Subject: [PATCH 14/56] formatted function names to camel case --- src/webqueue2api/parser/item.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/webqueue2api/parser/item.py b/src/webqueue2api/parser/item.py index 3c7b166..5e89193 100644 --- a/src/webqueue2api/parser/item.py +++ b/src/webqueue2api/parser/item.py @@ -78,7 +78,7 @@ def __init__(self, queue: str, number: int, headers_only: bool = False) -> None: self.department = self.__get_most_recent_header_by_type("Department") self.building = self.__get_most_recent_header_by_type("Building") self.date_received = format_date_string(self.__get_most_recent_header_by_type("Date")) - if not headers_only: self.content = self.__parseSections() + if not headers_only: self.content = self.__parse_sections() self.json_data = self.__generate_json_data() def __generate_json_data(self) -> dict: @@ -216,7 +216,7 @@ def __parse_headers(self) -> list: return headers - def __parseSections(self) -> list: + def __parse_sections(self) -> list: # Convert list of lines to single string raw_item_as_string = "".join(self.__raw_item) @@ -261,7 +261,7 @@ def __add_initial_message_headers(self, initial_message: dict) -> dict: return initial_message - def __getSortedSections(self, sectionsList: list) -> list: + def __get_sorted_sections(self, sectionsList: list) -> list: """Sorts the sections chronologically by datetime Example: From de2dd76e350e676533b5d3a057d173b8b91b400c Mon Sep 17 00:00:00 2001 From: benne238 Date: Tue, 29 Jun 2021 12:46:46 -0400 Subject: [PATCH 15/56] add function call to __get_sorted_sections to sort the parsed item sections by the datetime key --- src/webqueue2api/parser/item.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/webqueue2api/parser/item.py b/src/webqueue2api/parser/item.py index 5e89193..82119be 100644 --- a/src/webqueue2api/parser/item.py +++ b/src/webqueue2api/parser/item.py @@ -228,7 +228,9 @@ def __parse_sections(self) -> list: if section["type"] == "initial_message": body_sections[index] = self.__add_initial_message_headers(section) break - + + body_sections = self.__get_sorted_sections(body_sections) + return body_sections def __add_initial_message_headers(self, initial_message: dict) -> dict: From 4dfb974a69150e18334ac1dd357c2282679dbce2 Mon Sep 17 00:00:00 2001 From: benne238 Date: Tue, 29 Jun 2021 13:22:59 -0400 Subject: [PATCH 16/56] add assignment parsing to item.py --- src/webqueue2api/parser/item.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/webqueue2api/parser/item.py b/src/webqueue2api/parser/item.py index 82119be..4f6c585 100644 --- a/src/webqueue2api/parser/item.py +++ b/src/webqueue2api/parser/item.py @@ -228,6 +228,9 @@ def __parse_sections(self) -> list: if section["type"] == "initial_message": body_sections[index] = self.__add_initial_message_headers(section) break + + # Add assignment sections to all the other sections + body_sections.extend(self.__get_assignments()) body_sections = self.__get_sorted_sections(body_sections) @@ -302,6 +305,22 @@ def __get_sorted_sections(self, sectionsList: list) -> list: return sortedSections + def __get_assignments(self) -> list: + assignment_list = [] + + for index, header in enumerate(self.headers): + if header["type"] == "Assigned-To": + assignment = { + "type": "assignment", + "to": self.headers[index]["content"], + "datetime": self.headers[index + 1]["content"], + "by": self.headers[index + 2]["content"] + + } + assignment_list.append(assignment) + + return assignment_list + def __check_is_locked(self) -> Union[str, bool]: """Returns a string info about the lock if true and a bool False if false From d858c0cd846683cb10f21826552e01453586fa61 Mon Sep 17 00:00:00 2001 From: benne238 Date: Wed, 30 Jun 2021 10:21:09 -0400 Subject: [PATCH 17/56] modified repy_from_user_end_delimiter to not include an ending newline --- src/webqueue2api/parser/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webqueue2api/parser/parser.py b/src/webqueue2api/parser/parser.py index 261e0d6..3010939 100644 --- a/src/webqueue2api/parser/parser.py +++ b/src/webqueue2api/parser/parser.py @@ -22,7 +22,7 @@ status_start_delimiter = action_start_delimiter + "Status updated by: " reply_to_user_delimiter = action_start_delimiter + "Replied by: " reply_from_user_start_delimiter = "=== Additional information supplied by user ===" -reply_from_user_end_delimiter = "===============================================\n" +reply_from_user_end_delimiter = "===============================================" From 56798fb0ccc364dba5280ba3790e624d1c7e1718 Mon Sep 17 00:00:00 2001 From: benne238 Date: Wed, 30 Jun 2021 10:23:07 -0400 Subject: [PATCH 18/56] added a condition that checks for an ending reply from user delimiter to indicate malformed header information in the reply from user section --- src/webqueue2api/parser/parser.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/webqueue2api/parser/parser.py b/src/webqueue2api/parser/parser.py index 3010939..7afb665 100644 --- a/src/webqueue2api/parser/parser.py +++ b/src/webqueue2api/parser/parser.py @@ -148,7 +148,19 @@ def error_handler(original_string, match_start_index, tokens): 'datetime': format_date_string(str(datetime.datetime.now())), } - if token_string == reply_from_user_start_delimiter: + if token_string == reply_from_user_start_delimiter and \ + reply_from_user_end_delimiter in original_string[match_start_index:]: + + expected_token = "\n\n" + line_number = original_string[:original_string[match_start_index:].find(reply_from_user_end_delimiter) + match_start_index].count("\n") + 1 + + parse_error["expected"] = expected_token + parse_error["got"] = reply_from_user_end_delimiter + parse_error["line_num"] = line_number + parsed_item.append(parse_error) + raise ParseError(line_number, f"No reply from user end delimiter found") + + elif token_string == reply_from_user_start_delimiter: expected_token = reply_from_user_end_delimiter line_number = original_string.count('\n') + 1 From f61860e4f8a46bc6d24105907972d04a31e127cf Mon Sep 17 00:00:00 2001 From: benne238 Date: Wed, 30 Jun 2021 10:24:51 -0400 Subject: [PATCH 19/56] fixed error message for header information not seperated from content by a newline --- src/webqueue2api/parser/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webqueue2api/parser/parser.py b/src/webqueue2api/parser/parser.py index 7afb665..6d43170 100644 --- a/src/webqueue2api/parser/parser.py +++ b/src/webqueue2api/parser/parser.py @@ -158,7 +158,7 @@ def error_handler(original_string, match_start_index, tokens): parse_error["got"] = reply_from_user_end_delimiter parse_error["line_num"] = line_number parsed_item.append(parse_error) - raise ParseError(line_number, f"No reply from user end delimiter found") + raise ParseError(line_number, f"No newline found after header information") elif token_string == reply_from_user_start_delimiter: expected_token = reply_from_user_end_delimiter From 8289ae75e6692286ce6979185df7bf261f0875eb Mon Sep 17 00:00:00 2001 From: benne238 Date: Wed, 30 Jun 2021 11:34:59 -0400 Subject: [PATCH 20/56] Add logic to return an error_parse if a header is not formatted correctly in the reply_from_user section --- src/webqueue2api/parser/parser.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/webqueue2api/parser/parser.py b/src/webqueue2api/parser/parser.py index 6d43170..44db3db 100644 --- a/src/webqueue2api/parser/parser.py +++ b/src/webqueue2api/parser/parser.py @@ -2,10 +2,12 @@ import json import string import email +from email.policy import Policy +import email.errors import datetime from .utils import format_date_string from .errors import ParseError - +Policy.raise_on_defect = True parsed_item = [] @@ -58,7 +60,23 @@ def parse_section(original_string: str, match_start_index: int, tokens: pp.Parse # Parse reply-from-user headers if section_type == "reply_from_user": - headers = email.message_from_string(tokens_dictionary["headers"]) + try: + headers = email.message_from_string(tokens_dictionary["headers"]) + except email.errors.MissingHeaderBodySeparatorDefect as e: + parse_error = { + "type": "parse_error", + "datetime": format_date_string(str(datetime.datetime.now())), + "expected": "Header information with a key/value pair seperated by a colon or a newline to seperate the header from the content", + } + headers_list = tokens_dictionary["headers"].splitlines(keepends=True) + for line in headers_list: + if ":" not in line and not line.startswith(" "): + parse_error["got"] = line + line_number = original_string[:(match_start_index + original_string[match_start_index:].find(line))].count("\n") + 1 + parse_error["line_num"] = line_number + parsed_item.append(parse_error) + raise ParseError(parse_error["line_num"], f"{parse_error['got']} is a malfomred header or the start of message content without a newline") + headers_list = [] for key in headers.keys(): headers_list.append({"type": key, "content": headers[key]}) From f5e56192576922d9134f3858dfa299f282a960a9 Mon Sep 17 00:00:00 2001 From: benne238 Date: Wed, 30 Jun 2021 12:03:20 -0400 Subject: [PATCH 21/56] moved the email.policy configuration from the parser.py module to the item.py module --- src/webqueue2api/parser/item.py | 7 +++++++ src/webqueue2api/parser/parser.py | 2 -- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/webqueue2api/parser/item.py b/src/webqueue2api/parser/item.py index 4f6c585..fcef549 100644 --- a/src/webqueue2api/parser/item.py +++ b/src/webqueue2api/parser/item.py @@ -1,6 +1,7 @@ import os import time import email +from email.policy import Policy import re import datetime from dateutil.parser import parse @@ -220,9 +221,15 @@ def __parse_sections(self) -> list: # Convert list of lines to single string raw_item_as_string = "".join(self.__raw_item) + # Temporarily make the email package raise an error upon encountering a defect in the headers + Policy.raise_on_defect = True + # Parse body body_sections = parse_item(raw_item_as_string) + # Return the email oackage to its normal parsing state + Policy.raise_on_defect = False + # Add initial message headers to intial message section for index, section in enumerate(body_sections): if section["type"] == "initial_message": diff --git a/src/webqueue2api/parser/parser.py b/src/webqueue2api/parser/parser.py index 44db3db..24fe3bc 100644 --- a/src/webqueue2api/parser/parser.py +++ b/src/webqueue2api/parser/parser.py @@ -2,12 +2,10 @@ import json import string import email -from email.policy import Policy import email.errors import datetime from .utils import format_date_string from .errors import ParseError -Policy.raise_on_defect = True parsed_item = [] From 595b1be302ce4b503825dd31b485376bf9d908a0 Mon Sep 17 00:00:00 2001 From: benne238 Date: Wed, 30 Jun 2021 16:06:46 -0400 Subject: [PATCH 22/56] modified action end delimiter and modified the expected value error message when the parser encounters a malformed delimiter --- src/webqueue2api/parser/parser.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/webqueue2api/parser/parser.py b/src/webqueue2api/parser/parser.py index 24fe3bc..677d88f 100644 --- a/src/webqueue2api/parser/parser.py +++ b/src/webqueue2api/parser/parser.py @@ -16,7 +16,7 @@ # Delimiters ################################################################################ action_start_delimiter = "*** " -action_end_delimiter = " ***" +action_end_delimiter = "***" edit_start_delimiter = action_start_delimiter + "Edited by: " status_start_delimiter = action_start_delimiter + "Status updated by: " @@ -186,10 +186,10 @@ def error_handler(original_string, match_start_index, tokens): parsed_item.append(parse_error) raise ParseError(line_number, f"No reply from user end delimiter found") else: - expected_token = f"Action delimiter starting with {action_start_delimiter} or {reply_from_user_start_delimiter}" + expected_token = f"Action delimiter starting with '{action_start_delimiter}' and ending with '{action_end_delimiter}' or {reply_from_user_start_delimiter}" line_number = (original_string[:match_start_index]).count('\n') + 1 - parse_error["expected"] = f"Action start delimiter: '{action_start_delimiter}' or '{reply_from_user_start_delimiter}'" + parse_error["expected"] = expected_token parse_error["got"] = token_string parse_error["line_num"] = line_number parsed_item.append(parse_error) From 66b65d7bc1ae6011dfa120b522a8d14b12173cdd Mon Sep 17 00:00:00 2001 From: benne238 Date: Wed, 30 Jun 2021 16:41:01 -0400 Subject: [PATCH 23/56] modified the status rule to expect only a single line delimiter and made the skipTo look for delimiters at the begining of the line --- src/webqueue2api/parser/parser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/webqueue2api/parser/parser.py b/src/webqueue2api/parser/parser.py index 677d88f..664ad12 100644 --- a/src/webqueue2api/parser/parser.py +++ b/src/webqueue2api/parser/parser.py @@ -263,15 +263,16 @@ def error_handler(original_string, match_start_index, tokens): ).leaveWhitespace().setParseAction(parse_section_by_type("edit")) status_rule = ( + pp.LineStart() + pp.Literal(status_start_delimiter).suppress() + pp.Word(pp.alphanums).setResultsName("by") + pp.Literal(" at: ").suppress() + - pp.SkipTo(action_end_delimiter + pp.LineEnd()).setResultsName("datetime") + + pp.Word(pp.nums + "/-: ").setResultsName("datetime") + (pp.Literal(action_end_delimiter) + pp.LineEnd()).suppress() + pp.Group( pp.SkipTo( pp.Literal(reply_from_user_start_delimiter) - | pp.Literal(action_start_delimiter) + | (pp.LineStart() + pp.Literal(action_start_delimiter)) ) | pp.SkipTo(pp.StringEnd(), include=True) ).setResultsName("content") ).leaveWhitespace().setParseAction(parse_section_by_type("status")) From 5fbc3737d54a28a4e92b56b0536d976b6e29f9a4 Mon Sep 17 00:00:00 2001 From: benne238 Date: Wed, 30 Jun 2021 16:43:42 -0400 Subject: [PATCH 24/56] modified the edit rule to expect only a single line delimiter and made the skipTo look for delimiters at the begining of the line --- src/webqueue2api/parser/parser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/webqueue2api/parser/parser.py b/src/webqueue2api/parser/parser.py index 664ad12..3683dc7 100644 --- a/src/webqueue2api/parser/parser.py +++ b/src/webqueue2api/parser/parser.py @@ -249,15 +249,16 @@ def error_handler(original_string, match_start_index, tokens): ).leaveWhitespace().setParseAction(parse_section_by_type("reply_to_user")) edit_rule = ( + pp.LineStart() + pp.Literal(edit_start_delimiter).suppress() + pp.Word(pp.alphanums).setResultsName("by") + pp.Literal(" at: ").suppress() + - pp.SkipTo(action_end_delimiter + pp.LineEnd()).setResultsName("datetime") + + pp.Word(pp.nums + "/-: ").setResultsName("datetime") + (pp.Literal(action_end_delimiter) + pp.LineEnd()).suppress() + pp.Group( pp.SkipTo( pp.Literal(reply_from_user_start_delimiter) - | pp.Literal(action_start_delimiter) + | (pp.LineStart() + pp.Literal(action_start_delimiter)) ) | pp.SkipTo(pp.StringEnd(), include=True) ).setResultsName("content") ).leaveWhitespace().setParseAction(parse_section_by_type("edit")) From a3fafba667f9521a2e9c270cfa5480b51d6136ed Mon Sep 17 00:00:00 2001 From: benne238 Date: Wed, 30 Jun 2021 16:46:53 -0400 Subject: [PATCH 25/56] modified the reply to user rule to expect only a single line delimiter and made the skipTo look for delimiters at the begining of the line --- src/webqueue2api/parser/parser.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/webqueue2api/parser/parser.py b/src/webqueue2api/parser/parser.py index 3683dc7..baa4c2f 100644 --- a/src/webqueue2api/parser/parser.py +++ b/src/webqueue2api/parser/parser.py @@ -235,15 +235,16 @@ def error_handler(original_string, match_start_index, tokens): ).leaveWhitespace().setParseAction(parse_section_by_type("reply_from_user")) reply_to_user_rule = ( + pp.LineStart() + pp.Literal(reply_to_user_delimiter).suppress() + pp.Word(pp.alphanums).setResultsName("by")+ pp.Literal(" at: ").suppress() + - pp.SkipTo(action_end_delimiter + pp.LineEnd()).setResultsName("datetime") + + pp.Word(pp.nums + "/-: ").setResultsName("datetime") + (pp.Literal(action_end_delimiter) + pp.LineEnd()).suppress() + pp.Group( pp.SkipTo( pp.Literal(reply_from_user_start_delimiter) - | pp.Literal(action_start_delimiter) + | (pp.LineStart() + pp.Literal(action_start_delimiter)) ) | pp.SkipTo(pp.StringEnd(), include=True) ).setResultsName("content") ).leaveWhitespace().setParseAction(parse_section_by_type("reply_to_user")) From 1d0358c7cdd9d039d9d46bc4c4485d821b95c554 Mon Sep 17 00:00:00 2001 From: benne238 Date: Wed, 30 Jun 2021 16:50:46 -0400 Subject: [PATCH 26/56] added pp.LineStart() to explicity look for reply_from_user delimiter at the start of a given line in the status, edit, and reply-to-user rules --- src/webqueue2api/parser/parser.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/webqueue2api/parser/parser.py b/src/webqueue2api/parser/parser.py index baa4c2f..0252e29 100644 --- a/src/webqueue2api/parser/parser.py +++ b/src/webqueue2api/parser/parser.py @@ -243,7 +243,7 @@ def error_handler(original_string, match_start_index, tokens): (pp.Literal(action_end_delimiter) + pp.LineEnd()).suppress() + pp.Group( pp.SkipTo( - pp.Literal(reply_from_user_start_delimiter) + (pp.LineStart() + pp.Literal(reply_from_user_start_delimiter)) | (pp.LineStart() + pp.Literal(action_start_delimiter)) ) | pp.SkipTo(pp.StringEnd(), include=True) ).setResultsName("content") @@ -258,7 +258,7 @@ def error_handler(original_string, match_start_index, tokens): (pp.Literal(action_end_delimiter) + pp.LineEnd()).suppress() + pp.Group( pp.SkipTo( - pp.Literal(reply_from_user_start_delimiter) + (pp.LineStart() + pp.Literal(reply_from_user_start_delimiter)) | (pp.LineStart() + pp.Literal(action_start_delimiter)) ) | pp.SkipTo(pp.StringEnd(), include=True) ).setResultsName("content") @@ -273,7 +273,7 @@ def error_handler(original_string, match_start_index, tokens): (pp.Literal(action_end_delimiter) + pp.LineEnd()).suppress() + pp.Group( pp.SkipTo( - pp.Literal(reply_from_user_start_delimiter) + (pp.LineStart() + pp.Literal(reply_from_user_start_delimiter)) | (pp.LineStart() + pp.Literal(action_start_delimiter)) ) | pp.SkipTo(pp.StringEnd(), include=True) ).setResultsName("content") From f7114c235055346483be625d2e115686bf17a636 Mon Sep 17 00:00:00 2001 From: benne238 Date: Thu, 1 Jul 2021 10:28:19 -0400 Subject: [PATCH 27/56] Added logic to raise a parser error with the appropriate information if a Date header is not found in a reply from user section --- src/webqueue2api/parser/parser.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/webqueue2api/parser/parser.py b/src/webqueue2api/parser/parser.py index 0252e29..29af2f4 100644 --- a/src/webqueue2api/parser/parser.py +++ b/src/webqueue2api/parser/parser.py @@ -75,6 +75,17 @@ def parse_section(original_string: str, match_start_index: int, tokens: pp.Parse parsed_item.append(parse_error) raise ParseError(parse_error["line_num"], f"{parse_error['got']} is a malfomred header or the start of message content without a newline") + if "Date" not in headers.keys(): + content_start = tokens_dictionary["content"][0].strip().split("\n", 1)[0] + parse_error = { + "type": "parse_error", + "datetime": format_date_string(str(datetime.datetime.now())), + "expected": "A Date header in the reply from user section", + "got": content_start, + "line_num": original_string[:original_string.find(content_start)].count("\n") + 1 + } + raise ParseError(parse_error["line_num"], "Expected a 'Date' header in the reply from user section") + headers_list = [] for key in headers.keys(): headers_list.append({"type": key, "content": headers[key]}) From a84d8e7599b42b96bd5782854b862ee40b8d7bfe Mon Sep 17 00:00:00 2001 From: benne238 Date: Thu, 1 Jul 2021 11:13:32 -0400 Subject: [PATCH 28/56] Made expected message relevent: the datetime string doesn't need to be formatted in a particular way --- src/webqueue2api/parser/parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webqueue2api/parser/parser.py b/src/webqueue2api/parser/parser.py index 29af2f4..482b4b1 100644 --- a/src/webqueue2api/parser/parser.py +++ b/src/webqueue2api/parser/parser.py @@ -117,7 +117,7 @@ def parse_section(original_string: str, match_start_index: int, tokens: pp.Parse parsed_item.append({ "type": "parse_error", "datetime": format_date_string(str(datetime.datetime.now())), - "expected": "ISO 8601 formatted time string.", + "expected": "Expected a date and/or time", "got": headers[key], "line_num": line_number }) From 5383f8f55856e821fe4119e15932bc79f343ebd4 Mon Sep 17 00:00:00 2001 From: benne238 Date: Thu, 1 Jul 2021 11:50:29 -0400 Subject: [PATCH 29/56] added docstring to __get_assignments --- src/webqueue2api/parser/item.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/webqueue2api/parser/item.py b/src/webqueue2api/parser/item.py index fcef549..4883676 100644 --- a/src/webqueue2api/parser/item.py +++ b/src/webqueue2api/parser/item.py @@ -313,6 +313,14 @@ def __get_sorted_sections(self, sectionsList: list) -> list: return sortedSections def __get_assignments(self) -> list: + """Returns a list of dictionaries containing assignments + + Example: + + + Returns: + list: list of dictionaries which represent assignment sections + """ assignment_list = [] for index, header in enumerate(self.headers): From 23abdf30ba0eae6011829d1c5007d21cb52f0bff Mon Sep 17 00:00:00 2001 From: benne238 Date: Thu, 1 Jul 2021 11:54:55 -0400 Subject: [PATCH 30/56] added docstring to __parse_sections --- src/webqueue2api/parser/item.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/webqueue2api/parser/item.py b/src/webqueue2api/parser/item.py index 4883676..a6841c9 100644 --- a/src/webqueue2api/parser/item.py +++ b/src/webqueue2api/parser/item.py @@ -218,6 +218,14 @@ def __parse_headers(self) -> list: return headers def __parse_sections(self) -> list: + """Generates a list of dictionaries which represent all the secctions in an item + + Example: + [example] + + Returns: + list: list of dictionaries, which all containt a type key to deliminate what type of section the dictionary represents + """ # Convert list of lines to single string raw_item_as_string = "".join(self.__raw_item) From c9d04e5f6b7ee6269463206555870400682bd0e5 Mon Sep 17 00:00:00 2001 From: benne238 Date: Fri, 2 Jul 2021 13:46:32 -0400 Subject: [PATCH 31/56] Made initial message rule more similar to the edit, reply-to, and status rules --- src/webqueue2api/parser/parser.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/webqueue2api/parser/parser.py b/src/webqueue2api/parser/parser.py index 482b4b1..fb127eb 100644 --- a/src/webqueue2api/parser/parser.py +++ b/src/webqueue2api/parser/parser.py @@ -233,9 +233,8 @@ def error_handler(original_string, match_start_index, tokens): pp.SkipTo( pp.Literal(reply_from_user_start_delimiter) | pp.Literal(action_start_delimiter) - | pp.StringEnd() - ).leaveWhitespace() -).setResultsName("content").setParseAction(parse_section_by_type("initial_message")) + ) | pp.SkipTo(pp.StringEnd(), include=True) +).leaveWhitespace().setResultsName("content").setParseAction(parse_section_by_type("initial_message")) reply_from_user_rule = ( (reply_from_user_start_delimiter + pp.OneOrMore(pp.LineEnd())).suppress() + From 90ba300405209f452dec8c2125c8df93af3a1463 Mon Sep 17 00:00:00 2001 From: benne238 Date: Mon, 12 Jul 2021 09:38:05 -0400 Subject: [PATCH 32/56] import multiprocessing package in queue.py --- src/webqueue2api/parser/queue.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/webqueue2api/parser/queue.py b/src/webqueue2api/parser/queue.py index 36c22eb..ff5e0cc 100644 --- a/src/webqueue2api/parser/queue.py +++ b/src/webqueue2api/parser/queue.py @@ -1,3 +1,4 @@ +import multiprocessing import os, re from pathlib import Path from .item import Item From d8e722bb9aeefa9066db1661f01789739093116f Mon Sep 17 00:00:00 2001 From: benne238 Date: Mon, 12 Jul 2021 09:42:07 -0400 Subject: [PATCH 33/56] Added logic to parse multiple items at once in a given queue using the starmap_async function in the multiprocessing package --- src/webqueue2api/parser/queue.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/webqueue2api/parser/queue.py b/src/webqueue2api/parser/queue.py index ff5e0cc..79eff5e 100644 --- a/src/webqueue2api/parser/queue.py +++ b/src/webqueue2api/parser/queue.py @@ -74,6 +74,8 @@ def __get_items(self, headers_only: bool) -> list: list: a list of items for this Queue """ items = [] + valid_items = [] + multi_item_processes = multiprocessing.Pool(processes=multiprocessing.cpu_count()) for item in os.listdir(self.path): item_path = Path(self.path, item) @@ -81,7 +83,9 @@ def __get_items(self, headers_only: bool) -> list: is_file = True if os.path.isfile(item_path) else False if is_file and is_valid_item_name(item): - items.append(Item(self.name, item, headers_only)) + valid_items.append(item) + + items = multi_item_processes.starmap_async(Item, [(self.name, item, headers_only) for item in valid_items]).get() return items From 5a30f2dd4b072a833662f279705946810c4eb492 Mon Sep 17 00:00:00 2001 From: benne238 Date: Mon, 12 Jul 2021 09:43:21 -0400 Subject: [PATCH 34/56] Added code to wait until all of the items are loaded before returning the list of items --- src/webqueue2api/parser/queue.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/webqueue2api/parser/queue.py b/src/webqueue2api/parser/queue.py index 79eff5e..60fece4 100644 --- a/src/webqueue2api/parser/queue.py +++ b/src/webqueue2api/parser/queue.py @@ -86,6 +86,8 @@ def __get_items(self, headers_only: bool) -> list: valid_items.append(item) items = multi_item_processes.starmap_async(Item, [(self.name, item, headers_only) for item in valid_items]).get() + multi_item_processes.close() + multi_item_processes.join() return items From 55cb9ad3ba510ba26e711f844d42ed973163bb97 Mon Sep 17 00:00:00 2001 From: benne238 Date: Mon, 12 Jul 2021 09:57:36 -0400 Subject: [PATCH 35/56] Modified load_queues function in queue.py to accept *args (strings that represent the different queues) and a headers_only boolean value. Modified the docstring accordingly --- src/webqueue2api/parser/queue.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/webqueue2api/parser/queue.py b/src/webqueue2api/parser/queue.py index 60fece4..02a6d90 100644 --- a/src/webqueue2api/parser/queue.py +++ b/src/webqueue2api/parser/queue.py @@ -185,11 +185,18 @@ def get_queue_counts() -> list: return sorted_queue_info -def load_queues() -> list: - """Return a list of Queues for each queue. +def load_queues(*queues, headers_only: bool = True) -> list: + """Returns a list of queues + + Example: + [example] + + Args: + headers_only (bool, optional): Weather or not the content of items in the queue should be loaded. Defaults to True. + *queues: List of strings that represent Queue names. Returns: - list: list of Queues for each queue. + list: A list of all the queues that were given as arguments, or all of the queues if no queues were specified """ queues = [] From 81e28e5b4c2a8718db6d27476a0f1f25afb25147 Mon Sep 17 00:00:00 2001 From: benne238 Date: Mon, 12 Jul 2021 10:05:50 -0400 Subject: [PATCH 36/56] created a custom class that allows subprocesses to spawn other subprocesses --- src/webqueue2api/parser/queue.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/webqueue2api/parser/queue.py b/src/webqueue2api/parser/queue.py index 02a6d90..22b96be 100644 --- a/src/webqueue2api/parser/queue.py +++ b/src/webqueue2api/parser/queue.py @@ -1,4 +1,5 @@ import multiprocessing +import multiprocessing.pool import os, re from pathlib import Path from .item import Item @@ -198,6 +199,20 @@ def load_queues(*queues, headers_only: bool = True) -> list: Returns: list: A list of all the queues that were given as arguments, or all of the queues if no queues were specified """ + # custom class creation based on stackoverflow answer + class NoDaemonProcess(multiprocessing.Process): + # make 'daemon' attribute always return False + def _get_daemon(self): + return False + def _set_daemon(self, value): + pass + daemon = property(_get_daemon, _set_daemon) + + # We sub-class multiprocessing.pool.Pool instead of multiprocessing.Pool + # because the latter is only a wrapper function, not a proper class. + class MyPool(multiprocessing.pool.Pool): + Process = NoDaemonProcess + queues = [] for queue in get_valid_queues(): From ab565ec2ad42f2d8cc80dc001d70b9a41168cd36 Mon Sep 17 00:00:00 2001 From: benne238 Date: Mon, 12 Jul 2021 10:14:11 -0400 Subject: [PATCH 37/56] added logic to parse multiple queues with multiparsing using the the custom class from the previous comitt that allows subprocesses to spawn other subprocesses --- src/webqueue2api/parser/queue.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/webqueue2api/parser/queue.py b/src/webqueue2api/parser/queue.py index 22b96be..1ab1d6c 100644 --- a/src/webqueue2api/parser/queue.py +++ b/src/webqueue2api/parser/queue.py @@ -213,9 +213,13 @@ def _set_daemon(self, value): class MyPool(multiprocessing.pool.Pool): Process = NoDaemonProcess - queues = [] - - for queue in get_valid_queues(): - queues.append(Queue(queue)) + if len(queues) == 0: queues_to_load = get_valid_queues() + elif len(queues) == 1: return [Queue(name=queues[0], headers_only=headers_only)] + else: queues_to_load = queues + + multi_queue_process = MyPool(processes=multiprocessing.cpu_count()) + loaded_queues = multi_queue_process.starmap_async(Queue, [(queue, headers_only) for queue in queues_to_load]).get() + multi_queue_process.close() + multi_queue_process.join() - return queues \ No newline at end of file + return loaded_queues \ No newline at end of file From 60254aaada76a0247ef572d3fec40610530aea2d Mon Sep 17 00:00:00 2001 From: benne238 Date: Mon, 12 Jul 2021 12:07:46 -0400 Subject: [PATCH 38/56] modified logic to avoid directly reading lock file contents due to possible read/write permission issues --- src/webqueue2api/parser/item.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/webqueue2api/parser/item.py b/src/webqueue2api/parser/item.py index 31d9f7b..928ddc4 100644 --- a/src/webqueue2api/parser/item.py +++ b/src/webqueue2api/parser/item.py @@ -1096,11 +1096,9 @@ def __check_is_locked(self) -> Union[str, bool]: """ lock_file = str(self.path) + ".lck" if os.path.exists(lock_file): - with open(lock_file) as file: - lock_info = file.readline().split(" ") - locked_by = lock_info[4] - locked_using = lock_info[1] - return f"{self.queue} {self.number} is locked by {locked_by} using {locked_using}" + lock_file = Path(lock_file) + lock_file_owner = lock_file.owner() + return f"{self.queue} {self.number} is locked by {lock_file_owner}." else: return False From 3ec46b8b7efa723d1d4387394cf6f4a39107b72f Mon Sep 17 00:00:00 2001 From: benne238 Date: Wed, 14 Jul 2021 13:28:17 -0400 Subject: [PATCH 39/56] Added basic logic to check if the webqueue2api_config file exists in the root of the webqueue2api package directory and read the configurations from that file if so --- src/webqueue2api/__init__.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/webqueue2api/__init__.py b/src/webqueue2api/__init__.py index ba7b477..7bb54e8 100644 --- a/src/webqueue2api/__init__.py +++ b/src/webqueue2api/__init__.py @@ -1,2 +1,11 @@ from webqueue2api.parser import Item, Queue, load_queues -from .config import config \ No newline at end of file +from .config import config +import configparser, sys +from pathlib import Path as path + +config_parser_object = configparser.ConfigParser() +config_file_location = path.joinpath(path(sys.executable).parent.parent.parent, "webqueue2api_config") + +if path(config_file_location).exists(): + config_parser_object.read(config_file_location) + From 7dc383b09cab8c656fecee35dd9e90f09d48e4e6 Mon Sep 17 00:00:00 2001 From: benne238 Date: Wed, 14 Jul 2021 13:45:48 -0400 Subject: [PATCH 40/56] Logic to overwrite the queue_directory if the configuration file contains a queue_directory in the parser section --- src/webqueue2api/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/webqueue2api/__init__.py b/src/webqueue2api/__init__.py index 7bb54e8..95cbb1f 100644 --- a/src/webqueue2api/__init__.py +++ b/src/webqueue2api/__init__.py @@ -2,10 +2,15 @@ from .config import config import configparser, sys from pathlib import Path as path +import webqueue2api.parser config_parser_object = configparser.ConfigParser() config_file_location = path.joinpath(path(sys.executable).parent.parent.parent, "webqueue2api_config") if path(config_file_location).exists(): config_parser_object.read(config_file_location) + +if config_parser_object.has_section("parser"): + if config_parser_object.has_option("parser", "queue_directory"): + webqueue2api.parser.config.queue_directory = config_parser_object["parser"]["queue_directory"] From b11d2a8860a84d2e7ff910e348d5036b72c8d517 Mon Sep 17 00:00:00 2001 From: benne238 Date: Wed, 14 Jul 2021 13:48:12 -0400 Subject: [PATCH 41/56] Logic to overwrite the queues_to_ignore if it exists in the configuration file under the parser section --- src/webqueue2api/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/webqueue2api/__init__.py b/src/webqueue2api/__init__.py index 95cbb1f..fcf09eb 100644 --- a/src/webqueue2api/__init__.py +++ b/src/webqueue2api/__init__.py @@ -13,4 +13,5 @@ if config_parser_object.has_section("parser"): if config_parser_object.has_option("parser", "queue_directory"): webqueue2api.parser.config.queue_directory = config_parser_object["parser"]["queue_directory"] - + if config_parser_object.has_option("parser", "queues_to_ignore"): + webqueue2api.parser.config.queues_to_ignore = config_parser_object["parser"]["queues_to_ignore"] From e8115fc9063356a808211891eeebd41f248c8346 Mon Sep 17 00:00:00 2001 From: benne238 Date: Wed, 14 Jul 2021 13:50:43 -0400 Subject: [PATCH 42/56] Added logic to overwrite the environment variable if it exists in the config file under the api section --- src/webqueue2api/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/webqueue2api/__init__.py b/src/webqueue2api/__init__.py index fcf09eb..ed9c1ff 100644 --- a/src/webqueue2api/__init__.py +++ b/src/webqueue2api/__init__.py @@ -3,6 +3,7 @@ import configparser, sys from pathlib import Path as path import webqueue2api.parser +import webqueue2api.api config_parser_object = configparser.ConfigParser() config_file_location = path.joinpath(path(sys.executable).parent.parent.parent, "webqueue2api_config") @@ -15,3 +16,7 @@ webqueue2api.parser.config.queue_directory = config_parser_object["parser"]["queue_directory"] if config_parser_object.has_option("parser", "queues_to_ignore"): webqueue2api.parser.config.queues_to_ignore = config_parser_object["parser"]["queues_to_ignore"] + +if config_parser_object.has_section("api"): + if config_parser_object.has_option("api", "environment"): + webqueue2api.api.config.environment = config_parser_object["api"]["environment"] \ No newline at end of file From 0a30503fce8f7f36d3acb51a023d48753d0a3216 Mon Sep 17 00:00:00 2001 From: benne238 Date: Wed, 14 Jul 2021 13:55:47 -0400 Subject: [PATCH 43/56] Added logic to overwrite the jwt_secret_key if the variable exists in the config file under the api section --- src/webqueue2api/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/webqueue2api/__init__.py b/src/webqueue2api/__init__.py index ed9c1ff..8b36081 100644 --- a/src/webqueue2api/__init__.py +++ b/src/webqueue2api/__init__.py @@ -19,4 +19,6 @@ if config_parser_object.has_section("api"): if config_parser_object.has_option("api", "environment"): - webqueue2api.api.config.environment = config_parser_object["api"]["environment"] \ No newline at end of file + webqueue2api.api.config.environment = config_parser_object["api"]["environment"] + if config_parser_object.has_option("api", "jwt_secret_key"): + webqueue2api.api.config.jwt_secret_key = config_parser_object["api"]["jwt_secret_key"] \ No newline at end of file From 6c6c9908f8a2f2c91517be50f968ff8dc5ec32b4 Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Wed, 28 Jul 2021 12:52:41 -0400 Subject: [PATCH 44/56] Add docs about section parsing --- .../Parser/Section Parsing Definitions.md | 387 ++++++++++++++++++ .../Parser/awesome-pages.yaml | 4 + 2 files changed, 391 insertions(+) create mode 100644 docs-src/webqueue2api Package/Parser/Section Parsing Definitions.md create mode 100644 docs-src/webqueue2api Package/Parser/awesome-pages.yaml diff --git a/docs-src/webqueue2api Package/Parser/Section Parsing Definitions.md b/docs-src/webqueue2api Package/Parser/Section Parsing Definitions.md new file mode 100644 index 0000000..53324c4 --- /dev/null +++ b/docs-src/webqueue2api Package/Parser/Section Parsing Definitions.md @@ -0,0 +1,387 @@ +Below are the derived definitions for each section of an item as it appears in the item's plaintext file and as it is output from the parser. + +## Directory Information +Information about the user such as alias, phone number and office location. This only appears once right after the headers and right before the initial message. This only occurs if the item is submitted through the [Trouble Reporting](https://engineering.purdue.edu/ECN/AboutUs/ContactUs) page. + +### Fields +| Key | Value | +| - | - | +|`type`|`directory_information`| +| `Name` | The real name of the sender. | +| `Login` | The career account alias of the sender. | +| `Computer` | The computer the item is related to. Formatting may vary. | +| `Location` | Where the computer is located. | +| `Email` | The email address of the sender. | +| `Phone` | The phone number of the sender. | +| `Office` | The office location of the sender. | +| `UNIX Dir` | The home directory for the user on non-Windows systems | +| `Zero Dir` | The home directory for the user via Active Directory | +| `User ECNDB` | Link to the sender's username report in ECNDB | +| `Host ECNDB` | Link to the computer report in ECNDB | +| `Subject` | The subject of the email sent to the queue | + +### Delimiters +- **Start**: The second line after the first newline followed by a tab `\n\t`. +- **End**: The first non-empty line after the start that begins with whitespace then "Subject:" + +### Plain Text Example +``` +\tName: Jerry L Guerrero\n + Login: jerry\n" + Computer: x-ee27å0bpc1 (128.46.164.29)\n + Location: EE 270B\n + Email: jerry@purdue.edu\n + Phone: \n + Office: \n + UNIX Dir: /home/pier/c/jerry\n + Zero Dir: U=\\\\pier.ecn.purdue.edu\\jerry\n + User ECNDB: http://eng.purdue.edu/jump/bcafa8\n + Host ECNDB: http://eng.purdue.edu/jump/2dbd461 \n + Subject: Win7 to Win10 Migration List - kevin\n +``` + +### Parsed Example +```json +{ + "type": "directory_information", + "Name": "Jerry L Guerrero", + "Login": "jerry", + "Computer": "x-ee27å0bpc1 (128.46.164.29)", + "Location": "EE 270B", + "Email": "jerry@purdue.edu", + "Phone": "", + "Office": "", + "UNIX Dir": "/home/pier/c/jerry", + "Zero Dir": "U=\\\\pier.ecn.purdue.edu\\jerry", + "User ECNDB": "http://eng.purdue.edu/jump/bcafa8", + "Host ECNDB": "http://eng.purdue.edu/jump/2dbd461", + "Subject": "Win7 to Win10 Migration List - kevin" +} +``` + +## Initial Message +The body of the email the item originated from. This usually appears directly after the headers unless directory information is present. + +### Fields +| Key | Value | +| - | - | +| `type` | `initial_message` | +| `datetime` | RFC 8061 formatted datetime string. Pulled from the `Date` header of the item. | +| `from_name` | The sender's real name. Formatting may vary. Pulled from the `From` header of the item. This can be empty. | +| `from_email` | The sender's email address. Pulled from the `From` header of the item.| +| `to` | A list of names(s) and email(s) of people this message was sent to. Pulled from the `To` header of the item. | +| `cc` | A list of name(s) and email(s) of people who were CC'd. Pulled from the `Cc` header of the item. This can be empty. | +| `subject` | The subject of the initial message. Pulled from the `Subject` header of the item. | +| `content` | The content of the message as an list of strings. | + +### Delimiters +- **Start**: First newline after directory information if present, otherwise first newline. +- **End**: Beginning of another delimiter if present, otherwise end of file. + +### Plain Text Example +``` +I need some help with something.\n +My printer stopped working.\n +``` + +### Parsed Example +```json +{ + "type": "initial_message", + "datetime": "2020-09-11T01:26:45+00:00", + "from_name": "Justin Campbell", + "from_email": "campb303@purdue.edu", + "to": [ + { "name": "John Doe", "email": "johndoe@example.com" }, + ], + "cc": [ + { "name": "", "email": "janesmith@example.com" } + ], + "subject": Maps to item.subject, + "content": [ + "I need some help with something.\n", + "My printer stopped working.\n" + ] +} +``` + +## Edit +Information added by someone at ECN, usually for internal use and/or communication. This can occur anywhere in the item after the initial message. + +### Fields +| Key | Value | +| - | - | +| `type` | `edit` | +| `datetime` | RFC 8061 formatted datetime string. | +| `by` | The career account alias of the person who added the edit. | +| `content` | The content of the edit as a list of strings. | + +### Delimiters +- **Start**: Line starting with `*** Edited` +- **End**: Beginning of another delimiter if present, otherwise end of file. + +### Plain Text Example +``` +*** Edited by: knewell at: 04/22/20 16:39:51 *** +This is related to another item. I need to do X next. +``` + +### Parsed Example +```json +{ + "type": "edit", + "datetime": "2020-04-22T16:39:51", + "by": "knewell", + "content": [ + "This is related to another item. I need to do X next.\n" + ] +} +``` + +## Status +A short message about the progress of the item. This can occur anywhere in the item after the initial message. + +### Fields +| Key | Value | +| - | - | +| `type` | `status` | +| `datetime` | RFC 8061 formatted datetime string. | +| `by` | The career account alias of the person who updated the status. | +| `content` | The content of the status as a list of strings. | + + +### Delimiters +- **Start**: Line starting with `*** Status` +- **End**: Beginning of another delimiter if present, otherwise end of file. + +### Plain Text Example +``` +*** Status updated by: knewell at: 4/23/2020 10:35:47 *** +Doing X thing. +``` + +### Parsed Example +```json +{ + "type": "status", + "datetime": "2020-04-23T10:35:47", + "by": "knewell", + "content": [ + "Doing X thing." + ] +} +``` + +## Assignment +Assigning the item to someone. This does not occur in the body of the item. It it tracked in the headers using three different entries: + + +- `Assigned-To-Updated-By`: the career account alias of the person who updated the assignment +- `Assigned-To-Updated-Time`: the time the assignment was updated +- `Assigned-To`: the career account alias of the person the item was assigned to + +### Fields +| Key | Value | +| - | - | +| `type` | `assignment` | +| `datetime` | RFC 8061 formatted datetime string. | +| `by` | The career account alias of the person who changed the |assignment. | +| `to` | The career account alias of the person who the item was assigned to. | + +### Delimiters +N/A + +### Plain Text Example +``` +Assigned-To: campb303 +Assigned-To-Updated-Time: Tue, 23 Jun 2020 13:27:00 EDT +Assigned-To-Updated-By: harley +``` + +### Parsed Example +```json +{ + "type": "assignment", + "datetime": "2020-06-23T13:27:00", + "by": "harley", + "to": "campb303", +} +``` + +## Reply To User +A message from ECN to the user and/or related parties. This can occur anywhere in the item after the initial message. + +### Fields +| Key | Value | +| - | - | +| `type` | `reply_to_user` | +| `datetime` | RFC 8061 formatted datetime string. | +| `by` | The sender's real name. Formatting may vary. This can be empty. | +| `content` | The content of the message as an list of strings | + + +### Delimiters +- **Start**: Line starting with `*** Replied` +- **End**: Beginning of another delimiter if present, otherwise end of file. + +### Plain Text Example +``` +*** Replied by: ewhile at: 05/08/20 09:21:43 *** +Sascha, + +Chicken kevin biltong, flank jowl prosciutto shoulder meatball meatloaf sirloin. + +Ethan White +ECN +``` + +### Parsed Example +```json +{ + "type": "reply_to_user", + "datetime": "2020-05-08T09:21:43", + "by": "ewhile", + "content": [ + "Sascha,\n", + "\n", + "Chicken kevin biltong, flank jowl prosciutto shoulder meatball meatloaf sirloin.\n", + "\n", + "Ethan White\n", + "ECN" + ] +} +``` + +## Reply from User +A message from the user and/or related parties. This is only found after two or more items have been merged together. This can occur anywhere in the item after the initial message. + +### Fields +| Key | Value | +| - | - | +| `type` | `reply_from_user` | +| `datetime` | RFC 8061 formatted datetime string. | +| `from_name` | The sender's real name. Formatting may vary. This can be empty. | +| `from_email` | The sender's email address. | +| `cc` | A list of name(s) and email(s) of people who were CC'd. This can be empty. | +| `headers` | A dictionary of headers from the reply. | +| `subject` | The subject of the reply. | +| `content` | The content of the message as an list of strings | + +### Delimiters: +- **Start**: Line starting with `=== ` +- **End**: Line starting with `====` + +### Plain Text Example +``` +=== Additional information supplied by user === + +Subject: RE: New Computer Deploy +From: "Reckowsky, Michael J." +Date: Fri, 8 May 2020 13:57:17 +0000 + +Ethan, + +Biltong beef ribs doner chuck, pork chop jowl salami cow filet mignon pork. + +Mike +=============================================== +``` + +### Parsed Example +```json +{ + "type": "reply_from_user", + "datetime": "2020-05-08T13:57:18+00:00", + "from_name": "Reckowsky, Michael J.", + "from_email": "mreckowsky@purdue.edu", + "cc": [ + { "name": "John Doe", "email": "johndoe@example.com" }, + { "name": "", "email": "janesmith@example.com" } + ], + "headers" : [ + { + "type": "Subject", + "content": "RE: New Computer Deploy" + }, + { + "type": "From", + "content": "\"Reckowsky, Michael J.\" " + }, + { + "type": "Date", + "content": "Fri, 8 May 2020 13:57:17 +0000" + }, + ], + "subject": "RE: New Computer Deploy", + "content": [ + "Ethan,\n", + "\n", + "Biltong beef ribs doner chuck, pork chop jowl salami cow filet mignon pork.\n", + "\n", + "Mike\n", + ] +} +``` + +## Parse Error +!!! note + This is not found in the item but generated when the item is parsed. +An error caused by unexpected formatting. + +### Fields +| Key | Value | +| - | - | +|`type`|`parse_error`| +| `datetime` | RFC 8061 formatted datetime string. | +| `file_path` | Full path of the item with the error. | +| `expected` | Description of what the parser was expecting. | +| `got` | Line that cause the parse error. | +| `line_num` | The line number in the item that caused the parse error. | + +### Plain Text Example +``` +(item aae2 in qsnapshot) +=== Additional information supplied by user === + +Subject: RE: Help with hardware upgrades +From: "Ezra, Kristopher L" +Date: Wed, 5 Feb 2020 18:11:58 +0000 + +If it makes no difference between windows and linux for the fileserver I'd +rather service a linux machine. + +Considering the switches, i could do 2 8s and 2 16s. Two of the switches +I'm replacing already service 9 connections and I'd rather not daisy chain. +Is there something driving the price here? I see gigabit switches from +tplink on amazon right now for $50. I dont need managed switches or +anything fancy. + +Kris +*** Replied by: emuffley at: 02/05/20 13:22:02 *** + +Kris, + +Thank you on the server operating question. We will kick that off to our linux folks for discussion. + +No daisy chain, agreed. These switches are unmanaged. + +For the workstations, are you wanting Windows, Linux or a mix? + + +Eric Muffley + +Systems Engineer, Engineering Computer Network + +=============================================== +``` +### Parsed Example +```json +{ + "type": "parse_error", + "datetime": "2020-10-16T10:44:45", + "file_path": "/home/pier/e/benne238/webqueue2/q-snapshot/aae/2", + "expected": "Did not encounter a reply_from_user ending delimiter", + "got": "Kris", + "line_num": 468 +} +``` diff --git a/docs-src/webqueue2api Package/Parser/awesome-pages.yaml b/docs-src/webqueue2api Package/Parser/awesome-pages.yaml new file mode 100644 index 0000000..e502d65 --- /dev/null +++ b/docs-src/webqueue2api Package/Parser/awesome-pages.yaml @@ -0,0 +1,4 @@ +# YAML Configuration for Awesome Pages mkdocs Plugin +# See: https://github.com/lukasgeiter/mkdocs-awesome-pages-plugin +nav: + - ... \ No newline at end of file From a3387afb2860c1853fc9bbf9bcfe26edb1ce569a Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Wed, 28 Jul 2021 12:53:19 -0400 Subject: [PATCH 45/56] Add wildcard delimiter to webqueue2api Package directory for parser docs --- docs-src/webqueue2api Package/awesome-pages.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs-src/webqueue2api Package/awesome-pages.yaml b/docs-src/webqueue2api Package/awesome-pages.yaml index 7b85715..a33b701 100644 --- a/docs-src/webqueue2api Package/awesome-pages.yaml +++ b/docs-src/webqueue2api Package/awesome-pages.yaml @@ -2,4 +2,5 @@ # See: https://github.com/lukasgeiter/mkdocs-awesome-pages-plugin nav: - Getting Started.md - - Package Structure.md \ No newline at end of file + - Package Structure.md + - ... \ No newline at end of file From 84cb7b44ad7b571321025f06556c7ad0eecb1848 Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Mon, 2 Aug 2021 17:41:48 -0400 Subject: [PATCH 46/56] Remove unused datetime import --- src/webqueue2api/parser/item.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/webqueue2api/parser/item.py b/src/webqueue2api/parser/item.py index a6841c9..e3b764b 100644 --- a/src/webqueue2api/parser/item.py +++ b/src/webqueue2api/parser/item.py @@ -3,7 +3,6 @@ import email from email.policy import Policy import re -import datetime from dateutil.parser import parse from dateutil import tz from typing import Union From 80cc1cbbe468c1dfdd549c2a4a1eac546e90948e Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Mon, 2 Aug 2021 17:49:25 -0400 Subject: [PATCH 47/56] Correct __add_initial_message_headers doc block --- src/webqueue2api/parser/item.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/webqueue2api/parser/item.py b/src/webqueue2api/parser/item.py index e3b764b..bcecac8 100644 --- a/src/webqueue2api/parser/item.py +++ b/src/webqueue2api/parser/item.py @@ -251,16 +251,13 @@ def __parse_sections(self) -> list: return body_sections def __add_initial_message_headers(self, initial_message: dict) -> dict: - """Adds header information to the intial message - - Example: - [example] + """Adds header information to the intial message. Args: - initial_message (dict): the intial message dictionary without any headers + initial_message (dict): The intial message dictionary without headers. Returns: - dict: modified initial message dictionary to include headers + dict: Initial message dictionary with headers. """ raw_cc = self.__get_most_recent_header_by_type("CC") raw_to = self.__get_most_recent_header_by_type("To") From 5e0789587baf00ce5861678d1fcbd0c7bfd6f769 Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Mon, 2 Aug 2021 17:51:49 -0400 Subject: [PATCH 48/56] Cleanup __get_sorted_sections --- src/webqueue2api/parser/item.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/webqueue2api/parser/item.py b/src/webqueue2api/parser/item.py index bcecac8..1832758 100644 --- a/src/webqueue2api/parser/item.py +++ b/src/webqueue2api/parser/item.py @@ -278,16 +278,13 @@ def __add_initial_message_headers(self, initial_message: dict) -> dict: return initial_message def __get_sorted_sections(self, sectionsList: list) -> list: - """Sorts the sections chronologically by datetime - - Example: - [example] need to do + """Returns list of sections sorted chronologically. Args: - sections (list): the list of sections to be sorted + sections (list): List of sections to be sorted. Returns: - list: a list of sections sorted by datetime + list: List of sections sorted chronologically. """ sectionsLength = len(sectionsList) sortedSections = [] @@ -304,9 +301,6 @@ def __get_sorted_sections(self, sectionsList: list) -> list: if iteration == 0: oldestSection = currentSection - - #datetime.datetime.strptime(date_time_str, '%Y-%m-%d %H:%M:%S.%f') - elif parse(currentSection["datetime"]) < parse(oldestSection["datetime"]): oldestSection = currentSection From 206ed36e16ff183b3220d50154fe0178ea0f5ba1 Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Mon, 2 Aug 2021 17:55:25 -0400 Subject: [PATCH 49/56] Rename reply to user delimiter --- src/webqueue2api/parser/parser.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/webqueue2api/parser/parser.py b/src/webqueue2api/parser/parser.py index fb127eb..dee58ad 100644 --- a/src/webqueue2api/parser/parser.py +++ b/src/webqueue2api/parser/parser.py @@ -20,7 +20,7 @@ edit_start_delimiter = action_start_delimiter + "Edited by: " status_start_delimiter = action_start_delimiter + "Status updated by: " -reply_to_user_delimiter = action_start_delimiter + "Replied by: " +reply_to_user_start_delimiter = action_start_delimiter + "Replied by: " reply_from_user_start_delimiter = "=== Additional information supplied by user ===" reply_from_user_end_delimiter = "===============================================" @@ -150,7 +150,7 @@ def check_for_nested_action(original_string, match_start_index, tokens): strings_that_indicate_nesting = [ edit_start_delimiter, status_start_delimiter, - reply_to_user_delimiter, + reply_to_user_start_delimiter, reply_from_user_start_delimiter ] @@ -246,7 +246,7 @@ def error_handler(original_string, match_start_index, tokens): reply_to_user_rule = ( pp.LineStart() + - pp.Literal(reply_to_user_delimiter).suppress() + + pp.Literal(reply_to_user_start_delimiter).suppress() + pp.Word(pp.alphanums).setResultsName("by")+ pp.Literal(" at: ").suppress() + pp.Word(pp.nums + "/-: ").setResultsName("datetime") + From 49ad6e8e838afe1fb906b1d27c24ee3193777686 Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Mon, 2 Aug 2021 18:19:15 -0400 Subject: [PATCH 50/56] Docs cleanup --- src/webqueue2api/parser/queue.py | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/src/webqueue2api/parser/queue.py b/src/webqueue2api/parser/queue.py index 1ab1d6c..ceb1eb0 100644 --- a/src/webqueue2api/parser/queue.py +++ b/src/webqueue2api/parser/queue.py @@ -186,30 +186,23 @@ def get_queue_counts() -> list: return sorted_queue_info -def load_queues(*queues, headers_only: bool = True) -> list: - """Returns a list of queues - - Example: - [example] +def load_queues(*queues: tuple, headers_only: bool = True) -> list: + """Returns a list of queues. Args: headers_only (bool, optional): Weather or not the content of items in the queue should be loaded. Defaults to True. - *queues: List of strings that represent Queue names. + *queues (tuple): List of strings that represent Queue names. Returns: - list: A list of all the queues that were given as arguments, or all of the queues if no queues were specified + list: A list of all the queues that were given as arguments, or all of the queues if no queues were specified. """ - # custom class creation based on stackoverflow answer + # Custom mutliprocessing classes to allow for nested multi-threading. class NoDaemonProcess(multiprocessing.Process): - # make 'daemon' attribute always return False def _get_daemon(self): return False def _set_daemon(self, value): pass daemon = property(_get_daemon, _set_daemon) - - # We sub-class multiprocessing.pool.Pool instead of multiprocessing.Pool - # because the latter is only a wrapper function, not a proper class. class MyPool(multiprocessing.pool.Pool): Process = NoDaemonProcess From 462182df61c5f595615443353e7fcc2dc52e45b6 Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Mon, 2 Aug 2021 19:22:59 -0400 Subject: [PATCH 51/56] Simplify config overrides and change config ext --- src/webqueue2api/__init__.py | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/src/webqueue2api/__init__.py b/src/webqueue2api/__init__.py index 8b36081..2461467 100644 --- a/src/webqueue2api/__init__.py +++ b/src/webqueue2api/__init__.py @@ -1,24 +1,22 @@ from webqueue2api.parser import Item, Queue, load_queues -from .config import config import configparser, sys -from pathlib import Path as path -import webqueue2api.parser -import webqueue2api.api - -config_parser_object = configparser.ConfigParser() -config_file_location = path.joinpath(path(sys.executable).parent.parent.parent, "webqueue2api_config") +from pathlib import Path +from .config import config -if path(config_file_location).exists(): - config_parser_object.read(config_file_location) +config_parser = configparser.ConfigParser() +config_file_path = Path.joinpath(Path(sys.executable).parent.parent.parent, "webqueue2api_config.config") -if config_parser_object.has_section("parser"): - if config_parser_object.has_option("parser", "queue_directory"): - webqueue2api.parser.config.queue_directory = config_parser_object["parser"]["queue_directory"] - if config_parser_object.has_option("parser", "queues_to_ignore"): - webqueue2api.parser.config.queues_to_ignore = config_parser_object["parser"]["queues_to_ignore"] +if Path(config_file_path).exists(): + config_parser.read(config_file_path) + + if config_parser.has_section("parser"): + if config_parser.has_option("parser", "queue_directory"): + config.parser.queue_directory = config_parser["parser"]["queue_directory"] + if config_parser.has_option("parser", "queues_to_ignore"): + config.parser.queues_to_ignore = config_parser["parser"]["queues_to_ignore"] -if config_parser_object.has_section("api"): - if config_parser_object.has_option("api", "environment"): - webqueue2api.api.config.environment = config_parser_object["api"]["environment"] - if config_parser_object.has_option("api", "jwt_secret_key"): - webqueue2api.api.config.jwt_secret_key = config_parser_object["api"]["jwt_secret_key"] \ No newline at end of file + if config_parser.has_section("api"): + if config_parser.has_option("api", "environment"): + config.api.environment = config_parser["api"]["environment"] + if config_parser.has_option("api", "jwt_secret_key"): + config.api.jwt_secret_key = config_parser["api"]["jwt_secret_key"] \ No newline at end of file From 1600b7c80ec703379487662e451bbcde4cf40784 Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Mon, 2 Aug 2021 19:23:17 -0400 Subject: [PATCH 52/56] Fix config environment reference --- src/webqueue2api/api/app.py | 2 +- webqueue2api_config.config | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 webqueue2api_config.config diff --git a/src/webqueue2api/api/app.py b/src/webqueue2api/api/app.py index 99282be..477ce50 100644 --- a/src/webqueue2api/api/app.py +++ b/src/webqueue2api/api/app.py @@ -19,7 +19,7 @@ # Look for JWTs in headers (for access) then cookies (for refresh) app.config["JWT_TOKEN_LOCATION"] = ["headers", "cookies"] # Restrict cookies to HTTPS in prod, allow HTTP in dev -app.config["JWT_COOKIE_SECURE"] = False if config.jwt_secret_key == "dev" else True +app.config["JWT_COOKIE_SECURE"] = False if config.environment == "dev" else True # Restrict cookies using SameSite=strict flag app.config["JWT_COOKIE_SAMESITE"] = "strict" # Restrict refresh tokens to /token/refresh endpoint diff --git a/webqueue2api_config.config b/webqueue2api_config.config new file mode 100644 index 0000000..81d51dc --- /dev/null +++ b/webqueue2api_config.config @@ -0,0 +1,2 @@ +[api] +environment = dev \ No newline at end of file From 0c4523fac193989fd260dc998fd2c0ce5c23270a Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Mon, 2 Aug 2021 19:24:04 -0400 Subject: [PATCH 53/56] Remove testing config and rename config ext --- src/webqueue2api/__init__.py | 2 +- webqueue2api_config.config | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) delete mode 100644 webqueue2api_config.config diff --git a/src/webqueue2api/__init__.py b/src/webqueue2api/__init__.py index 2461467..96c62be 100644 --- a/src/webqueue2api/__init__.py +++ b/src/webqueue2api/__init__.py @@ -4,7 +4,7 @@ from .config import config config_parser = configparser.ConfigParser() -config_file_path = Path.joinpath(Path(sys.executable).parent.parent.parent, "webqueue2api_config.config") +config_file_path = Path.joinpath(Path(sys.executable).parent.parent.parent, "webqueue2api_config.ini") if Path(config_file_path).exists(): config_parser.read(config_file_path) diff --git a/webqueue2api_config.config b/webqueue2api_config.config deleted file mode 100644 index 81d51dc..0000000 --- a/webqueue2api_config.config +++ /dev/null @@ -1,2 +0,0 @@ -[api] -environment = dev \ No newline at end of file From 1ca725153e7cf87480196a1caa15f609b425aa9c Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Mon, 2 Aug 2021 22:28:23 -0400 Subject: [PATCH 54/56] Add basic validation for parser settings in config file --- src/webqueue2api/__init__.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/webqueue2api/__init__.py b/src/webqueue2api/__init__.py index 96c62be..eb61aa3 100644 --- a/src/webqueue2api/__init__.py +++ b/src/webqueue2api/__init__.py @@ -11,12 +11,17 @@ if config_parser.has_section("parser"): if config_parser.has_option("parser", "queue_directory"): - config.parser.queue_directory = config_parser["parser"]["queue_directory"] + queue_directory_from_config_file = Path(config_parser["parser"]["queue_directory"]) + if not queue_directory_from_config_file.exists(): + raise ValueError(f"{queue_directory_from_config_file} not found.") + config.parser.queue_directory = str(queue_directory_from_config_file.absolute()) + if config_parser.has_option("parser", "queues_to_ignore"): - config.parser.queues_to_ignore = config_parser["parser"]["queues_to_ignore"] + queues_to_ignore_from_config_file = config_parser["parser"]["queues_to_ignore"].split(", ") + config.parser.queues_to_ignore = queues_to_ignore_from_config_file if config_parser.has_section("api"): if config_parser.has_option("api", "environment"): config.api.environment = config_parser["api"]["environment"] if config_parser.has_option("api", "jwt_secret_key"): - config.api.jwt_secret_key = config_parser["api"]["jwt_secret_key"] \ No newline at end of file + config.api.jwt_secret_key = config_parser["api"]["jwt_secret_key"] \ No newline at end of file From c73052c9b0cde43549ff3a43a5274c8c988a105a Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Mon, 2 Aug 2021 22:42:40 -0400 Subject: [PATCH 55/56] Update docs --- .../webqueue2api Package/Configuration.md | 46 +++++++++++++++++++ .../webqueue2api Package/Getting Started.md | 25 ++++++++++ .../webqueue2api Package/Package Structure.md | 19 +------- .../webqueue2api Package/awesome-pages.yaml | 1 + 4 files changed, 73 insertions(+), 18 deletions(-) create mode 100644 docs-src/webqueue2api Package/Configuration.md diff --git a/docs-src/webqueue2api Package/Configuration.md b/docs-src/webqueue2api Package/Configuration.md new file mode 100644 index 0000000..e379c3d --- /dev/null +++ b/docs-src/webqueue2api Package/Configuration.md @@ -0,0 +1,46 @@ +The configuration of webqueue2api can be managed in two different ways: + +- At launch with a configuration file in the same directory as the virtual environment +- At runtime by modifying the `webqueue2api.config` symbol directly + +## Configuration File +On import, a [ConfigParser INI formatted file](https://docs.python.org/3/library/configparser.html#supported-ini-file-structure) is read for configuartion options. This file must be located in the same directory as the virtual environment folder and must be named `webqueue2api_config.ini`. + +!!! example "A Full Config File" + ```ini + [parser] + # Absolute path to directory containing queue data. + # Default: /home/pier/e/queue/Mail + queue_directory = /etc/queue-cli/queues + + # List of queues to ignore. (Comma and space ", " delimited.) + # Default: archives, drafts, inbox, coral + queues_to_ignore = dogs, cats, particularly_pugs + + [api] + # Runtime configuration for secure production or convenient development settings. + # Valid options are: prod | dev + # Default: prod + environment = dev + + # Token used to encrypt JWT payloads. + # Default: webqueue2api.api.config.generate_random_string + jwt_secret_key = uV0liG9$YgcGE7J! + ``` + +## Changing Settings at Runtime +Each package contains package level configuration objects (see: [dataclasses on PyPI](https://pypi.org/project/dataclasses/)) in `config.py` files. All of the configurations are combined into the top level `webqueue2api.config` symbol. + +These configuration objects store default values for each package. Default values can be changed by editing these objects directly or by editing the object values before their use. + +!!! example "Changing the parser's queue directory by editing object values." + ```python + import webqueue2api + + # Load Queue from default directory + ce_queue = Queue("ce") + + # load Queue from modified directory + webqueue2api.config.parser.queue_directory = "/absolute/path" + other_queue = Queue("other") + ``` \ No newline at end of file diff --git a/docs-src/webqueue2api Package/Getting Started.md b/docs-src/webqueue2api Package/Getting Started.md index 69c6a0f..12db716 100644 --- a/docs-src/webqueue2api Package/Getting Started.md +++ b/docs-src/webqueue2api Package/Getting Started.md @@ -10,6 +10,31 @@ The webqueue2api Package has the following structure: ## Basic Usage +!!! example "Load all queues." + ```python + import webqueue2api + all_queues = webqueue2api.load_queues() + for queue in all_queues: + print(f"{queue.name} has {len(queue.items)} items." + ``` + ```python + # Expected Output + "tech has 72 items" + "ce has 38 items" + ... + ``` + +!!! example "Load some queues." + ```python + import webqueue2api + all_queues = webqueue2api.load_queues("che", "tech", "ce") + print( len( all_queues ) ) + ``` + ```python + # Expected Output + 3 + ``` + !!! example "Load a queue and get the number of items in it." ```python import webqueue2api diff --git a/docs-src/webqueue2api Package/Package Structure.md b/docs-src/webqueue2api Package/Package Structure.md index cd45585..6b4c370 100644 --- a/docs-src/webqueue2api Package/Package Structure.md +++ b/docs-src/webqueue2api Package/Package Structure.md @@ -5,21 +5,4 @@ The webqueue2api Package consists of four packages in total: - `webqueue2api`: The root packages. This contains both the `api` and the `parser` packages as well as global configuration. - `api`: Contains a [Flask](https://flask.palletsprojects.com/) app that can be used with any WSGI server to host the webqueue2 API and utilities such as authentication code. - `resources`: Contains [Flask-RESTful](https://flask-restful.readthedocs.io/en/latest/) resources for the webqueue2 API. - - `parser`: Contains parers for a Queue and an Item as well as utilities and custom errors. - -## Configuration -Each package contains package level configuration objects (see: [dataclasses on PyPI](https://pypi.org/project/dataclasses/)) in `config.py` files. All of the configurations are combined into the top level `webqueue2api.config` symbol. - -These configuration objects store default values for each package. Default values can be changed by editing these objects directly or by editing the object values before their use. - -!!! example "Changing the parser's queue directory by editing object values." - ```python - import webqueue2api - - # Load Queue from default directory - ce_queue = Queue("ce") - - # load Queue from modified directory - webqueue2api.config.parser.queue_directory = "/absolute/path" - other_queue = Queue("other") - ``` \ No newline at end of file + - `parser`: Contains parers for a Queue and an Item as well as utilities and custom errors. \ No newline at end of file diff --git a/docs-src/webqueue2api Package/awesome-pages.yaml b/docs-src/webqueue2api Package/awesome-pages.yaml index a33b701..9c82ab4 100644 --- a/docs-src/webqueue2api Package/awesome-pages.yaml +++ b/docs-src/webqueue2api Package/awesome-pages.yaml @@ -2,5 +2,6 @@ # See: https://github.com/lukasgeiter/mkdocs-awesome-pages-plugin nav: - Getting Started.md + - Configuration.md - Package Structure.md - ... \ No newline at end of file From 44e97cf4ad7faf4870b4eace54d0e20cd34b4ac1 Mon Sep 17 00:00:00 2001 From: Justin Campbell Date: Mon, 2 Aug 2021 22:43:01 -0400 Subject: [PATCH 56/56] Rebuild docs --- docs/404.html | 85 +- docs/Dev Environment Setup Guide/index.html | 85 +- docs/Installation/index.html | 85 +- .../Material for mkdocs Formatting/index.html | 85 +- docs/api/Authentication/index.html | 85 +- docs/api/Getting Started/index.html | 85 +- docs/api/Items/index.html | 85 +- docs/api/Queues/index.html | 89 +- docs/api/Simulating Responses/index.html | 693 +++++++ .../assets/javascripts/bundle.2b46852b.min.js | 29 + .../javascripts/bundle.2b46852b.min.js.map | 7 + .../assets/javascripts/bundle.82b56eb2.min.js | 29 - .../javascripts/bundle.82b56eb2.min.js.map | 7 - ...477d984a.min.js => search.709b4209.min.js} | 18 +- ....min.js.map => search.709b4209.min.js.map} | 4 +- docs/assets/stylesheets/main.1118c9be.min.css | 2 + .../stylesheets/main.1118c9be.min.css.map | 1 + docs/assets/stylesheets/main.ca7ac06f.min.css | 2 - .../stylesheets/main.ca7ac06f.min.css.map | 1 - ...3b89f.min.css => palette.ba0d045b.min.css} | 4 +- .../stylesheets/palette.ba0d045b.min.css.map | 1 + .../stylesheets/palette.f1a3b89f.min.css.map | 1 - docs/index.html | 85 +- docs/search/search_index.json | 2 +- docs/sitemap.xml | 35 +- docs/sitemap.xml.gz | Bin 367 -> 415 bytes .../Configuration/index.html | 686 +++++++ .../Getting Started/index.html | 116 +- .../Package Structure/index.html | 133 +- .../Section Parsing Definitions/index.html | 1805 +++++++++++++++++ .../Parser/awesome-pages.yaml | 4 + docs/webqueue2api Package/awesome-pages.yaml | 4 +- 32 files changed, 4152 insertions(+), 201 deletions(-) create mode 100644 docs/api/Simulating Responses/index.html create mode 100644 docs/assets/javascripts/bundle.2b46852b.min.js create mode 100644 docs/assets/javascripts/bundle.2b46852b.min.js.map delete mode 100644 docs/assets/javascripts/bundle.82b56eb2.min.js delete mode 100644 docs/assets/javascripts/bundle.82b56eb2.min.js.map rename docs/assets/javascripts/workers/{search.477d984a.min.js => search.709b4209.min.js} (71%) rename docs/assets/javascripts/workers/{search.477d984a.min.js.map => search.709b4209.min.js.map} (83%) create mode 100644 docs/assets/stylesheets/main.1118c9be.min.css create mode 100644 docs/assets/stylesheets/main.1118c9be.min.css.map delete mode 100644 docs/assets/stylesheets/main.ca7ac06f.min.css delete mode 100644 docs/assets/stylesheets/main.ca7ac06f.min.css.map rename docs/assets/stylesheets/{palette.f1a3b89f.min.css => palette.ba0d045b.min.css} (67%) create mode 100644 docs/assets/stylesheets/palette.ba0d045b.min.css.map delete mode 100644 docs/assets/stylesheets/palette.f1a3b89f.min.css.map create mode 100644 docs/webqueue2api Package/Configuration/index.html create mode 100644 docs/webqueue2api Package/Parser/Section Parsing Definitions/index.html create mode 100644 docs/webqueue2api Package/Parser/awesome-pages.yaml diff --git a/docs/404.html b/docs/404.html index b6760fd..8b9621e 100644 --- a/docs/404.html +++ b/docs/404.html @@ -10,7 +10,7 @@ - + @@ -18,10 +18,10 @@ - + - + @@ -109,14 +109,18 @@