From d8e722bb9aeefa9066db1661f01789739093116f Mon Sep 17 00:00:00 2001 From: benne238 Date: Mon, 12 Jul 2021 09:42:07 -0400 Subject: [PATCH] Added logic to parse multiple items at once in a given queue using the starmap_async function in the multiprocessing package --- src/webqueue2api/parser/queue.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/webqueue2api/parser/queue.py b/src/webqueue2api/parser/queue.py index ff5e0cc..79eff5e 100644 --- a/src/webqueue2api/parser/queue.py +++ b/src/webqueue2api/parser/queue.py @@ -74,6 +74,8 @@ def __get_items(self, headers_only: bool) -> list: list: a list of items for this Queue """ items = [] + valid_items = [] + multi_item_processes = multiprocessing.Pool(processes=multiprocessing.cpu_count()) for item in os.listdir(self.path): item_path = Path(self.path, item) @@ -81,7 +83,9 @@ def __get_items(self, headers_only: bool) -> list: is_file = True if os.path.isfile(item_path) else False if is_file and is_valid_item_name(item): - items.append(Item(self.name, item, headers_only)) + valid_items.append(item) + + items = multi_item_processes.starmap_async(Item, [(self.name, item, headers_only) for item in valid_items]).get() return items