Skip to content

Commit

Permalink
Added logic to parse multiple items at once in a given queue using th…
Browse files Browse the repository at this point in the history
…e starmap_async function in the multiprocessing package
  • Loading branch information
benne238 committed Jul 12, 2021
1 parent 90ba300 commit d8e722b
Showing 1 changed file with 5 additions and 1 deletion.
6 changes: 5 additions & 1 deletion src/webqueue2api/parser/queue.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,14 +74,18 @@ def __get_items(self, headers_only: bool) -> list:
list: a list of items for this Queue
"""
items = []
valid_items = []
multi_item_processes = multiprocessing.Pool(processes=multiprocessing.cpu_count())

for item in os.listdir(self.path):
item_path = Path(self.path, item)

is_file = True if os.path.isfile(item_path) else False

if is_file and is_valid_item_name(item):
items.append(Item(self.name, item, headers_only))
valid_items.append(item)

items = multi_item_processes.starmap_async(Item, [(self.name, item, headers_only) for item in valid_items]).get()

return items

Expand Down

0 comments on commit d8e722b

Please sign in to comment.