Skip to content

Commit

Permalink
added logic to parse multiple queues with multiparsing using the the …
Browse files Browse the repository at this point in the history
…custom class from the previous comitt that allows subprocesses to spawn other subprocesses
  • Loading branch information
benne238 committed Jul 12, 2021
1 parent 81e28e5 commit ab565ec
Showing 1 changed file with 9 additions and 5 deletions.
14 changes: 9 additions & 5 deletions src/webqueue2api/parser/queue.py
Original file line number Diff line number Diff line change
Expand Up @@ -213,9 +213,13 @@ def _set_daemon(self, value):
class MyPool(multiprocessing.pool.Pool):
Process = NoDaemonProcess

queues = []

for queue in get_valid_queues():
queues.append(Queue(queue))
if len(queues) == 0: queues_to_load = get_valid_queues()
elif len(queues) == 1: return [Queue(name=queues[0], headers_only=headers_only)]
else: queues_to_load = queues

multi_queue_process = MyPool(processes=multiprocessing.cpu_count())
loaded_queues = multi_queue_process.starmap_async(Queue, [(queue, headers_only) for queue in queues_to_load]).get()
multi_queue_process.close()
multi_queue_process.join()

return queues
return loaded_queues

0 comments on commit ab565ec

Please sign in to comment.