From ab565ec2ad42f2d8cc80dc001d70b9a41168cd36 Mon Sep 17 00:00:00 2001 From: benne238 Date: Mon, 12 Jul 2021 10:14:11 -0400 Subject: [PATCH] added logic to parse multiple queues with multiparsing using the the custom class from the previous comitt that allows subprocesses to spawn other subprocesses --- src/webqueue2api/parser/queue.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/webqueue2api/parser/queue.py b/src/webqueue2api/parser/queue.py index 22b96be..1ab1d6c 100644 --- a/src/webqueue2api/parser/queue.py +++ b/src/webqueue2api/parser/queue.py @@ -213,9 +213,13 @@ def _set_daemon(self, value): class MyPool(multiprocessing.pool.Pool): Process = NoDaemonProcess - queues = [] - - for queue in get_valid_queues(): - queues.append(Queue(queue)) + if len(queues) == 0: queues_to_load = get_valid_queues() + elif len(queues) == 1: return [Queue(name=queues[0], headers_only=headers_only)] + else: queues_to_load = queues + + multi_queue_process = MyPool(processes=multiprocessing.cpu_count()) + loaded_queues = multi_queue_process.starmap_async(Queue, [(queue, headers_only) for queue in queues_to_load]).get() + multi_queue_process.close() + multi_queue_process.join() - return queues \ No newline at end of file + return loaded_queues \ No newline at end of file