Added threadpool

py34/list.py

@@ -2,8 +2,8 @@ from .post import Post
 from .scraper import scraper, ScraperException
 from .url import parse_thumbnail_url
 from .dockid import is_nochick, is_toodeep
+from .thread import Pool
 import urllib.parse
-from threading import Thread
 
 
 class ListException(Exception):
@@ -55,13 +55,10 @@ class List:
 
             # Download thumbnails
             if fetch_thumbnails:
-                threads = [Thread(target=Post.get_thumbnail, args=(post,)) for post in self.posts]
-
-                for thread in threads:
-                    thread.start()
-
-                for thread in threads:
-                    thread.join()
+                pool = Pool()
+                for post in self.posts:
+                    pool.submit(Post.get_thumbnail_data, post)
+                pool.join()
 
         except ScraperException as ex:
             raise ex

py34/thread.py

@@ -0,0 +1,37 @@
+from threading import Thread
+from typing import Callable
+
+
+class Pool:
+    def __init__(self, max_workers: int = 5):
+        self.max_workers = max_workers
+        self.jobs: list[Thread] = []
+        self.workers: list[Thread] = []
+
+    def submit(self, func: Callable, *vargs, **kwargs):
+        def proc(self, func: Callable, *vargs, **kwargs):
+            func(*vargs, **kwargs)
+            self._pool_proc()
+
+        self.jobs.append(Thread(
+            target = proc,
+            args = (self, func, *vargs, ),
+            kwargs = kwargs
+        ))
+
+        self._pool_proc()
+
+    def join(self):
+        while len(self.workers) != 0:
+            self.workers[-1].join()
+            self._pool_proc()
+
+    def _pool_proc(self):
+        # Remove any dead workers
+        self.workers = list(filter(Thread.is_alive, self.workers))
+
+        # Process jobs if any
+        while len(self.workers) < self.max_workers and len(self.jobs) != 0:
+            job = self.jobs.pop()
+            job.start()
+            self.workers.append(job)