From 081efaeb9f26dc7b3c806baee5eee7567cfd6be5 Mon Sep 17 00:00:00 2001 From: Tomuxs Date: Fri, 15 Aug 2025 19:27:21 +0200 Subject: [PATCH] Added more http methods to scraper --- py34/scraper.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/py34/scraper.py b/py34/scraper.py index 52a75a8..fc8bf2a 100644 --- a/py34/scraper.py +++ b/py34/scraper.py @@ -29,9 +29,9 @@ class Scraper: def __del__(self): self.close() - def _get(self, url: str, body: bool) -> bytes | Response: + def _request(self, method: str, url: str, body: bool) -> bytes | Response: while True: - res: Response = self._scraper.get(url) + res: Response = self._scraper.request(method, url) res.close() if res.status_code == 429: self.reset() @@ -44,8 +44,8 @@ class Scraper: return res.content @retry(Exception, tries=5, delay=5) - def _retry_get(self, url: str, body: bool) -> bytes | Response: - return self._get(url, body=body) + def _retry_request(self, method: str, url: str, body: bool) -> bytes | Response: + return self._get(method, url, body=body) def close(self): self._scraper.close() @@ -54,11 +54,17 @@ class Scraper: self._scraper.close() self._scraper = CloudScraper() - def get(self, url: str, retry: bool = True, body: bool = True): + def request(self, method: str, url: str, retry: bool = True, body: bool = True): if retry: - return self._retry_get(url, body=body) + return self._retry_get(method, url, body=body) else: - return self._get(url, body=body) + return self._get(method, url, body=body) + + def head(self, url: str, retry: bool = True, body: bool = True): + return request("HEAD", url, retry=retry, body=body) + + def get(self, url: str, retry: bool = True, body: bool = True): + return request("GET", url, retry=retry, body=body) def get_html(self, url: str, retry: bool = True) -> BeautifulSoup: return bs4(self.get(url=url, retry=retry, body=True))