From 1653e8db7c58680e35e380877e692c2c38de0b3c Mon Sep 17 00:00:00 2001 From: Tomuxs Date: Sat, 2 Aug 2025 15:41:50 +0200 Subject: [PATCH] Allow requesting without downloading --- py34/scraper.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/py34/scraper.py b/py34/scraper.py index d7c92b5..776bd05 100644 --- a/py34/scraper.py +++ b/py34/scraper.py @@ -15,24 +15,26 @@ class Scraper: def __init__(self): self._scraper: CloudScraper = CloudScraper() - def _get(self, url: str) -> bytes: + def _get(self, url: str, body: bool) -> bytes | Response: res: Response = self._scraper.get(url) + if not body: + return res if res.status_code != 200: raise ScraperException(res, "Request did not succeed") return res.content @retry(Exception, tries=5, delay=3) - def _retry_get(self, url: str) -> bytes: - return self._get(url) + def _retry_get(self, url: str, body: bool) -> bytes | Response: + return self._get(url, body=body) - def get(self, url: str, retry: bool = True): + def get(self, url: str, retry: bool = True, body: bool = True): if retry: - return self._retry_get(url) + return self._retry_get(url, body=body) else: - return self._get(url) + return self._get(url, body=body) def get_html(self, url: str, retry: bool = True) -> BeautifulSoup: - return bs4(self.get(url=url, retry=retry)) + return bs4(self.get(url=url, retry=retry, body=True)) # Default scraper