diff --git a/py34/scraper.py b/py34/scraper.py index d7c92b5..776bd05 100644 --- a/py34/scraper.py +++ b/py34/scraper.py @@ -15,24 +15,26 @@ class Scraper: def __init__(self): self._scraper: CloudScraper = CloudScraper() - def _get(self, url: str) -> bytes: + def _get(self, url: str, body: bool) -> bytes | Response: res: Response = self._scraper.get(url) + if not body: + return res if res.status_code != 200: raise ScraperException(res, "Request did not succeed") return res.content @retry(Exception, tries=5, delay=3) - def _retry_get(self, url: str) -> bytes: - return self._get(url) + def _retry_get(self, url: str, body: bool) -> bytes | Response: + return self._get(url, body=body) - def get(self, url: str, retry: bool = True): + def get(self, url: str, retry: bool = True, body: bool = True): if retry: - return self._retry_get(url) + return self._retry_get(url, body=body) else: - return self._get(url) + return self._get(url, body=body) def get_html(self, url: str, retry: bool = True) -> BeautifulSoup: - return bs4(self.get(url=url, retry=retry)) + return bs4(self.get(url=url, retry=retry, body=True)) # Default scraper