diff --git a/substack_api/category.py b/substack_api/category.py index 8f8f48c..afc400e 100644 --- a/substack_api/category.py +++ b/substack_api/category.py @@ -1,3 +1,4 @@ +from time import sleep from typing import Any, Dict, List, Optional, Tuple import requests @@ -127,6 +128,7 @@ def _fetch_newsletters_data( full_url = endpoint + str(page_num) r = requests.get(full_url, headers=HEADERS, timeout=30) r.raise_for_status() + sleep(2) # Be polite to the server resp = r.json() newsletters = resp["publications"] diff --git a/substack_api/newsletter.py b/substack_api/newsletter.py index 2ed4cbd..1377a03 100644 --- a/substack_api/newsletter.py +++ b/substack_api/newsletter.py @@ -52,9 +52,12 @@ def _make_request(self, endpoint: str, **kwargs) -> requests.Response: The response object from the request """ if self.auth and self.auth.authenticated: - return self.auth.get(endpoint, **kwargs) + resp = self.auth.get(endpoint, **kwargs) else: - return requests.get(endpoint, headers=HEADERS, **kwargs) + resp = requests.get(endpoint, headers=HEADERS, **kwargs) + + sleep(2) # Be polite to the server + return resp def _fetch_paginated_posts( self, params: Dict[str, str], limit: Optional[int] = None, page_size: int = 15 @@ -112,9 +115,6 @@ def _fetch_paginated_posts( if len(items) < batch_size: more_items = False - # Be nice to the API - sleep(0.5) - # Instead of creating Post objects directly, return the URLs # The caller will create Post objects as needed return results