diff --git a/uk_bin_collection/uk_bin_collection/councils/MidSuffolkDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/MidSuffolkDistrictCouncil.py index 1797f9c4dd..adaa65fa0e 100644 --- a/uk_bin_collection/uk_bin_collection/councils/MidSuffolkDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/MidSuffolkDistrictCouncil.py @@ -1,5 +1,5 @@ import datetime -import time +import re from datetime import datetime from bs4 import BeautifulSoup @@ -11,6 +11,16 @@ from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass +# Matches the council's date format e.g. "Wed 27 May 2026". Used to extract +# every date in a

tag — a single tag can contain multiple dates separated +# by commas when the council renders "Following Collections:" with two +# upcoming dates, and may be prefixed "Today - ". +_DATE_RE = re.compile( + r"\b(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun) \d{1,2} " + r"(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{4}\b" +) + + # import the wonderful Beautiful Soup and the URL grabber class CouncilClass(AbstractGetBinDataClass): """ @@ -41,15 +51,8 @@ def parse_data(self, page: str, **kwargs) -> dict: driver = create_webdriver(web_driver, headless, None, __name__) driver.get(url) - wait = WebDriverWait(driver, 30) - wait.until( - EC.presence_of_element_located( - (By.CSS_SELECTOR, '[aria-label="Postcode"]') - ) - ) - # Enter postcode - postcode_input = WebDriverWait(driver, 10).until( + postcode_input = WebDriverWait(driver, 30).until( EC.presence_of_element_located( (By.CSS_SELECTOR, '[aria-label="Postcode"]') ) @@ -63,43 +66,82 @@ def parse_data(self, page: str, **kwargs) -> dict: driver.execute_script("arguments[0].scrollIntoView();", find_address_button) driver.execute_script("arguments[0].click();", find_address_button) - time.sleep(5) - # Wait for address dropdown - select_address_input = WebDriverWait(driver, 10).until( - EC.presence_of_element_located((By.CSS_SELECTOR, "select")) - ) - - # Select address based on postcode and house number + # Wait for address dropdown to appear AND populate. Polling for a + # populated