Skip to content

fix: update Rugby Borough Council scraper #2148

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
CJskii wants to merge 2 commits into
base: master
Choose a base branch
from
+25 −126
Open

fix: update Rugby Borough Council scraper #2148

Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
36dc60c
fix: update Rugby Borough Council scraper
CJskii Jul 9, 2026
cba3349
fix: handle Rugby Council API errors
CJskii Jul 9, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
149 changes: 23 additions & 126 deletions uk_bin_collection/uk_bin_collection/councils/RugbyBoroughCouncil.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,9 @@
import time

from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import Select, WebDriverWait
import requests

from uk_bin_collection.uk_bin_collection.common import *
from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass


# import the wonderful Beautiful Soup and the URL grabber
class CouncilClass(AbstractGetBinDataClass):
"""
Concrete classes have to implement all abstract operations of the
Expand All @@ -19,129 +12,33 @@ class CouncilClass(AbstractGetBinDataClass):
"""

def parse_data(self, page: str, **kwargs) -> dict:
driver = None
try:
user_postcode = kwargs.get("postcode")
if not user_postcode:
raise ValueError("No postcode provided.")
check_postcode(user_postcode)

user_uprn = kwargs.get("uprn")
check_uprn(user_uprn)

headless = kwargs.get("headless")
web_driver = kwargs.get("web_driver")
user_agent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"
driver = create_webdriver(web_driver, headless, user_agent, __name__)
page = "https://www.rugby.gov.uk/check-your-next-bin-day"

driver.get(page)

wait = WebDriverWait(driver, 60)
accept_cookies_button = wait.until(
EC.element_to_be_clickable(
(
By.ID,
"ccc-recommended-settings",
)
)
)
accept_cookies_button.click()

postcode_input = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(
By.XPATH,
'//*[@id="_com_placecube_digitalplace_local_waste_portlet_CollectionDayFinderPortlet_postcode"]',
)
)
)
postcode_input.send_keys(user_postcode + Keys.TAB + Keys.ENTER)

time.sleep(5)
# Wait for address box to be visible
select_address_input = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(
By.ID,
"_com_placecube_digitalplace_local_waste_portlet_CollectionDayFinderPortlet_uprn",
)
)
)

# Select address based
select = Select(select_address_input)
for option in select.options:
if option.get_attribute("value") == str(user_uprn):
select.select_by_value(str(user_uprn))
break
else:
raise ValueError(f"UPRN {user_uprn} not found in address options")

select_address_input.send_keys(Keys.TAB + Keys.ENTER)

# Wait for the specified table to be present
target_table = WebDriverWait(driver, 10).until(
EC.presence_of_element_located(
(
By.XPATH,
'//*[@id="portlet_com_placecube_digitalplace_local_waste_portlet_CollectionDayFinderPortlet"]/div/div/div/div/table',
)
)
)

soup = BeautifulSoup(driver.page_source, "html.parser")
user_uprn = kwargs.get("uprn")
check_uprn(user_uprn)

# Initialize bin data dictionary
bin_data = {"bins": []}
api_url = (
"https://apps.cloud9technologies.com/rugby/citizenmobile/webapi/"
f"wastecollections/{user_uprn}"
)
json_data = requests.get(api_url).json()
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Outdated

# Find the table
table = soup.find("table", class_="table")
if table:
# Find all rows in tbody
rows = table.find("tbody").find_all("tr")
waste_collection_dates = json_data["wasteCollectionDates"]

for row in rows:
# Get all cells in the row
cells = row.find_all("td")
if len(cells) >= 4: # Ensure we have enough cells
bin_type = cells[0].text.strip()
next_collection = cells[1].text.strip()
following_collection = cells[3].text.strip()
data = {"bins": []}

# Parse the dates
for collection_date in [next_collection, following_collection]:
try:
# Convert date from "Friday 09 May 2025" format
parsed_date = datetime.strptime(
collection_date, "%A %d %b %Y"
)
formatted_date = parsed_date.strftime("%d/%m/%Y")
for key, collection in waste_collection_dates.items():
if not key.startswith("container") or not collection:
continue

bin_info = {
"type": bin_type,
"collectionDate": formatted_date,
}
bin_data["bins"].append(bin_info)
except ValueError as e:
print(f"Error parsing date {collection_date}: {e}")
else:
raise ValueError("Collection data table not found")
collection_date = datetime.fromisoformat(collection["collectionDate"])

# Sort the collections by date
bin_data["bins"].sort(
key=lambda x: datetime.strptime(x["collectionDate"], "%d/%m/%Y")
)
dict_data = {
"type": collection["containerDescription"],
"collectionDate": collection_date.strftime(date_format),
}
data["bins"].append(dict_data)

#print(bin_data)
data["bins"].sort(
key=lambda x: datetime.strptime(x.get("collectionDate"), date_format)
)

except Exception as e:
# Here you can log the exception if needed
print(f"An error occurred: {e}")
# Optionally, re-raise the exception if you want it to propagate
raise
finally:
# This block ensures that the driver is closed regardless of an exception
if driver:
driver.quit()
return bin_data
return data