Hey!
There is a 'NoneType' object has no attribute 'text' error because it isn't capturing any text.
I used Selenium because it works better for some reason, or maybe I'm not pointing to the right html tags. Anyways, the df has columns for name, region, address, and link.
I understand not using Selenium, it's significantly slower than BeautifulSoup.
import time
import pandas as pd
from selenium.webdriver.common.by import By
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
option = webdriver.ChromeOptions()
driver = webdriver.Chrome(options = option)
# URL of the club finder page
url = 'https://www.irishrugby.ie/club-finder/'
# Open the webpage
driver.get(url)
# Allow some time for the page to load
time.sleep(5)
# Initialize empty lists to store club data
clubs = []
# Find all club containers
club_containers = driver.find_elements(By.CSS_SELECTOR, 'div.d-inline-block.col-xs-12.py-0.px-1.pb-1.mt-1.bg-white.map-list__club')
for container in club_containers:
# Extract text or set to 'N/A' if not found
try:
club_name = container.find_element(By.CSS_SELECTOR, 'h3').text.strip()
except:
club_name = 'N/A'
try:
club_address = container.find_element(By.CSS_SELECTOR, 'div.map-list__address').text.strip()
except:
club_address = 'N/A'
try:
club_region = container.find_element(By.CSS_SELECTOR, 'p.m-0').text.strip()
except:
club_region = 'N/A'
try:
club_link = container.find_element(By.CSS_SELECTOR, 'a').get_attribute('href')
except:
club_link = 'N/A'
# Append club information to the list
clubs.append({
'Name': club_name,
'Address': club_address,
'Region': club_region,
'Link': club_link
})
# Close the WebDriver
driver.quit()
# Create a DataFrame from the list
df = pd.DataFrame(clubs)