How to click multiple spans on a page to scrape more data using Selenium in Python?

user_9795465 0
Viewed 62

I want to scrape broadband deals for a postcode "NE8 1SR" on https://www.virginmedia.com/broadband/broadband-only

namely the plan name, the price, the download and the upload speed (visible after clicking "Plan details" or "What's included").

I am trying to click on each of the "Plan details" span, but get the errors like "MoveTargetOutOfBoundsException".

from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC

driver = webdriver.Chrome(ChromeDriverManager().install())
...
driver.set_window_size(1024, 600)
driver.maximize_window()

find_length = len(driver.find_elements_by_xpath('''//span[contains(text(), "Plan details") or contains(text(), "What's included")]'''))
print(find_length)

list_count = 0


while (list_count < find_length):
    print("Tariff count: {}".format(list_count))

    find = driver.find_elements_by_xpath('''//span[contains(text(), "Plan details") or contains(text(), "What's included")]''')
    WebDriverWait(driver,15).until(EC.presence_of_all_elements_located((By.XPATH, '''//span[contains(text(), "Plan details") or contains(text(), "What's included")]''')))

    driver.execute_script("arguments[0].scrollIntoView();", WebDriverWait(driver, 20).until(EC.visibility_of_element_located(tuple(find[list_count]))))
    ActionChains(driver).move_to_element(WebDriverWait(driver, 20).until(EC.element_to_be_clickable(tuple(find[list_count])))).click().perform()
    list_count += 1
    driver.sleep(3)

How do I click each span and scrape the data inside?window example

selenium-webdriver web-scraping selenium python
1 Answers

To click on each of the "Plan details" span you can use the following solution:

driver.get('https://www.virginmedia.com/broadband/broadband-only')
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.CSS_SELECTOR, "button#onetrust-accept-btn-handler"))).click()
driver.execute_script("return arguments[0].scrollIntoView(true);", WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.XPATH, "//p/p[starts-with(., 'Whether you watch TV in 4K')]"))))
elements = driver.find_elements(By.XPATH, "//span[contains(text(), 'Plan details') or contains(text(), 's included')]")
for element in elements:
    Actions(driver).move_to_element(element).click().perform()

Note : You have to add the following imports :

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
user_74294470
Related