I am running the script below to review the latest filings by public companies listed on the London Stock Exchange. I have encountered a weird bug. It works 3/4 of the times, however, It randomly fails to locate the elements that I am looking for. I am using WebDriverWait in each attempt to wait until element is loaded, locate any element. So I am not sure what else I can do to ensure that these elements are found. Any idea, How can I ensure 100% success rate in locating these elements?
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.service import Service
chrome_options = Options()
chrome_options.add_argument("--window-size=1920,1080")
chrome_options.add_argument("--headless")
s = Service(ChromeDriverManager().install())
driver = webdriver.Chrome(service=s, options=chrome_options)
# click the cookies disclaimer
URL = f"https://www.londonstockexchange.com/news?tab=news-explorer&period=lastmonth&page=1"
driver.get(URL)
WebDriverWait(driver, 5).until(
EC.visibility_of_element_located((By.XPATH, "//button[@id='ccc-notify-accept']"))
).click()
while True:
try:
URL = f"https://www.londonstockexchange.com/news?tab=news-explorer&period=lastmonth&page=1"
driver.get(URL)
# find the form dropdown parent and click it
dropdown_path = '//*[@id="news-table-results"]/div[1]/form[1]'
WebDriverWait(driver, 5).until(
EC.visibility_of_element_located((By.XPATH, dropdown_path))
).click()
# search the dropdown for the largest viewing option (500 items) and select it
WebDriverWait(driver, 5).until(
EC.visibility_of_element_located(
(By.XPATH, "//*[contains(text(), 'Show 500 news')]")
)
).find_element(By.XPATH, "..").click()
# wait for the filing table to appear
WebDriverWait(driver, 5).until(EC.visibility_of_element_located((By.TAG_NAME, "table")))
print("Success")
except:
print("Failure")
raise
The elements that are periodically not found are usually:
//*[@id="news-table-results"]/div[1]/form[1]
//*[contains(text(), 'Show 500 news')]