Python selenium webriver misses elements without giving any error

Viewed 49

I've been using selenium ChromeDriver to parse some information. I've found elements I need by XPath, saved them into the list and then wrote a for loop to click on each of the element. It worked perfectly when I had a small amount of elements (>30), but now I'm facing a problem, that the length of my list is shorter than the actual amount of elements on the webpage (like 30 from 112, or 45 from 210). But all of these elements have the same XPath I've located initially.

# login function
def login(email,password):
    driver.get('https://crm.tender-win.ru/account/logon')
    driver.find_element_by_id('email').send_keys(email)
    driver.find_element_by_id('password').send_keys(password)
    driver.find_element_by_id('btnLogin').click()

df = []
login('somelogin','somepassword')
driver.get('https://crm.tender-win.ru/tenders/new')
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, 'MyTenderMenuTabsInfo')))
tenders = [driver.find_elements_by_xpath("//div[@class='panel card ']") 
           or driver.find_elements_by_xpath("//div[@class='panel card new']")][0]

for tender in tenders:
    tender.click()
    WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, 'tenderCurrencyCode')))
    data = driver \
        .find_element_by_xpath('//*[@id="infoScroll"]/div/div[3]/div[2]/table') \
        .get_attribute('outerHTML')
    
    df1 = pd.read_html(data)
    df.append(df1)
    print(df)

P.S.

I solved my issuse by turning for loop into function and using it until my list with elemenets is empty.

def clicker(tenders=None):
    for tender in tenders:
        tender.click()
        WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, 'tenderCurrencyCode')))
        data = driver \
            .find_element_by_xpath('//*[@id="infoScroll"]/div/div[3]/div[2]/table') \
            .get_attribute('outerHTML')
        df1 = pd.read_html(data)
        df.append(df1)
        print(df)
def scrape():
    tenders = [driver.find_elements_by_xpath("//div[@class='panel card ']")
          or driver.find_elements_by_xpath("//div[@class='panel card new']")][0]
    while len(tenders) > 0:
        clicker(tenders)
        tenders = [driver.find_elements_by_xpath("//div[@class='panel card ']") 
                   or driver.find_elements_by_xpath("//div[@class='panel card new']")][0]
0 Answers
Related