My code can not stay on page loaded by selenium Python

Viewed 34

I used selenium to scrap data for specific brand but my code pass to the next page but jump to first page again and not able to stay in current page and scrape data, I tried to use IE but got same issue. I used same code instructor to scrap data from other websites and I had not same problem.

 options = ChromeOptions()
 options.add_argument("headless") # to hide window in 'background'
 path="../Desktop/My Work/Price Monitoring/105/chromedriver.exe"
 driver = Chrome(executable_path=path)
 driver.get("https://www.cdw.com/search/?key=tp-link&ln=0&b=TPL&maxrecords=72")
 driver.maximize_window()
 time.sleep(5)
 print(driver.title)
 html = driver.page_source
 soup = Soup(html)

 page = soup.find("div", class_="search-pagination-list-container tagman search-pagination-footer")
 pages = page.text.split()
 pagenum = 0
 pagenum = int(pages[-1])
 print("pagenum = ", pagenum)

data_cdw = []

for n in range(pagenum):
     pages_url = f"https://www.cdw.com/search/?key=tp-link&ln=0&b=TPL&maxrecords=72&pcurrent={n+1}"
     driver.get(pages_url)
     WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, '[class="search-result coupon-check"]')))

    html = driver.page_source
    soup = Soup(html)

    for item in soup.select('[class="search-result coupon-check"]'):
        data_cdw.append({
            'title' : item.find("a", class_="search-result-product-url").text,
            'name' : item.find("span", class_="mfg-code").text[6:],
            'price' : item.find("div", class_="price-type-price").text[1:],
            'stock' : item.find("div", class_="is-available").text[52:],
            'link' : item.find("a", class_="search-result-product-url", href=True).get('href')
        })

driver.close()

df_cdw = pd.DataFrame(data_cdw)
df_cdw.reset_index(drop=True)
df_cdw['link'] = "https://www.cdw.com" + df_cdw['link'].astype(str)
df_cdw.drop_duplicates()
df_cdw
0 Answers
Related