I used selenium to scrap data for specific brand but my code pass to the next page but jump to first page again and not able to stay in current page and scrape data, I tried to use IE but got same issue. I used same code instructor to scrap data from other websites and I had not same problem.
options = ChromeOptions()
options.add_argument("headless") # to hide window in 'background'
path="../Desktop/My Work/Price Monitoring/105/chromedriver.exe"
driver = Chrome(executable_path=path)
driver.get("https://www.cdw.com/search/?key=tp-link&ln=0&b=TPL&maxrecords=72")
driver.maximize_window()
time.sleep(5)
print(driver.title)
html = driver.page_source
soup = Soup(html)
page = soup.find("div", class_="search-pagination-list-container tagman search-pagination-footer")
pages = page.text.split()
pagenum = 0
pagenum = int(pages[-1])
print("pagenum = ", pagenum)
data_cdw = []
for n in range(pagenum):
pages_url = f"https://www.cdw.com/search/?key=tp-link&ln=0&b=TPL&maxrecords=72&pcurrent={n+1}"
driver.get(pages_url)
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.CSS_SELECTOR, '[class="search-result coupon-check"]')))
html = driver.page_source
soup = Soup(html)
for item in soup.select('[class="search-result coupon-check"]'):
data_cdw.append({
'title' : item.find("a", class_="search-result-product-url").text,
'name' : item.find("span", class_="mfg-code").text[6:],
'price' : item.find("div", class_="price-type-price").text[1:],
'stock' : item.find("div", class_="is-available").text[52:],
'link' : item.find("a", class_="search-result-product-url", href=True).get('href')
})
driver.close()
df_cdw = pd.DataFrame(data_cdw)
df_cdw.reset_index(drop=True)
df_cdw['link'] = "https://www.cdw.com" + df_cdw['link'].astype(str)
df_cdw.drop_duplicates()
df_cdw