from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
import pandas as pd
import time as t
chrome_options = Options()
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument('disable-notifications')
chrome_options.add_argument("window-size=1280,720")
webdriver_service = Service("chromedriver.exe") ## path to where you saved chromedriver binary
browser = webdriver.Chrome(service=webdriver_service, options=chrome_options)
wait = WebDriverWait(browser, 20)
url = 'https://iremedy.com/search?query=Vital%20Signs%20Monitors'
browser.get(url)
items_list = []
while True:
elements_on_page = wait.until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, '[class^="card"]')))
print(len(elements_on_page), 'total items found')
if len(elements_on_page) > 100:
print('more than 100 found, stopping')
break
footer = wait.until (EC.presence_of_element_located((By.CSS_SELECTOR, 'footer[id="footer"]')))
footer.location_once_scrolled_into_view
t.sleep(2)
for el in elements_on_page:
title = el.find_element(By.CSS_SELECTOR, 'h3[class="title"]')
price = el.find_element(By.CSS_SELECTOR, 'div[class="price"]')
items_list.append((title.text.strip(), price.text.strip()))
df = pd.DataFrame(items_list, columns = ['Item', 'Price'])
print(df)
This website has an infinte type of scroll, the above code only scrolls the page and copies the information. I want to visit each product card link and copy the images links, product name, categories, short description, price, availibility, SKU and Additional Information as well. Firstly, several products links pops up a box with additional similar products, which I am not aware of how to remove it. I want to copy the required information and then go back and then resume from next. Is that something possible?
I want to make all this process as fast as possible. Another thought crossed my mind, is to copy all the product links present in infinite scrolling page till last product. After that, executing multiple threads (if a good solution for dividing it on multiple threads) to scrap from those links. I want to know what is the fastest way and how to do it. Thanks