from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.chrome.service import Service as ChromeService
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
import configparser
from datetime import datetime, timedelta, date
import time
import json
parser = configparser.RawConfigParser()
parser.read('config.ini')
url= parser['PROPERTIES']['URL']
OMIT_KEYWORDS= parser['FILTERS']['OMIT'].split(',')
INCLUDE_KEYWORDS=parser['FILTERS']['INCLUDE'].split(',')
END_DATE = datetime.strptime(parser['DATE']['END'], '%Y-%m-%d')
START_DATE=datetime.strptime(parser['DATE']['START'],'%Y-%m-%d')
minimum_comment_length = int(parser['PROPERTIES']['MIN_COMMENT_LENGTH'])
maximum_comment_length = int(parser['PROPERTIES']['MAX_COMMENT_LENGTH'])
# Setting up driver options
options = webdriver.ChromeOptions()
# Setting up Path to chromedriver executable file
CHROMEDRIVER_PATH =r'C:\Users\HP\Desktop\INTERNSHIP\Target\chromedriver.exe'
# Adding options
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option("useAutomationExtension", False)
# Setting up chrome service
service = ChromeService(executable_path=CHROMEDRIVER_PATH)
# Establishing Chrom web driver using set services and options
driver = webdriver.Chrome(service=service, options=options)
wait = WebDriverWait(driver, 20)
driver.get(url)
driver.implicitly_wait(10)
time.sleep(2)
driver.execute_script("window.scrollTo(0,document.body.scrollHeight)")
l= driver.find_element_by_xpath(".//a[contains(@class,'link__underlined view-more-trigger height block-content')]").click()
item_list=[]
while True:
try:
reviews=wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR,".bv-content-core")))
time.sleep(2)
for review in reviews:
item={
'stars': review.find_element_by_xpath(".//span[contains(@class,'bv-rating-stars-container')]//span[contains(@class,'bv-off-screen')]").text.replace("out of 5 stars.",""),
'username': review.find_element_by_xpath(".//span[contains(@class,'bv-author')]//span[contains(@itemprop,'name')]").text,
'userurl':"NA",
'title':review.find_element_by_xpath(".//div[contains(@class,'bv-content-title-container')]//h3[contains(@class,'bv-content-title')]").text,
'review_text':review.find_element_by_xpath(".//div[contains(@class,'bv-content-summary-body')]//div[contains(@class,'bv-content-summary-body-text')]").text,
'permalink': "NA",
'reviewlocation': "NA",
# 'reviewdate': this_review_datetime,
'subproductname': "NA",
'subproductlink': "NA",
}
item_list.append(item)
time.sleep(2)
wait.until(EC.element_to_be_clickable((By.XPATH, ".//button[contains(.,'Load More')]"))).click()
except:
break
print(item_list)
with open("output.json","r+") as outfile:
json.dump(item_list,outfile)
I am trying to scrape walgreens.com. All the reviews are being loaded by clicking on the load more button but on the output screen it shows just reviews for 1 page and that also repeated reviews. Can you please help me through it. The link I am using for scraping is https://www.walgreens.com/store/c/tylenol-extra-strength-caplets-with-500-mg-acetaminophen/ID=300396915-product?skuId=sku305332