I am trying to scrape bestbuy.com and I am able to scrape just one page rather than multiple pages

Question

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.chrome.service import Service as ChromeService
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.action_chains import ActionChains
import configparser
from datetime import datetime, timedelta, date
import time
import json
import requests



parser = configparser.RawConfigParser()

parser.read('config.ini')
page=parser['PROPERTIES']['PAGE']
url= parser['PROPERTIES']['URL']
OMIT_KEYWORDS= parser['FILTERS']['OMIT'].split(',')
INCLUDE_KEYWORDS=parser['FILTERS']['INCLUDE'].split(',')
END_DATE = datetime.strptime(parser['DATE']['END'], '%Y-%m-%d')
START_DATE=datetime.strptime(parser['DATE']['START'],'%Y-%m-%d')
minimum_comment_length = int(parser['PROPERTIES']['MIN_COMMENT_LENGTH'])
maximum_comment_length = int(parser['PROPERTIES']['MAX_COMMENT_LENGTH'])

# Setting up driver options
options = webdriver.ChromeOptions()
# Setting up Path to chromedriver executable file
CHROMEDRIVER_PATH =r'C:\Users\HP\Desktop\INTERNSHIP\Target\chromedriver.exe'
# Adding options
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option("useAutomationExtension", False)
# Setting up chrome service
service = ChromeService(executable_path=CHROMEDRIVER_PATH)
# Establishing Chrom web driver using set services and options
driver = webdriver.Chrome(service=service, options=options)
wait = WebDriverWait(driver, 20)      
driver.implicitly_wait(10)
time.sleep(2)
driver.execute_script("window.scrollTo(0,document.body.scrollHeight)")
item_list=[]  

driver.get(url)
reviews = wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, ".review-item")))
time.sleep(2)


   

for review in reviews:
    this_review_date_string= review.find_element_by_xpath(".//time[contains(@class,'submission-date')]") 
    this_review_date_string_ago= this_review_date_string.text
    date_today= date.today()
    

    if "month" in this_review_date_string_ago:
        date_period_string = this_review_date_string_ago.split("month")[0]
        date_period_int = int(date_period_string)*30
        temp_review_date = date_today - timedelta(days=date_period_int)

    elif "day" in this_review_date_string_ago: 
        date_period_string=this_review_date_string_ago.split("day")[0]
        date_period_int = int(date_period_string)
        temp_review_date = date_today - timedelta(days=date_period_int)

    elif "hour" in this_review_date_string_ago: 
        date_period_string=this_review_date_string_ago.split("hour")[0]
        date_period_int = int(date_period_string)
        temp_review_date = date_today - timedelta(hours=date_period_int)

    elif "year" in this_review_date_string_ago:
        date_period_string=this_review_date_string_ago.split("year")[0]
        date_period_int = int(date_period_string)*365
        temp_review_date = date_today - timedelta(days=date_period_int)

    this_review_datetime = temp_review_date.strftime('%d %B %Y')
    current_date= datetime.strptime( this_review_datetime, '%d %B %Y')


    if  (START_DATE< current_date < END_DATE):
            item={  
                'stars': review.find_element_by_xpath(".//p[contains(@class,'visually-hidden')]").text.replace("out of 5 stars","").replace("Rated",""),
                'username': review.find_element_by_xpath(".//div[contains(@class,'ugc-author v-fw-medium body-copy-lg')]").text,
                'userurl':"NA",
                'title':review.find_element_by_xpath(".//h4[contains(@class,'c-section-title review-title heading-5 v-fw-medium')]").text,
                'review_text':review.find_element_by_xpath(".//div[contains(@class,'ugc-review-body')]//p[contains(@class,'pre-white-space')]").text,
                'permalink': "NA",
                'reviewlocation': "NA",
                'reviewdate': this_review_datetime,
                'subproductname': "NA",
                'subproductlink': "NA",
            }
            item_list.append(item)
            
print(item_list)
with open("output.json","r+") as outfile:
    json.dump(item_list,outfile)

I want to scrape reviews from all pages but for now I am getting just one page reviews. The link I am using for scraping is https://www.bestbuy.com/site/reviews/bella-pro-series-12-6-qt-digital-air-fryer-oven-stainless-steel/6412331?variant=A&skuId=6412331&page=1. I want to paginate. Please tell me how to run the loop so that I can scrape all pages.

Arundeep Chohan · Answer

page=2
while True:
    try:
        #your code
        driver.find_element(By.XPATH,f"//a[text()='{page}']").click()
        page+=1
    except:
        break

Should be a simple way to click the a tags with the new page number.

userpr5s · Answer

i don't write python code that much so its not going to be super clear. you just need to go to the page with dynamic page number (start with range to 200-300), and whenever you see no review elements found on the page, assume thats end of the comments, then move to next product. ...

item_list=[]
def getPage():
  for i in range(arbitary_number):
    time.sleep(2)
    driver.execute_script(
      "window.scrollTo(0,document.body.scrollHeight)")
    url = https://www.bestbuy.com/site/reviews/bella-pro-series-12-6-qt-digital-air-fryer-oven-stainless-steel/6412331?variant=A&skuId=6412331&page={i}
    driver.get(url)
reviews = wait.until(EC.visibility_of_all_elements_located((By.CSS_SELECTOR, ".review-item")))
time.sleep(2)

I am trying to scrape bestbuy.com and I am able to scrape just one page rather than multiple pages

2 Answers