Python webscraping - Repeated data while webscraping using BeautifulSoup

Viewed 23

I am trying to retrieve online customer reviews from the Lush USA website using BeautifulSoup. The website page is dynamic, so I'm webscraping using pagination and page number. I've managed to successfully webscrape the first 11 pages of product reviews but after that, my program webscrapes from URL page 2 again. The program also has a tendency to end before all the data has been scraped. How can I fix this?

Lush USA website: https://www.lushusa.com/hair/shampoo-bars/soak-and-float/9999905130.html

Here's my code:

from gettext import find
from requests_html import HTMLSession
from bs4 import BeautifulSoup
import requests
import pandas as pd

s = HTMLSession()
url = 'https://www.lushusa.com/hair/shampoo-bars/soak-and-float/9999905130.html'
reviewlist = []

def get_soup(url):
    r = requests.get('http://localhost:8050/render.html', params={'url': url, 'wait': 2})
    soup = BeautifulSoup(r.text, 'html.parser')
    return soup

def getnextpage(soup):
    pages = soup.find('div', {'class': 'pr-rd-pagination'})
    if pages.find_all('a', {'class': 'pr-rd-pagination-btn pr-rd-pagination-btn--next'}):
        next_page = soup.find_all('a', {'class': 'pr-rd-pagination-btn pr-rd-pagination-btn--next'})
        print(next_page)
        for element in next_page:
            url = element.get("href")
            print(url)
            return url
    else:
        return print("No URL found.")

def get_reviews(soup):
    reviews = soup.find_all('div', {'class': 'pr-review'})
    for item in reviews:
        review = {
    'title': item.find('span', {'class': 'pr-rd-review-headline pr-h2'}).text,
    'rating': float(item.find('div', {'class': 'pr-rd-star-rating'}).text),
    'body': item.find('p', {'class': 'pr-rd-description-text'}).text,
        }
        reviewlist.append(review)

while True:   
    try:
        soup = get_soup(url)
        get_reviews(soup)
        print("Number of rows: ", len(reviewlist))

        #get new URL
        url = getnextpage(soup)
    except:
        print("No more URLs.")
        break

df = pd.DataFrame(reviewlist)
df.to_excel('soakandfloat-reviews.xlsx', index=False)
print("Fin.")

Here's my output to show that it repeats webscraping from page 2 onwards.

0 Answers
Related