How to prevent duplicates during scraping reviews?

Viewed 37

I'm scraping reviews from Glassdoor.com and each time I run the code I get duplicate pages. For instance, the first page is scraped twice.

If you check the code, from was is printed in the terminal you notice that page 2 and page 3 both have 20. It means it skipped to scrape the reviews of page 3, therefore, page 2 is duplicated.

Any Solution for this?

import requests
from bs4 import BeautifulSoup
import pandas as pd #we will need a datset 
 

headers = {'user-agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.80 Safari/537.36'}
ReviewsList = []

def extract(pg): 
    
    url = f'https://www.glassdoor.com/Reviews/Meta-Information-Technology-Reviews-EI_IE40772.0,4_DEPT1011_IP{pg}.htm?sort.sortType=RD&sort.ascending=false&filter.iso3Language=eng&filter.employmentStatus=REGULAR&filter.employmentStatus=PART_TIME'
    r = requests.get(url, headers)
    soup = BeautifulSoup(r.content, 'html.parser')# this a soup function that retuen the whole html

    #to get reviews

    divs = soup.find_all('div', class_='gdReview')
    try:
        for item in divs:       
            Title = item.find('h2', class_= 'mb-xxsm mt-0 css-93svrw el6ke055').text.strip()
            Rating = item.find('span', class_= 'ratingNumber mr-xsm').text.replace('<span class="ratingNumber mr-xsm">', '').strip()
            Employee_Situation= item.find('span', class_= 'pt-xsm pt-md-0 css-1qxtz39 eg4psks0').text.replace('<span class="pt-xsm pt-md-0 css-1qxtz39 eg4psks0">', '').strip()
            Pros = item.find('span',  {'data-test':'pros'}).text.replace('<span data-test="pros">', '').strip()
            Cons = item.find('span',  {'data-test':'cons'}).text.replace('<span data-test="cons">', '')
            Author_Info = item.find('span', class_= 'common__EiReviewDetailsStyle__newUiJobLine').text.replace('<span class="common__EiReviewDetailsStyle__newUiJobLine"><span><span class="middle common__EiReviewDetailsStyle__newGrey">', '').strip()
            
           
            Reviews = {
                'Title' : Title,
                'Rating': Rating,
                'Employee_Situation' : Employee_Situation,
                'Pros' : Pros,
                'Cons' : Cons, 'Auhtor_Info' : Author_Info,

              
            } 

            ReviewsList.append(Reviews) 
        return
    except:
        pass
   



#loop into pages
for i in range(1,10,1):
    soup = extract( f'https://www.glassdoor.com/Reviews/Meta-Information-Technology-Reviews-EI_IE40772.0,4_DEPT1011_IP{i}.htm?sort.sortType=RD&sort.ascending=false&filter.iso3Language=eng&filter.employmentStatus=REGULAR&filter.employmentStatus=PART_TIME')
    print(f' page {i}')  
    extract(soup)
print(len(ReviewsList))



df = pd.DataFrame(ReviewsList)
df.to_csv('GlassdoorReviews2.csv')

print(len(ReviewsList))

And this is was is printed in the terminal:

page 1
10
 page 2
20
 page 3
20
 page 4
30
 page 5
50
 page 6
70
 page 7
80
 page 8
80
 page 9
100
100
1 Answers

well I faced the same issue before and I easily solved the problem by using a set to keep only unique values
here is an example of set

myset = {"apple", "banana", "cherry"} # i used string here as an example you can use any other object 
myset.add("apple") # here i added duplicated value
print(myset)

and here is the output

{'apple', 'cherry', 'banana'}
Related