I'm scraping reviews from Glassdoor.com and each time I run the code I get duplicate pages. For instance, the first page is scraped twice.
If you check the code, from was is printed in the terminal you notice that page 2 and page 3 both have 20. It means it skipped to scrape the reviews of page 3, therefore, page 2 is duplicated.
Any Solution for this?
import requests
from bs4 import BeautifulSoup
import pandas as pd #we will need a datset
headers = {'user-agent' : 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5112.80 Safari/537.36'}
ReviewsList = []
def extract(pg):
url = f'https://www.glassdoor.com/Reviews/Meta-Information-Technology-Reviews-EI_IE40772.0,4_DEPT1011_IP{pg}.htm?sort.sortType=RD&sort.ascending=false&filter.iso3Language=eng&filter.employmentStatus=REGULAR&filter.employmentStatus=PART_TIME'
r = requests.get(url, headers)
soup = BeautifulSoup(r.content, 'html.parser')# this a soup function that retuen the whole html
#to get reviews
divs = soup.find_all('div', class_='gdReview')
try:
for item in divs:
Title = item.find('h2', class_= 'mb-xxsm mt-0 css-93svrw el6ke055').text.strip()
Rating = item.find('span', class_= 'ratingNumber mr-xsm').text.replace('<span class="ratingNumber mr-xsm">', '').strip()
Employee_Situation= item.find('span', class_= 'pt-xsm pt-md-0 css-1qxtz39 eg4psks0').text.replace('<span class="pt-xsm pt-md-0 css-1qxtz39 eg4psks0">', '').strip()
Pros = item.find('span', {'data-test':'pros'}).text.replace('<span data-test="pros">', '').strip()
Cons = item.find('span', {'data-test':'cons'}).text.replace('<span data-test="cons">', '')
Author_Info = item.find('span', class_= 'common__EiReviewDetailsStyle__newUiJobLine').text.replace('<span class="common__EiReviewDetailsStyle__newUiJobLine"><span><span class="middle common__EiReviewDetailsStyle__newGrey">', '').strip()
Reviews = {
'Title' : Title,
'Rating': Rating,
'Employee_Situation' : Employee_Situation,
'Pros' : Pros,
'Cons' : Cons, 'Auhtor_Info' : Author_Info,
}
ReviewsList.append(Reviews)
return
except:
pass
#loop into pages
for i in range(1,10,1):
soup = extract( f'https://www.glassdoor.com/Reviews/Meta-Information-Technology-Reviews-EI_IE40772.0,4_DEPT1011_IP{i}.htm?sort.sortType=RD&sort.ascending=false&filter.iso3Language=eng&filter.employmentStatus=REGULAR&filter.employmentStatus=PART_TIME')
print(f' page {i}')
extract(soup)
print(len(ReviewsList))
df = pd.DataFrame(ReviewsList)
df.to_csv('GlassdoorReviews2.csv')
print(len(ReviewsList))
And this is was is printed in the terminal:
page 1
10
page 2
20
page 3
20
page 4
30
page 5
50
page 6
70
page 7
80
page 8
80
page 9
100
100