BeautifulSoup- Accessing More Reviews

user_12677713 0
Viewed 121

I am trying to web-scrape reviews from an IMDB movie link and extracting usernames for reviews, I am only getting 25 usernames since thats what the page shows until you press "Show More". I need a way to accesss all reviews, is there a way to do this besides using Selenium because for some reason I get a SSL cert error when trying to import that.

import requests
from time import sleep
url='https://www.imdb.com/title/tt0068646/reviews?ref_=tt_urv'
response= requests.get(url,verify=False)
response
import bs4
soup=bs4.BeautifulSoup(response.content, 'html5lib')
name=soup.find_all('span', class_='display-name-link')
len(name)
selenium request beautifulsoup python
3 Answers

To scrape all usernames (a total of 4041), send a GET request to simulate clicking the button:

import requests
from bs4 import BeautifulSoup

main_url = "https://www.imdb.com/title/tt0068646/reviews?ref_=tt_urv"
ajax_url = "https://www.imdb.com/title/tt0068646/reviews/_ajax?ref_=undefined&paginationKey={}"
soup = BeautifulSoup(requests.get(main_url).content, "html5lib")

while True:
    for tag in soup.select(".display-name-link"):
        print(tag.text)
    print("-" * 30)

    button = soup.select_one(".load-more-data")
    if not button:
        break

    key = button["data-key"]
    soup = BeautifulSoup(requests.get(ajax_url.format(key)).content, "html5lib")

Output:

CalRhys
gogoschka-1
SJ_1
andrewburgereviews
alexkolokotronis
MR_Heraclius
b-a-h TNT-6
danielfeerst
mattrochman
Godz365
winnantonio
Trevizolga
DaveDiggler
ks4
...
... All the way until
Steven Bray
Castor-5
BLDJ
pinky67
dean keaton
rejoefrankel
Timothy
user_123497340

I cannot think of clicking elements without Selenium. You can add options in browsers for ignoring SSL certificate errors.

Firefox

from selenium import webdriver

profile = webdriver.FirefoxProfile()
profile.accept_untrusted_certs = True

driver = webdriver.Firefox(firefox_profile=profile)
driver.get('https://www.imdb.com/title/tt0068646/reviews?ref_=tt_urv')

driver.close()

Chrome

from selenium import webdriver

options = webdriver.ChromeOptions()
options.add_argument('ignore-certificate-errors')

driver = webdriver.Chrome(chrome_options=options)
driver.get('https://www.imdb.com/title/tt0068646/reviews?ref_=tt_urv')

driver.close()

IE

from selenium import webdriver

capabilities = webdriver.DesiredCapabilities().INTERNETEXPLORER
capabilities['acceptSslCerts'] = True

driver = webdriver.Ie(capabilities=capabilities)
driver.get('https://www.imdb.com/title/tt0068646/reviews?ref_=tt_urv')

driver.close()
user_122698570

Here's what you're looking for:

import requests as r
from time import sleep
from bs4 import BeautifulSoup
# We'll use this link that gets only the reviews
url_reviews = 'https://www.imdb.com/title/tt0068646/reviews/_ajax'
# We need to get a key everytime to scrape the next page
url_reviews_next = 'https://www.imdb.com/title/tt0068646/reviews/_ajax?paginationKey='
response= r.get(url_reviews)
soup = BeautifulSoup(response.text, 'html.parser')
name = soup.find_all('span', class_='display-name-link')
# Get your data here
paginationKey = ''
# If there's only one page and more, there's won't be the class load_more_data
try:
    paginationKey = soup.find_all('div', class_='load-more-data')[0]['data-key']
except:
    paginationKey = ''
print(paginationKey)
while paginationKey != '':
    response= r.get(url_reviews_next+paginationKey)
    soup = BeautifulSoup(response.text, 'html.parser')
    # Get your data here
    try:
        paginationKey = soup.find_all('div', class_='load-more-data')[0]['data-key']
    except:
        paginationKey = ''
    print(paginationKey)
    # If the page has no more pagination, it will catch the exception

Everytime we need to extract the paginationKey to scrape the next page, no need for selenium.

user_144033760
Related