How to handle load more button for IMDB using scrapy?

Viewed 21

I want to scrape imdb movies data. I am using scrapy and I am able to extract all the movies title and other related information but I am having issue while trying to scrape movies review with load more button. My start_url is: https://www.imdb.com/search/title/title_type=feature&primary_language=hi&sort=num_votes,desc'

def parse(self, response):

    for result in response.css('.mode-advanced'):

        primaryTitle = result.css('div.lister-item-content h3.lister-item-header a::text').extract_first()          
        link = result.css('div.lister-item-content h3.lister-item-header a::attr(href)').extract_first()
        duration = result.css('div.lister-item-content p:nth-child(2) span.runtime::text').extract_first()
        genre = result.css('div.lister-item-content p:nth-child(2) span.genre::text').extract_first()

        yield scrapy.Request(url='https://www.imdb.com'+link,
                                callback=self.parse_listing,
                                meta={'primaryTitle':primaryTitle,
                                    'duration':duration,
                                    'genre':genre})
        

    next_page = response.css('.lister-page-next::attr(href)').get()
    if next_page is not None:
        yield scrapy.Request(url='https://www.imdb.com'+next_page, callback=self.parse)


def parse_listing(self, response):
    primaryTitle = response.meta['primaryTitle']
    duration = response.meta['duration']
    genre = response.meta['genre']

    fullTitle = response.css('.fbQftq h1.eKrKux::text').extract_first()
    description = response.css('div.hrgVKw span::text').extract()
    year = response.css('.fbQftq div.iJtmbR ul.baseAlt li:nth-child(1) a.WIUyh::text').extract_first()
    rating = response.css('.fAePGh div.kYEdvH span.jGRxWM::text').extract_first()
    director = response.css('div.fjLeDR ul.ipc-metadata-list--baseAlt li:nth-child(1) div ul.baseAlt li a::text').extract()
    writer = response.css('div.fjLeDR ul.ipc-metadata-list--baseAlt li:nth-child(2) div ul.baseAlt li a::text').extract()
    stars = response.css('div.fjLeDR ul.ipc-metadata-list--baseAlt li:nth-child(3) div ul.baseAlt li a::text').extract()
    votes = response.css('div.bZeUlh div.gopMqI::text').extract_first()

    review_link = response.css('div.gcCaSg ul.baseAlt li.ghlYSH a.isReview::attr(href)').extract_first()

    yield scrapy.Request(url='https://www.imdb.com'+review_link,
                        callback=self.parse_review,
                        meta={'primaryTitle':primaryTitle,
                                'duration':duration,
                                'genre':genre,
                                'fullTitle':fullTitle,
                                'description':description,
                                'year':year,
                                'rating':rating,
                                'director':director,
                                'writer': writer,
                                'stars':stars,
                                'votes':votes})

def parse_review(self, response):
    primaryTitle = response.meta['primaryTitle']
    duration = response.meta['duration']
    genre = response.meta['genre']
    fullTitle = response.meta['fullTitle']
    description = response.meta['description']
    year = response.meta['year']
    rating = response.meta['rating']
    director = response.meta['director']
    writer = response.meta['writer']
    stars = response.meta['stars']
    votes = response.meta['votes']

    for review_details in response.css('div.lister-item-content'):

        review_rating = review_details.css('div.ipl-ratings-bar span.rating-other-user-rating span::text').extract_first()
        review_title = review_details.css('a.title::text').extract_first()
        username = review_details.css('div.display-name-date span.display-name-link a::text').extract_first()
        review_date = review_details.css('div.display-name-date span.review-date::text').extract_first()
        review_description = review_details.css('div.content div.show-more__control::text').extract_first()

        yield{'url': response.url,
            'primaryTitle': primaryTitle,
            'duration': duration,
            'genre': genre,
            'fullTitle': fullTitle,
            'description': description,
            'year': year,
            'duration':duration,
            'rating':rating,
            'director':director,
            'rating':rating,
            'director':director,
            'writer':writer,
            'stars':stars,
            'votes':votes,
            'review_rating':review_rating,
            'review_title':review_title,
            'username':username,
            'review_date':review_date,
            'description':description}

    
    base_url = 'https://www.imdb.com'
    review_ajaxurl = '/title/tt10811166/reviews/_ajax'
    paginationKey = 'g4wp7bjfqy2tg3yl7swh7nzqrht4uabhzfmxvlnomwklyczuf43o6ss4pe2vlmjncr4k4wendyfpwpblleaqtzauqo4rtea'


    load_more_url = base_url+review_ajaxurl+'?ref_=undefined&paginationKey='+paginationKey
    if load_more_url is not None:
        yield scrapy.Request(url = load_more_url, callback=self.parse_review)

The problem is how can I crawl all the reviews for movies using scrapy from load more button (IMDB).

0 Answers
Related