Scrapy: Having trouble to get value from another page

Viewed 65

I started to use Scrapy yesterday, following this modified version of Scrapy: https://github.com/prncc/steam-scraper to get Steam Reviews information. The existing code allows for continuous scrolling until there is no review left to scrape. However, I need to modify it a bit to be able to get values from another page; more specifically, on a webpage like this https://steamcommunity.com/app/416600/reviews for instance, I would like to get the number of reviews of each reviewer, which are displayed only on their review page (like this one https://steamcommunity.com/profiles/76561197993023168/recommended/, who has 14 reviews).

The original code reads:

class ReviewSpider(scrapy.Spider):
name = 'reviews'
test_urls = [
    # Full Metal Furies
    'http://steamcommunity.com/app/416600/reviews/?browsefilter=mostrecent&p=1',
]

def __init__(self, url_file=None, steam_id=None, *args, **kwargs):
    super().__init__(*args, **kwargs)
    self.url_file = url_file
    self.steam_id = steam_id

def read_urls(self):
    with open(self.url_file, 'r') as f:
        for url in f:
            url = url.strip()
            if url:
                yield scrapy.Request(url, callback=self.parse)

def start_requests(self):
    if self.steam_id:
        url = (
            f'http://steamcommunity.com/app/{self.steam_id}/reviews/'
            '?browsefilter=mostrecent&p=1'
        )
        yield Request(url, callback=self.parse)
    elif self.url_file:
        yield from self.read_urls()
    else:
        for url in self.test_urls:
            yield Request(url, callback=self.parse)

def parse(self, response):
    page = get_page(response)
    product_id = get_product_id(response)

    # Load all reviews on current page.
    reviews = response.css('div .apphub_Card')
    for i, review in enumerate(reviews):
        yield load_review(review, product_id, page, i)
        


    # Navigate to next page.
    form = response.xpath('//form[contains(@id, "MoreContentForm")]')
    if form:
        yield self.process_pagination_form(form, page, product_id)

def process_pagination_form(self, form, page=None, product_id=None):
    action = form.xpath('@action').extract_first()
    names = form.xpath('input/@name').extract()
    values = form.xpath('input/@value').extract()

    formdata = dict(zip(names, values))
    meta = dict(prev_page=page, product_id=product_id)

    return FormRequest(
        url=action,
        method='GET',
        formdata=formdata,
        callback=self.parse,
        meta=meta
    )

What I tried to do is to add this in the parse function, just to get the number of reviews for a given user:

def parse(self, response):
    page = get_page(response)
    product_id = get_product_id(response)

    # Load all reviews on current page.
    reviews = response.css('div .apphub_Card')
    for i, review in enumerate(reviews):
        yield load_review(review, product_id, page, i)
        Reviewers = response.xpath("/html/body/div[1]/div[5]/div[5]/div/div[1]/div/div/a[1]") #Get the path for each reviewer
        for IndividualReview in Reviewers:
            num_reviews = IndividualReview.xpath(".//@href").get()
            yield {
                'num_reviews': num_reviews
            }


    # Navigate to next page.
    form = response.xpath('//form[contains(@id, "MoreContentForm")]')
    if form:
        yield self.process_pagination_form(form, page, product_id)

But it did not work. The main issue is that I am not familiar in xpath in general, and I do not really understand how Scrapy is supposed to go to the other page, get the information desired and then go back, iteratively for each review on a given game. How can I tackle this issue?

0 Answers
Related