I started to use Scrapy yesterday, following this modified version of Scrapy: https://github.com/prncc/steam-scraper to get Steam Reviews information. The existing code allows for continuous scrolling until there is no review left to scrape. However, I need to modify it a bit to be able to get values from another page; more specifically, on a webpage like this https://steamcommunity.com/app/416600/reviews for instance, I would like to get the number of reviews of each reviewer, which are displayed only on their review page (like this one https://steamcommunity.com/profiles/76561197993023168/recommended/, who has 14 reviews).
The original code reads:
class ReviewSpider(scrapy.Spider):
name = 'reviews'
test_urls = [
# Full Metal Furies
'http://steamcommunity.com/app/416600/reviews/?browsefilter=mostrecent&p=1',
]
def __init__(self, url_file=None, steam_id=None, *args, **kwargs):
super().__init__(*args, **kwargs)
self.url_file = url_file
self.steam_id = steam_id
def read_urls(self):
with open(self.url_file, 'r') as f:
for url in f:
url = url.strip()
if url:
yield scrapy.Request(url, callback=self.parse)
def start_requests(self):
if self.steam_id:
url = (
f'http://steamcommunity.com/app/{self.steam_id}/reviews/'
'?browsefilter=mostrecent&p=1'
)
yield Request(url, callback=self.parse)
elif self.url_file:
yield from self.read_urls()
else:
for url in self.test_urls:
yield Request(url, callback=self.parse)
def parse(self, response):
page = get_page(response)
product_id = get_product_id(response)
# Load all reviews on current page.
reviews = response.css('div .apphub_Card')
for i, review in enumerate(reviews):
yield load_review(review, product_id, page, i)
# Navigate to next page.
form = response.xpath('//form[contains(@id, "MoreContentForm")]')
if form:
yield self.process_pagination_form(form, page, product_id)
def process_pagination_form(self, form, page=None, product_id=None):
action = form.xpath('@action').extract_first()
names = form.xpath('input/@name').extract()
values = form.xpath('input/@value').extract()
formdata = dict(zip(names, values))
meta = dict(prev_page=page, product_id=product_id)
return FormRequest(
url=action,
method='GET',
formdata=formdata,
callback=self.parse,
meta=meta
)
What I tried to do is to add this in the parse function, just to get the number of reviews for a given user:
def parse(self, response):
page = get_page(response)
product_id = get_product_id(response)
# Load all reviews on current page.
reviews = response.css('div .apphub_Card')
for i, review in enumerate(reviews):
yield load_review(review, product_id, page, i)
Reviewers = response.xpath("/html/body/div[1]/div[5]/div[5]/div/div[1]/div/div/a[1]") #Get the path for each reviewer
for IndividualReview in Reviewers:
num_reviews = IndividualReview.xpath(".//@href").get()
yield {
'num_reviews': num_reviews
}
# Navigate to next page.
form = response.xpath('//form[contains(@id, "MoreContentForm")]')
if form:
yield self.process_pagination_form(form, page, product_id)
But it did not work. The main issue is that I am not familiar in xpath in general, and I do not really understand how Scrapy is supposed to go to the other page, get the information desired and then go back, iteratively for each review on a given game. How can I tackle this issue?