How to loop over multiple pages ? how to change page number for every request?

Question

I want to loop over multiple pages in scrapy but can't figure out the best way to do so . i just need to change the page (key in query , inside start_requests function) when i manually change number of the page it works perfect but i want to loop over all pages to extract all data from all pages because there are more then 100 pages

here is my code .

import scrapy
from Thrifts.utils import URL, get_cookie_parser
from Thrifts.items import ThriftsItem
from scrapy.loader import ItemLoader
import json


class ThriftSpider(scrapy.Spider):
    name = 'thrift'
    allowed_domains = ['www.thriftbooks.com']

def start_requests(self):  
     yield scrapy.Request(
        url=URL,
        method='POST',
        body=json.dumps({
                "searchTerms": [
                    "comic"
                ],
                "sortBy": "mostPopular",
                "sortDirection": "desc",
                "page": 1,
                "itemsPerPage": 30,
                "displayType": 2,
                "isInStock": True
            }),
        headers={
            'Content-Type': 'application/json',
            'Accept': '*/*',
            'Accept-Language': 'en-US,en;q=0.9',
            'Cache-Control': 'no-cache',
            'Connection': 'keep-alive',
            'Content-Type': 'application/json',
            'Origin': 'https://www.thriftbooks.com',
            'Pragma': 'no-cache',
            'Referer': 'https://www.thriftbooks.com/browse/?b.search=comic',
            'Request-Context': 'appId=cid-v1:c94469ec-8052-40ab-adb5-6507651e43b2',
            'Request-Id': '|e44eba662232434bbeb1b14f7ec26f11.71bdf1c0ffba40d1',
            'Sec-Fetch-Dest': 'empty',
            'Sec-Fetch-Mode': 'cors',
            'Sec-Fetch-Site': 'same-origin',
            'sec-ch-ua': '"Microsoft Edge";v="105", " Not;A Brand";v="99", "Chromium";v="105"',
            'sec-ch-ua-mobile': '?0',
            'sec-ch-ua-platform': '"Windows"',
            'traceparent': '00-e44eba662232434bbeb1b14f7ec26f11-71bdf1c0ffba40d1-01'
        },
        callback=self.parse,

        cookies=get_cookie_parser()
    )

def parse(self, response):
    # with open('initial.json', 'wb') as f:
    #     f.write(response.body)
    json_resp = json.loads(response.body)
    # print(json_resp)
    books = json_resp.get('works')
    for book in books:
        loader = ItemLoader(item=ThriftsItem())
        loader.add_value('Title', book.get('title'))
        loader.add_value('Disc_price', book.get('buyNowPrice'))
        loader.add_value('Total_price', book.get('listPrice'))
        loader.add_value('Isbn', book.get('iSBN'))
        # loader.add_value('Auther_name', book.get('authors')[0].get('authorName'))
        loader.add_value('Media', book.get('media'))

        yield loader.load_item()

user7wAF · Answer

How do you know there are approximately 100 pages? If you are able to view it somewhere on their website, try making an initial scrapy request to the page with that information and scrape that number. Your code doesnt show where you declare the value of URL that you are using in scrapy.Request. You could try something like this. Just be sure to follow the url pattern they use for each page, and fill in whatever is needed from one page to the next

number_of_pages = self.get_all_pages()
url_template = 'http://www.thriftbooks.com/page/{}'

for i in range(number_of_pages):
    url_to_specific_page = url_template.format(i)
    yield scrapy.Request(
        url=url_to_specific_page,
        ....rest of request
        callback=self.parse,
    )


def get_all_pages():
   return scrapy.request(
     url=url_of_page_that_shows_total_number,
     ....rest of request
   )

How to loop over multiple pages ? how to change page number for every request?

1 Answers