I have a page
where products are automatically loaded after you scroll down (only 48 products showed initially). There should be in total about 630 products.
Here is my spider code. I always get only 48 results instead of 630+. Any idea why it is not loaded all?
import scrapy
from scrapy_playwright.page import PageMethod
class PicturesSpider(scrapy.Spider):
name = 'pictures'
allowed_domains = ['www.tradeinn.com']
start_urls = ['http://www.tradeinn.com/']
def start_requests(self):
yield scrapy.Request(url='https://www.tradeinn.com/runnerinn/en/mens-shoes-trail-running-shoes/10005/s#fq=id_familia=10002&sort=v30_sum;desc@tm10;asc&fe=&pf=id_subfamilia=10005&&start=144',
meta={'playwright': True,
'playwright_include_page': True,
'playwright_page_method': [PageMethod('wait_for_selector', 'div::boton_cargar_mas.color_runnerinn'),
PageMethod("evaluate", "window.scrollBy(0, document.body.scrollHeight)")]},
callback=self.parse)
def parse(self, response):
images = response.css('div.BoxImage')
for image in images:
image_link = image.css('img::attr(src)').get()
image_description = image.css('img::attr(alt)').get()
yield {
'image_link': image_link,
'image_description': image_description
}
Any advise what should I change to get the whole content?