Using scrapy they scrape incomplete data

Viewed 36

I have an script done by myself that is not getting all data. If the pages have a total of 870 items, it's getting about 700, I think it´s something relates to the user agent or delay of rendering...I attached the script so u can see it. what the problem they give me incomplete data Thank in advance

import scrapy
from scrapy_splash import SplashRequest
from concurrent.futures import process
from scrapy.crawler import CrawlerProcess
from datetime import datetime
import os
from scrapy.utils.response import open_in_browser
from scrapy.linkextractors import LinkExtractor
import random
import logging


script = """
    function main(splash)
        assert(splash:go(splash.args.url))
        splash:wait(0.5)
        local title = splash:evaljs('document.title')
        return {title=title}
    end
"""

if os.path.exists('jfs_hombre.csv'):
    os.remove('jfs_hombre.csv')
    print("The file has been deleted successfully")
else:
    print("The file does not exist!")

class JfsSpider_hombre(scrapy.Spider):
    name = 'jfs_hombre'
    start_urls = ["https://www.justforsport.com.ar/hombre?page=1"]



        

    def parse(self,response):
        
        #total_products=int(int(response.css('div.vtex-search-result-3-x-totalProducts--layout.pv5.ph9.bn-ns.bt-s.b--muted-5.tc-s.tl.t-action--small span::text').get())/32) + 1
        total_products=30
        for count in range(1, total_products):
            yield SplashRequest(url=f'https://www.justforsport.com.ar/hombre?page={count}',
            callback=self.parse_links, endpoint='render.html', args={'wait': 2.5, 'lua_source': script},
            headers={'User-Agent': 'Googlebot/2.1 (+http://www.google.com/bot.html)',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'})
                          
                          
                        
   #Extrae links de cada pagina de la seccion
    def parse_links(self,response):
        links=response.css('a.vtex-product-summary-2-x-clearLink.vtex-product-summary-2-x-clearLink--shelf-product.h-100.flex.flex-column::attr(href)').getall()  
        for link in links:
            yield SplashRequest(response.urljoin('https://www.justforsport.com.ar' + link), 
            self.parse_article_detail,endpoint='render.html', args={'wait': 2.5, 'lua_source': script},
            headers={'User-Agent': 'Googlebot/2.1 (+http://www.google.com/bot.html)',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'})
        
      
    def parse_article_detail(self, response):
        #precio0=response.css('span.vtex-product-price-1-x-currencyContainer.vtex-product-price-1-x-currencyContainer--product')[0]
        yield {
            'Casa':'Just_For_Sports',
            'Sku' :response.css('span.vtex-product-identifier-0-x-product-identifier__value::text').get(),
            'Name':response.css('span.vtex-store-components-3-x-productBrand::text').get() ,
            'precio': response.xpath('//meta[@property="product:price:amount"]/@content').get(),
            #'precio':''.join(precio0.css('span.vtex-product-price-1-x-currencyInteger.vtex-product-price-1-x-currencyInteger--product::text').getall()),
            'Link':response.url,
            'Date':datetime.today().strftime('%Y-%m-%d')
                 }
0 Answers
Related