I have an script done by myself that is not getting all data. If the pages have a total of 870 items, it's getting about 700, I think it´s something relates to the user agent or delay of rendering...I attached the script so u can see it. what the problem they give me incomplete data Thank in advance
import scrapy
from scrapy_splash import SplashRequest
from concurrent.futures import process
from scrapy.crawler import CrawlerProcess
from datetime import datetime
import os
from scrapy.utils.response import open_in_browser
from scrapy.linkextractors import LinkExtractor
import random
import logging
script = """
function main(splash)
assert(splash:go(splash.args.url))
splash:wait(0.5)
local title = splash:evaljs('document.title')
return {title=title}
end
"""
if os.path.exists('jfs_hombre.csv'):
os.remove('jfs_hombre.csv')
print("The file has been deleted successfully")
else:
print("The file does not exist!")
class JfsSpider_hombre(scrapy.Spider):
name = 'jfs_hombre'
start_urls = ["https://www.justforsport.com.ar/hombre?page=1"]
def parse(self,response):
#total_products=int(int(response.css('div.vtex-search-result-3-x-totalProducts--layout.pv5.ph9.bn-ns.bt-s.b--muted-5.tc-s.tl.t-action--small span::text').get())/32) + 1
total_products=30
for count in range(1, total_products):
yield SplashRequest(url=f'https://www.justforsport.com.ar/hombre?page={count}',
callback=self.parse_links, endpoint='render.html', args={'wait': 2.5, 'lua_source': script},
headers={'User-Agent': 'Googlebot/2.1 (+http://www.google.com/bot.html)',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'})
#Extrae links de cada pagina de la seccion
def parse_links(self,response):
links=response.css('a.vtex-product-summary-2-x-clearLink.vtex-product-summary-2-x-clearLink--shelf-product.h-100.flex.flex-column::attr(href)').getall()
for link in links:
yield SplashRequest(response.urljoin('https://www.justforsport.com.ar' + link),
self.parse_article_detail,endpoint='render.html', args={'wait': 2.5, 'lua_source': script},
headers={'User-Agent': 'Googlebot/2.1 (+http://www.google.com/bot.html)',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'})
def parse_article_detail(self, response):
#precio0=response.css('span.vtex-product-price-1-x-currencyContainer.vtex-product-price-1-x-currencyContainer--product')[0]
yield {
'Casa':'Just_For_Sports',
'Sku' :response.css('span.vtex-product-identifier-0-x-product-identifier__value::text').get(),
'Name':response.css('span.vtex-store-components-3-x-productBrand::text').get() ,
'precio': response.xpath('//meta[@property="product:price:amount"]/@content').get(),
#'precio':''.join(precio0.css('span.vtex-product-price-1-x-currencyInteger.vtex-product-price-1-x-currencyInteger--product::text').getall()),
'Link':response.url,
'Date':datetime.today().strftime('%Y-%m-%d')
}