why my spider is crawling 700+ items actually there are 245 items? how?
there are not more than 245 items but how my spider is scraping 700+ items even i use for loop to crawl only selected pages and tried to set my CLOSESPIDER_ITEMCOUNT = 244 in setting.py
any possible solution?
here is my code.
import scrapy
import json
from ..items import HmsItem
from scrapy.loader import ItemLoader
class HmSpider(scrapy.Spider):
name = 'hm'
allowed_domains = ['hm.com']
def start_requests(self):
for i in range(36,252, 36): #there is a diff of 36 on each next url
yield scrapy.Request(
url = f"https://www2.hm.com/en_us/men/new-arrivals/view-all/_jcr_content/main/productlisting.display.json?sort=stock&image-size=small&image=model&offset=0&page-size={i}",
method='GET',
callback= self.parse
)
def parse(self, response):
# with open('initial.json', 'wb') as f:
# f.write(response.body)
json_resp = json.loads(response.body)
products = json_resp.get('products')
for product in products:
loader = ItemLoader(item=HmsItem())
title = loader.add_value('title', product.get('title'))
articleCode = loader.add_value('articleCode', product.get('articleCode'))
category = loader.add_value('category', product.get('category'))
src = loader.add_value('src', product.get('image')[0].get('src'))
price = loader.add_value('price', product.get('price'))
swatchesTotal = loader.add_value('swatchesTotal', product.get('swatchesTotal'))
brandName =loader.add_value('brandName', product.get('brandName'))
yield loader.load_item()
here is my setting.py
BOT_NAME = 'hms'
SPIDER_MODULES = ['hms.spiders']
NEWSPIDER_MODULE = 'hms.spiders'
USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36"
ROBOTSTXT_OBEY = False
DOWNLOAD_DELAY = 8
FEED = 'json'
FEED_EXPORT_ENCODING = 'utf-8'
CLOSESPIDER_ITEMCOUNT = 244
AUTOTHROTTLE_ENABLED = True
AUTOTHROTTLE_TARGET_CONCURRENCY = 4.0
here is my items.py
import scrapy
class HmsItem(scrapy.Item):
title = scrapy.Field()
articleCode = scrapy.Field()
category = scrapy.Field()
src = scrapy.Field()
price = scrapy.Field()
swatchesTotal = scrapy.Field()
brandName = scrapy.Field()