I got some problems while scrapying the secod page from the first page link in Amazon

Viewed 54

I was doing an experiment with scraping information on amazon,so I wanted to go from the link on the first page, and then go to the link to download some information. but I had some problems, it happened so many times that I couldn't continue . I am appeciating for your help

shuju.py

from AmazonsPro.items import AmazonsproItem


class ShujuSpider(scrapy.Spider):
    name = 'shuju'
    #allowed_domains = ['www.amazon.com']
    start_urls = ['https://www.amazon.com/Best-Sellers-Office-Products-Woodcase-Lead-Pencils/zgbs/office-products/490674011/ref=zg_bs_pg_1?_encoding=UTF8&pg=1']

    def parse(self, response):
        li_list=response.xpath('//ol[@id="zg-ordered-list"]/li')
        link_list=[]
        for li in li_list:
            #get the link
            link='www.amazon.com'+li.xpath('.//span[@class="aok-inline-block zg-item"]/a/@href').extract_first()
            #get the rank
            rank=li.xpath('.//span[@class="a-size-small aok-float-left zg-badge-body zg-badge-color"]/span/text()').extract_first()
            link_list.append(link)
        for link in link_list:
            print(link)
            yield scrapy.Request(url=link,callback=self.sec_parse)
    #get the the second page 
    def sec_parse(self,response):
        item=AmazonsproItem()
        print('star second page')
        title=response.xpath('.//*[@id="productTitle"]/text()').extract_first()
        brand=response.xpath('.//*[@id="bylineInfo"]/text()').extract_first()
        item['title']=title
        item['brand']=brand
        print('done')
        yield item

setting.py


ROBOTSTXT_OBEY = False

ITEM_PIPELINES = {
    'AmazonsPro.pipelines.AmazonsproPipeline': 300,
    }

pipelines.py

    fp=None
    def open_spider(self,spider):
        print('star scrapy')
        self.fp = open('./Asin.txt','w')

    def process_item(self,item,spider):
        self.fp.write(item['rank']+"+"+item['brand']+"+"+item['title']+"+"+item['star'])
        return item

    def close_spider(self,spider):
        self.fp.close()
        print('over')

items.py

    # define the fields for your item here like:
    # name = scrapy.Field()
    rank=scrapy.Field()
    title=scrapy.Field()
    link=scrapy.Field()
    brand=scrapy.Field()

There is no error occur but it running the def sec_parse() function just once(actually 50 times).hopefuly you can help

1 Answers

Gotcha,I didn't add https:// at link ,so it won't get work . Thxs

Related