going to a different webpage in Scrapy(not the next page)

Viewed 27

Hey so I wanna scrape some data from the response object in parse function and then get a new response object to scrape data from a different page

if there's ANY way to do it using scrapy please let me know (don't use selenium for this, I need a faster alternative...hence scrapy's inbuilt features

here's what I tried but it doesn't work...

class QuotesSpider(scrapy.Spider):
    name = "Bider"
    def start_requests(self):
        urls = [
            "https://www.flipkart.com/clothing-and-accessories/bottomwear/pr?sid=clo%2Cvua&p%5B%5D=facets.ideal_for%255B%255D%3DMen&p%5B%5D=facets.ideal_for%255B%255D%3Dmen&otracker=categorytree&fm=neo%2Fmerchandising&iid=M_b9164440-7b7c-4ab9-a5a2-080bad737908_2_372UD5BXDFYS_MC.8HARX8UX7IX5&otracker=hp_rich_navigation_2_2.navigationCard.RICH_NAVIGATION_Fashion%7EMen%2527s%2BBottom%2BWear_8HARX8UX7IX5&otracker1=hp_rich_navigation_PINNED_neo%2Fmerchandising_NA_NAV_EXPANDABLE_navigationCard_cc_2_L1_view-all&cid=8HARX8UX7IX5&sort=recency_desc"
        ]
        for url in urls:
                yield scrapy.Request(url=url,callback=self.parse)

    def parse(self, response):
        global SN
     
        global Sno
        
        website="Flipkart"
        Brand_name=response.css("div._2WkVRV::text").getall()
        category=response.css("a.IRpwTa::text").getall()
        rating="NA"
        price=response.css("div._30jeq3::text").getall()
        link=response.css("a.IRpwTa::attr(href)").getall()

        length=len(response.css("div._2WkVRV::text").getall())

        x=0
        
        while x<length:
            
                
            #VARIABLES
            BN=Brand_name[x]
            pr=price[x]
            pr=pr.replace(",","")
            ct=category[x]
            ct=ct.replace(",","")
            no=str(Sno)
            num_ratings="NA"  #REVIEWS
            CF="NA"               #CUSTOMER REVIEWS
            SN=""               #SELLER NAME
            SV="NA"               #SERVICES
            RT="NA, NA, NA, NA, NA,"              #RATINGS 5,4,3,2,1
            lk="https://www.flipkart.com"
            lk=lk+link[x]
            
            scrapy.Request(url=lk,callback=self.parseprod)

            with open(File_Name,"a",encoding='utf-8') as f:
                # f.write(lk)
                f.write(SN+"\n")

            
            x+=1
            Sno+=1
        next_page=response.css("a._1LKTO3::attr(href)").getall()
        if next_page is not None:
            next_page=next_page[-1]
        elif next_page is None:
            time.sleep(1)
            next_page=response.css("a._1LKTO3::attr(href)").getall()
            next_page=next_page[-1]
        
        print(response.urljoin(next_page))
        if next_page is not None:
            next_page=response.urljoin(next_page)
            

            yield scrapy.Request(next_page, callback=self.parse)

    

    def parseprod(self,response):
        #SELLER NAME
        global SN
        SN=response.css("div#sellerName>span>span::text").get()
        if SN is None or SN=="":
            time.sleep(2)
            SN=response.css("div#sellerName>span>span::text").get()
        if SN is None or SN=="":
            SN="NA"
0 Answers
Related