Hey so I wanna scrape some data from the response object in parse function and then get a new response object to scrape data from a different page
if there's ANY way to do it using scrapy please let me know (don't use selenium for this, I need a faster alternative...hence scrapy's inbuilt features
here's what I tried but it doesn't work...
class QuotesSpider(scrapy.Spider):
name = "Bider"
def start_requests(self):
urls = [
"https://www.flipkart.com/clothing-and-accessories/bottomwear/pr?sid=clo%2Cvua&p%5B%5D=facets.ideal_for%255B%255D%3DMen&p%5B%5D=facets.ideal_for%255B%255D%3Dmen&otracker=categorytree&fm=neo%2Fmerchandising&iid=M_b9164440-7b7c-4ab9-a5a2-080bad737908_2_372UD5BXDFYS_MC.8HARX8UX7IX5&otracker=hp_rich_navigation_2_2.navigationCard.RICH_NAVIGATION_Fashion%7EMen%2527s%2BBottom%2BWear_8HARX8UX7IX5&otracker1=hp_rich_navigation_PINNED_neo%2Fmerchandising_NA_NAV_EXPANDABLE_navigationCard_cc_2_L1_view-all&cid=8HARX8UX7IX5&sort=recency_desc"
]
for url in urls:
yield scrapy.Request(url=url,callback=self.parse)
def parse(self, response):
global SN
global Sno
website="Flipkart"
Brand_name=response.css("div._2WkVRV::text").getall()
category=response.css("a.IRpwTa::text").getall()
rating="NA"
price=response.css("div._30jeq3::text").getall()
link=response.css("a.IRpwTa::attr(href)").getall()
length=len(response.css("div._2WkVRV::text").getall())
x=0
while x<length:
#VARIABLES
BN=Brand_name[x]
pr=price[x]
pr=pr.replace(",","")
ct=category[x]
ct=ct.replace(",","")
no=str(Sno)
num_ratings="NA" #REVIEWS
CF="NA" #CUSTOMER REVIEWS
SN="" #SELLER NAME
SV="NA" #SERVICES
RT="NA, NA, NA, NA, NA," #RATINGS 5,4,3,2,1
lk="https://www.flipkart.com"
lk=lk+link[x]
scrapy.Request(url=lk,callback=self.parseprod)
with open(File_Name,"a",encoding='utf-8') as f:
# f.write(lk)
f.write(SN+"\n")
x+=1
Sno+=1
next_page=response.css("a._1LKTO3::attr(href)").getall()
if next_page is not None:
next_page=next_page[-1]
elif next_page is None:
time.sleep(1)
next_page=response.css("a._1LKTO3::attr(href)").getall()
next_page=next_page[-1]
print(response.urljoin(next_page))
if next_page is not None:
next_page=response.urljoin(next_page)
yield scrapy.Request(next_page, callback=self.parse)
def parseprod(self,response):
#SELLER NAME
global SN
SN=response.css("div#sellerName>span>span::text").get()
if SN is None or SN=="":
time.sleep(2)
SN=response.css("div#sellerName>span>span::text").get()
if SN is None or SN=="":
SN="NA"