I was practice using scrapy and I successfully run this code and scraped all the page’s elements:
import scrapy
from scrapy.crawler import CrawlerProcess
from scrapy.selector import Selector
class ecomspider(scrapy.Spider):
name = 'ecom'
headers = {
'user_agents': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Applewebkit/537.36 (KHTML, like Gecko)'
}
def start_requests(self):
url = 'https://maroof.sa/'
yield scrapy.Request(url=url,
headers=self.headers,
callback=self.parse)
def parse(self, response):
with open('res.html', 'w') as html_file:
html_file.write(response.text)
process = CrawlerProcess()
process.crawl(ecomspider)
process.start()
Now, i goy be more spefic about elements but everytime i run this code i got "headers = {^ IndentationError: unexpected indent":
class ecomspider(scrapy.Spider):
name = 'ecom'
custom_settings = {
'FEED_FORMAT': 'csv',
'FEED_URI': 'ecom.csv',
'LOG_FILE': 'ecom.log'
}
headers = {
'user_agents': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36 Edg/105.0.1343.33'
}
def start_requests(self):
url = 'https://maroof.sa/'
yield scrapy.Request(url=url,
headers=self.headers,
callback=self.parse)
def parse(self, response):
# extract data
for card in response.css('a.tab-pro-container'):
items = {
'title': card.css('a.media-heading').css('a::text').get(),
}
yield items
process = CrawlerProcess()
process.crawl(ecomspider)
process.start()
So what’s wrong?