Python "unexpected indent" using Scrapy

Viewed 20

I was practice using scrapy and I successfully run this code and scraped all the page’s elements:

import scrapy

from scrapy.crawler import CrawlerProcess

from scrapy.selector import Selector

class ecomspider(scrapy.Spider):
name = 'ecom'
headers = {
    'user_agents': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) Applewebkit/537.36 (KHTML, like Gecko)'
}

def start_requests(self):
    url = 'https://maroof.sa/'
    yield scrapy.Request(url=url,
                         headers=self.headers,
                         callback=self.parse)
def parse(self, response):
    with open('res.html', 'w') as html_file:
        html_file.write(response.text)


process = CrawlerProcess()
process.crawl(ecomspider)
process.start()

Now, i goy be more spefic about elements but everytime i run this code i got "headers = {^ IndentationError: unexpected indent":

class ecomspider(scrapy.Spider):
name = 'ecom'

custom_settings = {
    'FEED_FORMAT': 'csv',
    'FEED_URI': 'ecom.csv',
    'LOG_FILE': 'ecom.log'
}

 headers = {
    'user_agents': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36 Edg/105.0.1343.33'
}

def start_requests(self):
    url = 'https://maroof.sa/'
    yield scrapy.Request(url=url,
                         headers=self.headers,
                         callback=self.parse)   
def parse(self, response):
    # extract data
    for card in response.css('a.tab-pro-container'):
        items = {
            'title': card.css('a.media-heading').css('a::text').get(),
        }
        
        yield items

   process = CrawlerProcess()
   process.crawl(ecomspider)
   process.start()

So what’s wrong?

1 Answers

you didn't properly indented the code.header should be perfecly below the custom_setting .

 custom_setting
 |
 |
 |
 header 

Try this and let me know ..

custom_settings = {
    'FEED_FORMAT': 'csv',
    'FEED_URI': 'ecom.csv',
    'LOG_FILE': 'ecom.log'
}
headers = {
    'user_agents': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) 
AppleWebKit/537.36 (KHTML, like Gecko) Chrome/105.0.0.0 Safari/537.36 
Edg/105.0.1343.33'
}

hope you get my point

Related