Making separate output files for every category in scrapy

user_11656244 0
Viewed 437

I tried to scrape yellow pages according to its categories. So i load categories from a text file and feed it to the start_urls. The problem i am facing here is saving the output separately for each category. Following is the code i tried to implement:

CATEGORIES = []
with open('Catergories.txt', 'r') as f:
    data = f.readlines()

    for category in data:
        CATEGORIES.append(category.strip())

Opening the file in settings.py and making a list to access in the spider.

The spider:

# -*- coding: utf-8 -*-
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule

from ..items import YellowItem
from scrapy.utils.project import get_project_settings

settings = get_project_settings()


class YpSpider(CrawlSpider):
    categories = settings.get('CATEGORIES')

    name = 'yp'
    allowed_domains = ['yellowpages.com']

    start_urls = ['https://www.yellowpages.com/search?search_terms={0}&geo_location_terms=New%20York'
                      '%2C '
                      '%20NY'.format(*categories)]
    rules = (

        Rule(LinkExtractor(restrict_xpaths='//a[@class="business-name"]', allow=''), callback='parse_item',
             follow=True),

        Rule(LinkExtractor(restrict_xpaths='//a[@class="next ajax-page"]', allow=''),
             follow=True),
    )

    def parse_item(self, response):
        categories = settings.get('CATEGORIES')
        print(categories)
        item = YellowItem()
        # for data in response.xpath('//section[@class="info"]'):
        item['title'] = response.xpath('//h1/text()').extract_first()
        item['phone'] = response.xpath('//p[@class="phone"]/text()').extract_first()
        item['street_address'] = response.xpath('//h2[@class="address"]/text()').extract_first()
        email = response.xpath('//a[@class="email-business"]/@href').extract_first()
        try:
            item['email'] = email.replace("mailto:", '')
        except AttributeError:
            pass
        item['website'] = response.xpath('//a[@class="primary-btn website-link"]/@href').extract_first()
        item['Description'] = response.xpath('//dd[@class="general-info"]/text()').extract_first()
        item['Hours'] = response.xpath('//div[@class="open-details"]/descendant-or-self::*/text()[not(ancestor::*['
                                       '@class="hour-category"])]').extract()
        item['Other_info'] = response.xpath(
            '//dd[@class="other-information"]/descendant-or-self::*/text()').extract()
        category_ha = response.xpath('//dd[@class="categories"]/descendant-or-self::*/text()').extract()
        item['Categories'] = " ".join(category_ha)
        item['Years_in_business'] = response.xpath('//div[@class="number"]/text()').extract_first()
        neighborhood = response.xpath('//dd[@class="neighborhoods"]/descendant-or-self::*/text()').extract()
        item['neighborhoods'] = ' '.join(neighborhood)
        item['other_links'] = response.xpath('//dd[@class="weblinks"]/descendant-or-self::*/text()').extract()

        item['category'] = '{0}'.format(*categories)

        return item

and here is the pipelines.py file:

from scrapy import signals
from scrapy.exporters import CsvItemExporter
from scrapy.utils.project import get_project_settings

settings = get_project_settings()


class YellowPipeline(object):
    @classmethod
    def from_crawler(cls, crawler):
        pipeline = cls()
        crawler.signals.connect(pipeline.spider_opened, signals.spider_opened)
        crawler.signals.connect(pipeline.spider_closed, signals.spider_closed)
        return pipeline

    def spider_opened(self, spider):
        self.exporters = {}
        categories = settings.get('CATEGORIES')

        file = open('{0}.csv'.format(*categories), 'w+b')

        exporter = CsvItemExporter(file, encoding='cp1252')
        exporter.fields_to_export = ['title', 'phone', 'street_address', 'website', 'email', 'Description',
                                     'Hours', 'Other_info', 'Categories', 'Years_in_business', 'neighborhoods',
                                     'other_links']
        exporter.start_exporting()
        for category in categories:
            self.exporters[category] = exporter

    def spider_closed(self, spider):

        for exporter in iter(self.exporters.items()):
            exporter.finish_exporting()

    def process_item(self, item, spider):

        self.exporters[item['category']].export_item(item)
        return item

After running the code i get the following error:

exporter.finish_exporting()
AttributeError: 'tuple' object has no attribute 'finish_exporting'

I need seperate csv file for each category. Any help would be appreciated.

scrapy export-to-csv csv python
2 Answers

I would do this in post-processing. Export all items to one .csv file with a category field. I think you are not thinking about this problem the right way and over complicating it. Not sure if this would even work but its worth a shot :)

with open('parent.csv', 'r') as file:
    reader = csv.reader(file)
    for row in reader:
        with open('{}.csv'.format(row[category]), 'a') as f:
            writer = csv.writer(f)
            writer.writerow(row)

You could apply this code using the spider closed signal as well.

https://docs.scrapy.org/en/latest/topics/signals.html#scrapy.signals.spider_closed

user_116462970

dict.items() returns iterable, each item in which looks like tuple (key, value) To get rid of this error you need to remove iter and unpack that items like for category, exporter in self.exporter.items():

user_64796660
Related