Different Scrapy feeds export destination for each request

userU9Xo 0
Viewed 25

I'm trying to save image urls for individual properties in their respective csv files via feeds export, in order for this to work, the FEEDS csv_path in custom_settings will have to be changed every time a scrapy.Request is yielded in start_requests. Every time a scrapy.Request is yielded, the self.get_csv_path in __init__ is assigned a new csv file path correspondent to the property id, it is then fetched to FEEDS by def get_feeds_csv_path as in the code below. The self.feeds_csv_path in custom_settings doesn't seem to be able to access def get_feeds_csv_path, where is the error here?

import asyncio
from configparser import ConfigParser
import os
import pandas as pd
import scrapy
import requests
import json

class GetpropertyimgurlsSpider(scrapy.Spider):
    name = 'GetPropertyImgUrls'
    custom_settings = {
        "FEEDS": {
            self.feeds_csv_path: {
                "format": "csv",
                "overwrite": True
            }
        }
    }

    def __init__(self, *args, **kwargs):
        self.feeds_csv_path = None
        super(GetpropertyimgurlsSpider, self).__init__(*args, **kwargs)

    def start_requests(self):
        files = self.get_html_files()  # List of html file full paths
        for file in files[:2]:
            self.feeds_csv_path = self.get_feeds_csv_path(file)
            yield scrapy.Request(file, callback=self.parse)

    def parse(self, response):
        texts = response.xpath("//text()").getall()
        text = texts[1]
        json_text = json.loads(text)
        #print(text)
        photos = json_text["@graph"][3]["photo"]
        for photo in photos:
            yield photo["contentUrl"]

    def get_feeds_csv_path(self, html_file_path):
        property_id = html_file_path.split("/")[-2].split("_")[1]
        feeds_csv_path = f"{html_file_path}/images/Property_{property_id}_ImgSrcs.csv"
        return feeds_csv_path

    def get_path(self):
        config = ConfigParser()
        config.read("config.ini")  # Location relative to main.py
        path = config["scrapezoopla"]["path"]
        return path

    #Returns a list of html file dirs
    def get_html_files(self):
        path = self.get_path()
        dir = f"{path}/data/properties/"
        dir_list = os.listdir(dir)
        folders = []
        for ins in dir_list:
            if os.path.isdir(f"{dir}{ins}") == True:
                append_ins = folders.append(ins)

        html_files = []
        for folder in folders:
            html_file = f"{dir}{folder}/{folder}.html"
            if os.path.isfile(html_file) == True:
                append_html_file = html_files.append(f"file:///{html_file}")
        return html_files
scrapy python
1 Answers

The first problem I see is that you are using the self keyword in the namespace scope of your spider class. The self keyword is only available inside of instance methods where you pass the keyword in as the first argument. e.g. def __init__(self...).

Even if self was available it still wouldn't work though, because once you create the custom_settings dictionary, the self.feeds_csv_path is immediately converted to it's string value at runtime, so updating the instance variable would have no effect on the custom_settings propert.

Another issue is that scrapy collects all of the custom settings and stores them internally before the crawl is actually started, and updating the custom_settings dictionary mid crawl might not actually have an effect. I am not certain about that though.

All of that being said, your goal is still achievable. One means that I can think of is by creating the FEEDS dictionary runtime but prior to initiating the crawl and filtering using custom scrapy.Item classes to filter which item belongs to which output.

I have no way of testing it so it might be buggy but here is an example of what I am referring to:

from configparser import ConfigParser
import json
import os
import scrapy

def get_path():
    config = ConfigParser()
    config.read("config.ini")  # Location relative to main.py
    path = config["scrapezoopla"]["path"]
    return path

#Returns a list of html file dirs
def get_html_files():
    path = get_path()
    folder = f"{path}/data/properties/"
    dir_list = os.listdir(folder)
    html_files = []
    for ins in dir_list:
        if os.path.isdir(f"{folder}{ins}"):
            if os.path.isfile(f"{folder}{ins}/{ins}.html"):
                html_files.append(f"file:///{folder}{ins}/{ins}.html")
    return html_files

def get_feeds_csv_path(self, html_file_path):
    property_id = html_file_path.split("/")[-2].split("_")[1]
    feeds_csv_path = f"{html_file_path}/images/Property_{property_id}_ImgSrcs.csv"
    return feeds_csv_path

def create_custom_item():
    class Item(scrapy.Item):
        contentUrl = scrapy.Field()
    return Item

def customize_settings():
    feeds = {}
    files = get_html_files()
    start_urls = {}
    for path in files:
        custom_class = create_custom_item()
        output_path = get_feeds_csv_path(path)
        start_urls[path] = custom_class
        feeds[output_path] = {
            "format": "csv",
            "item_classes": [custom_class],
        }
    custom_settings = {"FEEDS": feeds}
    return custom_settings, start_urls

class GetpropertyimgurlsSpider(scrapy.Spider):
    name = 'GetPropertyImgUrls'
    custom_settings, start_urls = customize_settings()

    def start_requests(self):
        for uri, itemclass in self.start_urls.items():
            yield scrapy.Request(uri, callback=self.parse, cb_kwargs={'itemclass': itemclass})

    def parse(self, response, itemclass):
        texts = response.xpath("//text()").getall()
        text = texts[1]
        json_text = json.loads(text)
        photos = json_text["@graph"][3]["photo"]
        for photo in photos:
            item = itemclass()
            item['contentUrl'] = photo["contentUrl"]
            yield item
userphOc10
Related