Storing multiple loaders into a list

user_15675231 0
Viewed 54

I'm extracting webpage data and I need to store the output of some loaders into a dictionary list. For example, part of my output would look like this:

{"nine:"character","ten":"condition", "eleven":"score", "twelve":"graded":,"v9": "Electabuzz", "v10": "Near Mint", "v11": "8", "v12": "Yes"}

I instead want to store these values as two separate lists, one for those beginning with v and the other for the numerals. For example,

{"numeral":["character", "condition", "score","graded],"values":["Electabuzz", "Near Mint", "8", "Yes"]}

I'm trying to accomplish this within scrapy, however I cannot get an output like the above, for example here's my code:

import scrapy
from scrapy.item import Field
from itemloaders.processors import TakeFirst
from scrapy.crawler import CrawlerProcess

from scrapy.loader import ItemLoader


class EbayItem(scrapy.Item):
    category = Field(output_processor=TakeFirst())
    name = Field(output_processor=TakeFirst())
    price = Field(output_processor=TakeFirst())
    product_url = Field(output_processor=TakeFirst())
    eleven = Field(output_processor=TakeFirst())
    twelve = Field(output_processor=TakeFirst())
    v11 = Field(output_processor=TakeFirst())
    v12 = Field(output_processor=TakeFirst())

class EbaySpider(scrapy.Spider):
    name = 'ebay'
    start_urls = {
        'english': 'https://www.ebay.com/sch/i.html?_from=R40&_nkw=pokemon+cards&_sacat=2536&LH_TitleDesc=0&_sop=16&LH_All=1&rt=nc&Language=English&_dcat=183454', 
        'japanese':'https://www.ebay.com/sch/i.html?_from=R40&_nkw=pokemon+cards&_sacat=2536&LH_TitleDesc=0&_sop=16&LH_All=1&_oaa=1&rt=nc&Language=Japanese&_dcat=183454'
      
    }
    
    def start_requests(self):

        for category, url in self.start_urls.items():
            yield scrapy.Request(
                url=url, 
                callback=self.parse,
                cb_kwargs={
                    'category': category
                }
            )

    def parse(self, response, category):
        all_cards = response.xpath('//div[@class="s-item__wrapper clearfix"]')
        for card in all_cards:
            
            loader = ItemLoader(EbayItem(), selector=card)
            loader.add_value('category', category)
            loader.add_xpath('name', './/h3/text()')
            loader.add_xpath('price', './/span[@class="s-item__price"]//text()')
            loader.add_xpath('product_url', './/a[@class="s-item__link"]//@href')
            
            yield scrapy.Request(
                card.xpath('.//a[@class="s-item__link"]//@href').get(),
                callback=self.parse_product_details,
                cb_kwargs={'loader': loader}
            )

    def parse_product_details(self, response, loader):
        #content - names
        data11 = response.xpath("//div[@class='ux-layout-section__item ux-layout-section__item--table-view']/div[@class='ux-layout-section__row'][5]/div[@class='ux-labels-values__labels'][2]/div[@class='ux-labels-values__labels-content']/div/span//text()").get()
        loader.add_value('eleven', data11)
        data12 = response.xpath("//div[@class='ux-layout-section__item ux-layout-section__item--table-view']/div[@class='ux-layout-section__row'][6]/div[@class='ux-labels-values__labels'][2]/div[@class='ux-labels-values__labels-content']/div/span//text()").get()
        loader.add_value('twelve', data12)

        #values
        val11 = response.xpath("//div[@class='ux-layout-section__item ux-layout-section__item--table-view']/div[@class='ux-layout-section__row'][5]/div[@class='ux-labels-values__values'][2]/div[@class='ux-labels-values__values-content']/div/span//text()").get()
        loader.add_value('v11', val11)
        val12 = response.xpath("//div[@class='ux-layout-section__item ux-layout-section__item--table-view']/div[@class='ux-layout-section__row'][6]/div[@class='ux-labels-values__values'][2]/div[@class='ux-labels-values__values-content']/div/span//text()").get()
        loader.add_value('v12', val12)

        yield loader.load_item()


process = CrawlerProcess(
    settings={
        'FEED_URI': 'test.jl',
        'FEED_FORMAT': 'jsonlines'
    }
)
process.crawl(EbaySpider)
process.start()

I have also tried formatting the loaders likeso:

data11 = response.xpath("//div[@class='ux-layout-section__item ux-layout-section__item--table-view']/div[@class='ux-layout-section__row'][5]/div[@class='ux-labels-values__labels'][2]/div[@class='ux-labels-values__labels-content']/div/span//text()").get()
data12 = response.xpath("//div[@class='ux-layout-section__item ux-layout-section__item--table-view']/div[@class='ux-layout-section__row'][6]/div[@class='ux-labels-values__labels'][2]/div[@class='ux-labels-values__labels-content']/div/span//text()").get()

loader.add_value('eleven', [data11, data12])

        #values
val11 = response.xpath("//div[@class='ux-layout-section__item ux-layout-section__item--table-view']/div[@class='ux-layout-section__row'][5]/div[@class='ux-labels-values__values'][2]/div[@class='ux-labels-values__values-content']/div/span//text()").get()
val12 = response.xpath("//div[@class='ux-layout-section__item ux-layout-section__item--table-view']/div[@class='ux-layout-section__row'][6]/div[@class='ux-labels-values__values'][2]/div[@class='ux-labels-values__values-content']/div/span//text()").get()

loader.add_value('v11', [val11, val12])

However, I believe this only selects the values from the last response and the first variable in loader.add_value

It seems to work fairly effectively when I do this:

loader.add_value("v11", {"v11":[val11, val12]})
scrapy python
1 Answers

If I understand you question correctly then your xpath is wrong.

import scrapy
from scrapy.item import Field
from itemloaders.processors import TakeFirst
from scrapy.crawler import CrawlerProcess

from scrapy.loader import ItemLoader


class EbayItem(scrapy.Item):
    category = Field(output_processor=TakeFirst())
    name = Field(output_processor=TakeFirst())
    price = Field(output_processor=TakeFirst())
    product_url = Field(output_processor=TakeFirst())
    numeral = Field(output_processor=TakeFirst())
    values = Field(output_processor=TakeFirst())


class EbaySpider(scrapy.Spider):
    name = 'ebay'
    start_urls = {
        'english': 'https://www.ebay.com/sch/i.html?_from=R40&_nkw=pokemon+cards&_sacat=2536&LH_TitleDesc=0&_sop=16&LH_All=1&rt=nc&Language=English&_dcat=183454',
        'japanese':'https://www.ebay.com/sch/i.html?_from=R40&_nkw=pokemon+cards&_sacat=2536&LH_TitleDesc=0&_sop=16&LH_All=1&_oaa=1&rt=nc&Language=Japanese&_dcat=183454'
    }

    def start_requests(self):
        for category, url in self.start_urls.items():
            yield scrapy.Request(
                url=url,
                callback=self.parse,
                cb_kwargs={
                    'category': category
                }
            )

    def parse(self, response, category):
        all_cards = response.xpath('//div[@class="s-item__wrapper clearfix"]')
        for card in all_cards:

            loader = ItemLoader(EbayItem(), selector=card)
            loader.add_value('category', category)
            loader.add_xpath('name', './/h3/text()')
            loader.add_xpath('price', './/span[@class="s-item__price"]//text()')
            loader.add_xpath('product_url', './/a[@class="s-item__link"]//@href')

            yield scrapy.Request(
                card.xpath('.//a[@class="s-item__link"]//@href').get(),
                callback=self.parse_product_details,
                cb_kwargs={'loader': loader}
            )

    def parse_product_details(self, response, loader):
        numeral = ["character", "condition", "score", "graded"]
        values = []

        character = response.xpath('//div[div[div[span[text()="Character:"]]]]/following-sibling::div//span/text()').get(default='Not-found')
        values.append(character)

        condition = response.xpath('//div[div[div[span[text()="Card Condition:"]]]]/following-sibling::div//span/text()').get(default='Unknown')
        values.append(condition)

        grade = response.xpath('//div[div[div[span[text()="Grade:"]]]]/following-sibling::div//span/text()').get(default='Unknown')
        values.append(grade)

        graded = response.xpath('//div[div[div[span[text()="Graded:"]]]]/following-sibling::div//span/text()').get(default='No')
        values.append(graded)

        loader.add_value(None, {'numeral': [numeral], 'values': [values]})
        yield loader.load_item()


process = CrawlerProcess(
    settings={
        'FEED_URI': 'test.jl',
        'FEED_FORMAT': 'jsonlines'
    }
)
process.crawl(EbaySpider)
process.start()

Output:

{'category': 'english',
 'name': 'PSA 10 1st Edition Fossil COMPLETE Set 62/62 Pokemon Cards',
 'numeral': ['character', 'condition', 'score', 'graded'],
 'price': 'GBP 69,420.00',
 'product_url': 'https://www.ebay.com/itm/292969293305?hash=item443654a5f9%3Ag%3ACrcAAOSwlR9cavzM&LH_All=1',
 'values': ['Dragonite', 'Near Mint or better', 'Unknown', 'No']}
[scrapy.core.scraper] DEBUG: Scraped from <200 https://www.ebay.com/itm/175035501592?hash=item28c0edcc18%3Ag%3A39EAAOSwE%7EthnY31&LH_All=1>
{'category': 'japanese',
 'name': 'Beckett Gem Mint No Rarity Venusaur BGS 9.5 1st edition BCCG Pokemon '
         'Card 1996',
 'numeral': ['character', 'condition', 'score', 'graded'],
 'price': 'AU $85,000.00',
 'product_url': 'https://www.ebay.com/itm/175035501592?hash=item28c0edcc18%3Ag%3A39EAAOSwE%7EthnY31&LH_All=1',
 'values': ['Venusaur', 'Unknown', '9.5', 'Yes']}

And so on
user_164297800
Related