Scrapy only returning a single item from list

user_15675231 0
Viewed 84

I believe that I have my xpaths coded in the incorrect way, as I only get a single result for each url. Whereas, there are in total 25 job posts for each url (not included those in the next page.) How can I correct my xpaths to get all the results?

Here's my scraper:

from scrapy.item import Field
import scrapy
from scrapy.loader import ItemLoader
from scrapy.crawler import CrawlerProcess
from itemloaders.processors import TakeFirst
import pandas as pd
from collections import defaultdict

class CvItem(scrapy.Item):
    category = Field(output_processor = TakeFirst())
    salary = Field(output_processor = TakeFirst())
    title =  Field(output_processor = TakeFirst())
    organisation = Field(output_processor = TakeFirst())

class CvSpider(scrapy.Spider):
    name = 'cv'
    start_urls = {'Accountancy_finance': ['https://www.cv-library.co.uk/Degree-Finance-jobs?us=1',
              'https://www.cv-library.co.uk/Degree-Accounting-jobs?us=1'],
             'Aeronautical_Engineering': ['https://www.cv-library.co.uk/Degree-Aeronautical-Engineering-jobs?us=1'],
             'Manufacturing_Engineering': ['https://www.cv-library.co.uk/Degree-Manufacturing-Engineering-jobs?us=1'],
             'Agriculture_and_Forestry': ['https://www.cv-library.co.uk/Degree-Forestry-jobs?us=1']}

    def start_requests(self):
        for items, urls in self.start_urls.items():
            for url in urls:
                yield scrapy.Request(
                url = url, 
                callback = self.parse, 
                cb_kwargs = {
                    'items':items
                }
            )
    def parse(self, response, items):
        container = response.xpath('//ol[@id="searchResults"]')
        for lists in container:
            loader = ItemLoader(CvItem(), selector = lists)
            loader.add_value('category', items)
            loader.add_xpath('title', '//article[@id]//a[@title]/@title')
            loader.add_xpath('salary', '//article[@id]//dl//dd[@class="job__details-value salary"]//text()')
            loader.add_xpath('organisation', '//article[@id]/div//div/p/a//text()')
            yield loader.load_item()

There was a slight mistake with the requests that I updated for those of you that had checked for the first 15minutes since I uploaded it.

scrapy python
1 Answers

The problem was in the container's xpath. You only get the container without actually the items in it so you only loop once on the container itself and not the actual items you want to scrape.

from scrapy.item import Field
import scrapy
from scrapy.loader import ItemLoader
from scrapy.crawler import CrawlerProcess
from itemloaders.processors import TakeFirst
import pandas as pd
from collections import defaultdict


class CvItem(scrapy.Item):
    category = Field(output_processor=TakeFirst())
    salary = Field(output_processor=TakeFirst())
    title = Field(output_processor=TakeFirst())
    organisation = Field(output_processor=TakeFirst())


class CvSpider(scrapy.Spider):
    name = 'cv'
    start_urls = {'Accountancy_finance': ['https://www.cv-library.co.uk/Degree-Finance-jobs?us=1',
                                          'https://www.cv-library.co.uk/Degree-Accounting-jobs?us=1'],
                  'Aeronautical_Engineering': ['https://www.cv-library.co.uk/Degree-Aeronautical-Engineering-jobs?us=1'],
                  'Manufacturing_Engineering': ['https://www.cv-library.co.uk/Degree-Manufacturing-Engineering-jobs?us=1'],
                  'Agriculture_and_Forestry': ['https://www.cv-library.co.uk/Degree-Forestry-jobs?us=1']}

    def start_requests(self):
        for items, urls in self.start_urls.items():
            for url in urls:
                yield scrapy.Request(
                    url=url,
                    cb_kwargs={
                        'items': items
                    }
                )

    def parse(self, response, items):
        container = response.xpath('//ol[@id="searchResults"]//li[@class="results__item"]')
        for lists in container:
            loader = ItemLoader(CvItem(), selector=lists)
            loader.add_value('category', items)
            loader.add_xpath('title', '//article[@id]//a[@title]/@title')
            loader.add_xpath('salary', '//article[@id]//dl//dd[@class="job__details-value salary"]//text()')
            loader.add_xpath('organisation', '//article[@id]/div//div/p/a//text()')
            yield loader.load_item()
user_164297800
Related