What is Scrapy returning the same item instead of all items in the listing?

Kgosi Kekana 0
Viewed 17

I have created a spider which is meant to get me a historic view of lotto numbers. The script is working but it is returning the first item only and then duplicating it.

Here is my code which is returning the first row but then repeating it the number of times there is a row but it is supposed to return each row.

import scrapy


class LottoSpider(scrapy.Spider):
    name = 'lottos'
    start_urls = [
        'https://www.lotteryextreme.com/powerball/results'
    ]


    def parse(self, response):

        for results in response.xpath('//*[contains(concat( " ", @class, " " ), concat( " ", "wyn", " " ))]'):

                date = response.css('span.f20::text').get()
                number1 = response.css('.results2 td::text')[0].extract()
                number2 = response.css('.results2 td::text')[1].extract()
                number3 = response.css('.results2 td::text')[2].extract()
                number4 = response.css('.results2 td::text')[3].extract()
                number5 = response.css('.results2 td::text')[4].extract()
                powerball = response.css('.results2 td::text')[6].extract()


                yield {

                    'date': date,
                    'number1': number1,
                    'number2': number2,
                    'number3': number3,
                    'number4': number4,
                    'number5': number5,
                    'powerball': powerball,

                }
scrapy
1 Answers

In the for loop you need to use results and not response.

import scrapy


class LottoSpider(scrapy.Spider):
    name = 'lottos'
    start_urls = ['https://www.lotteryextreme.com/powerball/results']

    def parse(self, response):
        for results in response.xpath('//*[contains(concat( " ", @class, " " ), concat( " ", "wyn", " " ))]'):
            date = results.css('span.f20::text').get()
            number1 = results.css('.results2 td::text')[0].get()
            number2 = results.css('.results2 td::text')[1].get()
            number3 = results.css('.results2 td::text')[2].get()
            number4 = results.css('.results2 td::text')[3].get()
            number5 = results.css('.results2 td::text')[4].get()
            powerball = results.css('.results2 td::text')[6].get()

            yield {
                'date': date,
                'number1': number1,
                'number2': number2,
                'number3': number3,
                'number4': number4,
                'number5': number5,
                'powerball': powerball,
            }
SuperUser0
Related