Not returning any results even though I scraped a different section of the domain without an issue

TAG_You're_It 0
Viewed 22

I used basically this same code on lottery.net/powerball/numbers/#year no problem. Why is it not working this time? I have changed all the info I need to do as in the links and the XPath differences.

import scrapy


class MegaMillionsDrawingsSpider(scrapy.Spider):
        name = 'mega_millions_drawings'
        allowed_domains = ['www.lottery.net']
        user_agent = # my user agent
  

def start_request(self):
    start_urls = []
    for i in reversed(range(1996,2023)):
        current_url = 'http://www.lottery.net/mega-millions/numbers/'+ str(i)
        start_urls.append(current_url)
        
    for url in start_urls:
        yield scrapy.Request(    
            url=url, 
            callback=self.parse,
            headers={
                'User-Agent': self.user_agent
            }
        )

def parse(self, response):
    from scrapy.shell import inspect_response
    inspect_response(response, self)

    for drawing in response.xpath("//table[@class='prizes archive ']/tbody/tr"):
        yield {
            'date': drawing.xpath(".//td/a/text()[2]").get(),
            #'url': response.urljoin(drawing.xpath(".//")).get(),
            'first': drawing.xpath(".//td/ul[@class='multi results mega-millions']/li[@class='ball'][position() = 1]/text()").get(),
            'second': drawing.xpath(".//td/ul[@class='multi results mega-millions']/li[@class='ball'][position() = 2]/text()").get(),
            'third': drawing.xpath(".//td/ul[@class='multi results mega-millions']/li[@class='ball'][position() = 3]/text()").get(),
            'fourth': drawing.xpath(".//td/ul[@class='multi results mega-millions']/li[@class='ball'][position() = 4]/text()").get(),
            'fifth': drawing.xpath(".//td/ul[@class='multi results mega-millions']/li[@class='ball'][position() = 5]/text()").get(),
            'mega-ball': drawing.xpath(".//td/ul[@class='multi results mega-millions']/li[@class='mega-ball']/text()").get()
        }
scrapy
1 Answers

There are a few issues that I can see. Some of your xpath expressions are off, the indentation is way off, you are using http instead of https.

Using the slight modifications to the formatting and your instance methods I made in the example below will fix the issues.

import scrapy

class MegaMillionsDrawingsSpider(scrapy.Spider):
    name = 'mega-millions-drawings'
    allowed_domains = ['lottery.net']

    def start_requests(self):
        for num in reversed(range(1996, 2023)):  # only need one loop
            url = "https://www.lottery.net/mega-millions/numbers/" + str(num)
            yield scrapy.Request(url)  

    def parse(self, response):
        for drawing in response.xpath("//table[@class='prizes archive ']/tbody/tr"):
            item = {"date" : drawing.xpath(".//td/a/text()[2]").get().strip()}
            for number, text in zip(    # use a loop to gather all the numbers
                ["first", "second", "third", "fourth", "fifth"],
                drawing.xpath(".//td/ul/li[@class='ball']/text()").getall()
            ):
                item[number] = text.strip()  # remove whitespace
            item["mega-ball"] = drawing.xpath(".//td/ul/li[@class='mega-ball']/text()").getall().strip()
            yield item
userphOc10
Related