Python scrapy working (only half of the time)

Question

I created a python scrapy project to extract the prices of some google flights.

I configured the middleware to use PhantomJS instead of a normal browser.

class JSMiddleware(object):
    def process_request(self, request, spider):
        driver = webdriver.PhantomJS()
        try:
            driver.get(request.url)
            time.sleep(1.5)
        except e:
            raise ValueError("request url failed - 
 url: {},
 error:
                  {}").format(request.url, e)
        body = driver.page_source
        #encoding='utf-8' - add to html response if necessary
        return HtmlResponse(driver.current_url, body=body,encoding='utf-8', 
               request=request)

In the settings.py i added:

DOWNLOADER_MIDDLEWARES = {
# key path intermediate class, order value of middleware
'scraper_module.middlewares.middleware.JSMiddleware' : 543 ,
# prohibit the built-in middleware
'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware' : None ,  } `

I also created the following spider class:

import scrapy
from scrapy import Selector

class Gspider(scrapy.Spider):
    name = "google_spider"

    def __init__(self):
        self.start_urls = ["https://www.google.pt/flights/#search;f=LIS;t=POR;d=2017-06-18;r=2017-06-22"]
        self.prices = []
        self.links = []

    def clean_price(self, part):
        #part received as a list
        #the encoding is utf-8
        part = part[0]
        part = part.encode('utf-8')
        part = filter(str.isdigit, part)
        return part

    def clean_link(self, part):
        part = part[0]
        part = part.encode('utf-8')
        return part

    def get_part(self, var_holder, response, marker, inner_marker, amount = 1):
        selector = Selector(response)
        divs = selector.css(marker)
        for n, div in enumerate(divs):
            if n < amount:
                part = div.css(inner_marker).extract()
                if inner_marker == '::text':
                    part = self.clean_price(part)
                else:
                    part = self.clean_link(part)
                var_holder.append(part)
            else:
                break
        return var_holder

    def parse(self, response):
        prices, links = [], []
        prices = self.get_part(prices, response, 'div.OMOBOQD-d-Ab', '::text')
        print prices
        links = self.get_part(links, response, 'a.OMOBOQD-d-X', 'a::attr(href)')
        print links

The problem is, I run the code in the shell, and around half of the times I successfully get the prices and links requested, but another half of the time, the final vectors which should contain the extracted data, are empty.

I do not get any errors during execution.

Does anyone have any idea about why this is happening? here are the logs from the command line:

Python scrapy working (only half of the time)

1 Answers