Scrapy Splash and 504 timeouts

Question

I use Scrapy with Splash on windows 10 using VirtualBox and Docker. I've written about half a dozen spiders which have worked but the latest two I've written for different sites are returning a lot of 504 gateway timeouts.

The Splash screen displays properly at 192.168.99.100:8050 and some urls from the text file process correctly if I leave the script running but they are few and far between. Any ideas how I can solve this?

#coding=utf-8
import scrapy
import datetime
import urlparse
import os.path
import socket
import urllib2
import collections
import re
import io

from scrapy.loader.processors import MapCompose, Join
from scrapy.loader import ItemLoader
from scrapy.utils.markup import remove_tags
from scrapy.http import Request

from properties.items import PropertiesItem
from properties.items import ImageItem
from scrapy_splash import SplashRequest
from time import sleep
from xml.etree import ElementTree as ET

class MySpider(scrapy.Spider):
    name = 'myspider'
    allowed_domains = ["https://www.example.com"]
    start_urls = []
    root_url = 'https://www.example.com'

    products = []
    headings = [] #csv table headers
    product_id = 1

    #custom pipelines
    custom_settings = {
        'ITEM_PIPELINES': {
            'properties.pipelines.MyImagesPipeline': 0,
        #'properties.pipelines.PropertiesPipeline': 0 #comment out if use yield itemloader
        }
    }

    def __init__(self, filename=None):
        with open('/home/vagrant/properties/properties/vagrant_syslink/urls.txt') as f:
             self.start_urls = [url.strip() for url in f.readlines()]

    def start_requests(self):
        for url in self.start_urls:
            yield SplashRequest(url, self.parse, 
                endpoint='render.html',
                args={'wait': 15}, #, 'lua_source': self.script},
                headers={'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'},
        )

    def parse(self, response):
        ...
        ...

Thanks

Scrapy Splash and 504 timeouts

0 Answers