Python: Scrapy-Splash recursive crawl using CrawlSpider not working

Viewed 541

I have integrated scrapy-splash in my CrawlSpider and it only crawl renders the start_urls. Wondering how to have scrapy-splash crawl the internal links. I've been searching through the internet for a solution and it doesn't seem like there has been one that works.

The following is my code:

import scrapy
from scrapy.selector import Selector
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import Rule, CrawlSpider

from scrapy.item import Item, Field
from scrapy import Request

#import requests
from scrapy_splash import SplashRequest


class Website(scrapy.Item):
    url = Field()
    response = Field()

class houzzspider(CrawlSpider):
    handle_httpstatus_list = [404, 500]
    name = "example"
    allowed_domains = ["localhost","www.example.com"]
    start_urls = ["https://www.example.com/"]    

    rules = (
    Rule(
        LinkExtractor(
            allow=(),
            deny=(),process_value=''),
            callback="parse_items",
            process_links="process_links",
            follow=True,
            ),
    Rule(
        LinkExtractor(
            allow=(),
            deny=(),process_value=''),
            follow=True,
            ),
    )

    def process_links(self, links):
        for link in links:
            if "http://localhost:8050/render.html?&" not in link.url:
                link.url = "http://localhost:8050/render.html?&" + urlencode({'url':link.url,'wait':2.0})
        return links   

    def start_requests(self):
        for url in self.start_urls:
            yield SplashRequest(url, self.parse_items,
                endpoint='render.html',
                args={'wait': 0.5},)    

    def parse_items(self, response):
        hxs = Selector(response)
        sites = response.selector.xpath('//html')
        items = []

        for site in sites:
            #print site
            item = Website()     
            item['url'] = response.url
            item['response'] = response.status
            items.append(item)
            return items      
0 Answers
Related