scrapy crawlspider with splash does not execute process_request

Viewed 88

I have a scrapy's crawlspider with splash in order to crawl alibaba site for learning purposes. But my spider does not execute the process_request function in rules in order to execute the splash request. As a result is that the crawler extracts only the urls that does not require javascript. What am I doing wronk? My code is below

import logging

import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from scrapy import signals
from scrapy.crawler import CrawlerRunner
from scrapy.item import Item, Field
from scrapy.signalmanager import dispatcher
from scrapy_splash import SplashRequest
from urllib.parse import urlencode
from crochet import setup, wait_for

setup()
# logging.getLogger('scrapy').propagate = False

class GooglePatentsSpider(CrawlSpider):
    name = "alibaba_spider"
    number_of_suppliers = ''

    script = """
        function main(splash)
            local num_scrolls = 10
            local scroll_delay = 1.0

            local scroll_to = splash:jsfunc("window.scrollTo")
            local get_body_height = splash:jsfunc(
                "function() {return document.body.scrollHeight;}"
            )
            assert(splash:go(splash.args.url))
            splash:wait(splash.args.wait)

            for _ = 1, num_scrolls do
                scroll_to(0, get_body_height())
                splash:wait(scroll_delay)
            end        
            return splash:html()
        end
    """

    rules = (
        Rule(LinkExtractor(deny=''), callback='parse_start_url', follow=True, process_request='splash_request'),
        Rule(LinkExtractor(restrict_xpaths='//h2[@class="elements-title-normal__outter"]/a'), callback='parse_item', follow=True),
    )

    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        spider = super(GooglePatentsSpider, cls).from_crawler(crawler, *args, **kwargs)
        crawler.signals.connect(spider.item_scraped, signal=signals.item_scraped)
        return spider

    def item_scraped(self, item):
        return item

    def splash_request(self, request):
        print('splash request')
        return SplashRequest(
            request.url,
            callback=request.callback,
            endpoint='execute',
            args={
                'wait': 3,
                'lua_source': self.script
            },
        )

    def parse_start_url(self, response):
        print('start url')
        print(response.url)

    def parse_item(self, response):
       print('url')
       print(response.url)

@wait_for(timeout=5000.0)
async def run_spider():
    """Returns all the scraped items of the provided publication number"""
    results = []

    def crawler_results(signal, sender, item, response, spider):
        results.append(item)

    dispatcher.connect(crawler_results, signal=signals.item_scraped)
    runner = CrawlerRunner(settings={
        'BOT_NAME': 'web_page_crawler',
        'USER_AGENT': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36',
        'ROBOTSTXT_OBEY': False,
        'FEED_EXPORT_ENCODING' : 'utf-8',
        # 'DOWNLOAD_DELAY' : 3,
        'SPLASH_URL': 'http://172.17.0.2:8050',
        'DOWNLOADER_MIDDLEWARES': {
            'scrapy_splash.SplashCookiesMiddleware': 723,
            'scrapy_splash.SplashMiddleware': 725,
            'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810,
        },
        'SPIDER_MIDDLEWARES': {
            'scrapy_splash.SplashDeduplicateArgsMiddleware': 100,
        },
        'DUPEFILTER_CLASS': 'scrapy_splash.SplashAwareDupeFilter',
        'HTTPCACHE_STORAGE': 'scrapy_splash.SplashAwareFSCacheStorage'
    })
    await runner.crawl(GooglePatentsSpider, start_urls=[f'https://www.alibaba.com/trade/search?fsb=y&IndexArea=product_en&CatId=&SearchText=vitamin+c'])
    if results:
        return results
    else:
        return 'No suppliers'

r = run_spider()
print(r)
0 Answers
Related