How to pass Selenium WebDriver response to Scrapy parse method?

user_12844566 0
Viewed 774

I have two problems to solve:

First: correctly locate an element on a website using driver. Second: pass the link generated as a result of this action to parse method or LinkExtractor.

ad. 1.

I need to locate the "Load more" button and click it to crawl the resulting page.

<div class="col-sm-4 col-sm-offset-4 col-md-2 col-md-offset-5 col-xs-12 col-xs-offset-0">
            <button class="btn btn-secondary">Load more</button>
        </div>

ad.2.

I have defined LinkExtractor rules which work correctly on static websites and a parse method. I've seen many other examples in similar questions, but I cannot figure out, how shall I stick it together?

This is my last try:

from scrapy.spiders import CrawlSpider, Rule
from scrapy.linkextractors import LinkExtractor
from os.path import join as path_join
import json
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
import time


class MySpider(CrawlSpider):
    input_dir = '../input'
    encoding = 'utf8'
    passing_file = path_join(input_dir, 'site_to_scrap.txt')
    with open(passing_file, 'r', encoding = encoding) as f:
        input_file = str(f.readlines()[0]).replace('\n', '')
        f.close()

    with open(path_join(input_dir, input_file + '.json'), 'r') as json_file:
        data = json.load(json_file)
        json_file.close()
    name = data['name']
    allowed_domains = data['allowed_domains']
    start_urls = data['start_urls']

    rules = (

        Rule(LinkExtractor(allow=data['allow'], deny=data['deny'],
                           deny_domains=data['deny_domains'],
                           restrict_text=data['restrict_text']),
             callback='parse_page', follow=True),

    )

    driver = webdriver.Chrome(ChromeDriverManager().install())

    def parse_page(self, response):
        self.driver.get(response.url)
        self.driver.implicitly_wait(2)
        self.driver.find_element_by_class_name('btn btn-secondary').click()
        time.sleep(2)

        for p in response.css('p'):
            yield {
                'p': p.css('p::text').get(),
            }
        for div in response.css('div'):
            yield {
                'div': div.css('div::text').get(),
            }
        for title in response.css('div.title'):
            yield {
                'header': title.css('div.title::text').get(),
            }
        for head in response.css('head'):
            yield {
                'header': head.css('head::text').get(),
            }

        # write visited urls into a file
        output_dir = './output'
        file = 'visited_urls.txt'
        with open(path_join(output_dir, file), 'a') as outfile:
            print(response.url, file=outfile)
            outfile.close()

I've removed significant part of parse_page method for readability (it doesn't affect the execution).

Another example that I tried out without success:

    driver = webdriver.Chrome(ChromeDriverManager().install())

    def parse_url(self, response):
        self.driver.get(response.url)

        while True:
            self.driver.implicitly_wait(2)
            next = self.driver.find_element_by_class_name('btn btn-secondary')

            try:
                next.click()
                time.sleep(2)
                # get the data and write it to scrapy items
            except:
                break

        self.driver.close()

Any hint on one of the two questions would be much appreciated.

scrapy selenium python
1 Answers

You could implement this as a middleware which makes requests through Selenium and then returns a HtmlResponse.
For that purpose you should first create a sub class of Request.

from scrapy import Request

class SeleniumRequest(Request):
    pass

This sub class only serves the purpose of helping the middleware to know if Selenium should be used.
When you implement your own middleware, make sure that you call quit on the driver when the spider is closing. 

class SeleniumMiddleware:
    def __init__(self):
        self.driver = webdriver.Chrome(ChromeDriverManager().install())

    @classmethod
    def from_crawler(cls, crawler):
        middleware = cls()
        crawler.signals.connect(middleware.spider_closed, signals.spider_closed)
        return middleware

    def process_request(self, request, spider):
        if not isinstance(request, SeleniumRequest):
            return None
        self.driver.get(request.url)

        # loop to click "load more" button on JS-page
        while True:
            try:
                button = WebDriverWait(self.driver, 10).until(
                    EC.presence_of_element_located((By.CLASS_NAME, 'btn btn-secondary'))
                )
                self.driver.execute_script("arguments[0].click();", button)
            except:
                break

        return HtmlResponse(
            self.driver.current_url,
            body=str.encode(self.driver.page_source),
            encoding='utf-8',
            request=request
        )

    def spider_closed(self):
        self.driver.quit()

Then enable it in your settings.

DOWNLOADER_MIDDLEWARES = {
    'your_project.middleware_location.SeleniumMiddleware': 500
}

You should now be able to yield SeleniumRequest in your Spider's start_requests method. 

def start_requests(self):
    yield SeleniumRequest(your_url, your_callback)

Now all your requests should go through Selenium and return HtmlResponses which should work with your LinkExtractors and certainly can be passed to your parse method. Also I find this github repository to be most useful when I do stuff like this.

user_102035970
Related