How to I marry Scrapy and Selenium to scroll down this webpage and scrape all the text?

Viewed 146

So the reason I'm using both is because I have other functions written in my Scrapy class already but I'm left with scrolling (hence Selenium) and scraping all the text in this webpage

import scrapy
from scrapy.selector import Selector
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from shutil import which
import time

class ActSpider(scrapy.Spider):
    name = 'ACT'
    allowed_domains = ['sso.agc.gov.sg']
    start_urls = [
        'https://sso.agc.gov.sg/Act/AA2004'
    ]

    def __init__(self):
        chrome_options = Options()
        chrome_options.add_argument('--headless')

        chrome_path = which('chromedriver')

        driver =  webdriver.Chrome(executable_path=chrome_path,  options=chrome_options)
        driver.set_window_size(1920, 1080)
        driver.get("https://sso.agc.gov.sg/Act/AA2004")

        #insert scroller function?

        self.html = driver.page_source 
        driver.close()

    def parse(self, response):
        resp = Selector(text=self.html)
        for table in resp.xpath("//div[@class='body']"):
            yield {
                'info': table.xpath('.//table/text()').getall()
            }

I have found a number of infinite scrollers with Selenium but am unsure how to deploy it in the scrapy class to work.

0 Answers
Related