So the reason I'm using both is because I have other functions written in my Scrapy class already but I'm left with scrolling (hence Selenium) and scraping all the text in this webpage
import scrapy
from scrapy.selector import Selector
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from shutil import which
import time
class ActSpider(scrapy.Spider):
name = 'ACT'
allowed_domains = ['sso.agc.gov.sg']
start_urls = [
'https://sso.agc.gov.sg/Act/AA2004'
]
def __init__(self):
chrome_options = Options()
chrome_options.add_argument('--headless')
chrome_path = which('chromedriver')
driver = webdriver.Chrome(executable_path=chrome_path, options=chrome_options)
driver.set_window_size(1920, 1080)
driver.get("https://sso.agc.gov.sg/Act/AA2004")
#insert scroller function?
self.html = driver.page_source
driver.close()
def parse(self, response):
resp = Selector(text=self.html)
for table in resp.xpath("//div[@class='body']"):
yield {
'info': table.xpath('.//table/text()').getall()
}
I have found a number of infinite scrollers with Selenium but am unsure how to deploy it in the scrapy class to work.