Waiting for element to be visible

Question

I'm practicing with a playwright and scrapy integration towards clicking on a selector with a hidden selector. The aim is to click the selector and wait for the other two hidden selectors to load, then click on one of these and then move on. However, I'm getting the following error:

waiting for selector "option[value='type-2']"
  selector resolved to hidden 
attempting click action
  waiting for element to be visible, enabled and stable
    element is not visible - waiting...

I think the issue is when the selector is clicked, it disappears for some reason. I have implemented a wait on the selector, but the issue still persists.

from scrapy.crawler import CrawlerProcess
import scrapy
from scrapy_playwright.page import PageCoroutine

class JobSpider(scrapy.Spider):

    name = 'job_play'
    custom_settings = {
        'USER_AGENT':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.2 Safari/605.1.15',
    }
    def start_requests(self):
        yield scrapy.Request(
                url = 'https://www.student-loan-calculator.co.uk/',
                callback = self.parse,
                meta= dict(
                        playwright = True,
                        playwright_include_page = True,
                        playwright_page_coroutines = [
                        PageCoroutine("fill", "#salary", '28000'),
                        PageCoroutine("fill", "#debt", '25000'),
                        PageCoroutine("click", selector="//select[@id='loan-type']"),
                        PageCoroutine('wait_for_selector', "//select[@id='loan-type']"),
                        PageCoroutine('click', selector = "//select[@id='loan-type']/option[2]"),
                        PageCoroutine('wait_for_selector', "//div[@class='form-row calculate-button-row']"),
                        PageCoroutine('click', selector = "//button[@class='btn btn-primary calculate-button']"),
                        PageCoroutine('wait_for_selector', "//div[@class='container results-table-container']"),
                        PageCoroutine("wait_for_timeout", 5000),
                                ]
                            ),
                )
    def parse(self, response):
        container = response.xpath("//div[@class='container results-table-container']")
        for something in container:
            yield {
                'some':something
            }
if __name__ == "__main__":
    process = CrawlerProcess(
        settings={
            "TWISTED_REACTOR": "twisted.internet.asyncioreactor.AsyncioSelectorReactor",
            "DOWNLOAD_HANDLERS": {
                "https": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
                "http": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
            },
            "CONCURRENT_REQUESTS": 32,
            "FEED_URI":'loans.jl',
            "FEED_FORMAT":'jsonlines',
        }
    )
    process.crawl(JobSpider)
    process.start()

Waiting for element to be visible

0 Answers