Websites using scrapy-playwright and only playwright work differently

Viewed 538

I am trying to log into a webpage using scrapy-playwright, as I want the nice integration with scrapy. I can't log in using scrapy-playwright, as it redirects to a page that does not exist. I have also tried doing a post request instead of clicking, that doesn't work either.

However, if I try the same thing using only Playwright, it works perfectly... Is there a difference between websites opened with scrapy-playwright compared to only Playwright? And does anyone know how to solve this using scrapy-playwright?

scrapy-playwright code:

def start_requests(self):
    yield scrapy.Request(
        url = self.url,
        meta = dict(
            playwright = True,
            playwright_include_page = True,
            playwright_page_methods = [PageMethod('wait_for_selector', 'a[data-toggle=dropdown]')],
                ),
        callback = self.sign_in,
        )

async def sign_in(self, response):
    page = response.meta['playwright_page']
    while await page.is_visible("button[class='close close-news']"):
        await page.click("button[class='close close-news']")
    await page.click('button#declineAllConsentSummary')
    await page.click('div.my-account-sub > a[data-toggle=dropdown]', timeout=10000)
    await page.fill('input#j_username_header', os.getenv(self.usernameKey), timeout=10000)
    await page.fill('input#j_password_header', os.getenv(self.passwordKey), timeout=10000)
    await page.click('button#responsiveMyAccLoginGA')

Playwright code:

async def test_async_playwright(self):
    async with async_playwright() as playwright:
        browser = await playwright.chromium.launch(headless=False)
        context = await browser.new_context(base_url=self.url)
        page = await context.new_page()
        
        await page.goto(self.url, wait_until='commit')
        while await page.is_visible("button[class='close close-news']"):
            await page.click("button[class='close close-news']")
        await page.click('button#declineAllConsentSummary')
        await page.wait_for_selector('a[data-toggle=dropdown]')
        await page.click('div.my-account-sub > a[data-toggle=dropdown]', timeout=5000)
        await page.fill('input#j_username_header', os.getenv(self.usernameKey), timeout=5000)
        await page.fill('input#j_password_header', os.getenv(self.passwordKey), timeout=5000)
        await page.click('button#responsiveMyAccLoginGA')
0 Answers
Related