I am trying to log into a webpage using scrapy-playwright, as I want the nice integration with scrapy. I can't log in using scrapy-playwright, as it redirects to a page that does not exist. I have also tried doing a post request instead of clicking, that doesn't work either.
However, if I try the same thing using only Playwright, it works perfectly... Is there a difference between websites opened with scrapy-playwright compared to only Playwright? And does anyone know how to solve this using scrapy-playwright?
scrapy-playwright code:
def start_requests(self):
yield scrapy.Request(
url = self.url,
meta = dict(
playwright = True,
playwright_include_page = True,
playwright_page_methods = [PageMethod('wait_for_selector', 'a[data-toggle=dropdown]')],
),
callback = self.sign_in,
)
async def sign_in(self, response):
page = response.meta['playwright_page']
while await page.is_visible("button[class='close close-news']"):
await page.click("button[class='close close-news']")
await page.click('button#declineAllConsentSummary')
await page.click('div.my-account-sub > a[data-toggle=dropdown]', timeout=10000)
await page.fill('input#j_username_header', os.getenv(self.usernameKey), timeout=10000)
await page.fill('input#j_password_header', os.getenv(self.passwordKey), timeout=10000)
await page.click('button#responsiveMyAccLoginGA')
Playwright code:
async def test_async_playwright(self):
async with async_playwright() as playwright:
browser = await playwright.chromium.launch(headless=False)
context = await browser.new_context(base_url=self.url)
page = await context.new_page()
await page.goto(self.url, wait_until='commit')
while await page.is_visible("button[class='close close-news']"):
await page.click("button[class='close close-news']")
await page.click('button#declineAllConsentSummary')
await page.wait_for_selector('a[data-toggle=dropdown]')
await page.click('div.my-account-sub > a[data-toggle=dropdown]', timeout=5000)
await page.fill('input#j_username_header', os.getenv(self.usernameKey), timeout=5000)
await page.fill('input#j_password_header', os.getenv(self.passwordKey), timeout=5000)
await page.click('button#responsiveMyAccLoginGA')