Playwright cannot assess the returned defined variable

Viewed 1110

Updated: console.log(foodGroupIdsOnPage1) works no matter outside or within the for-loop. So I think the problem is playwright(node package) cannot assess the "id" variable.

I made a data scraper for one page of html which is successful to scrape the data. And the code is as below.

const playwright = require('playwright');

async function initialize() {                               //nodeJS

  const browser = await playwright.chromium.launch({
    headless: false
  });
  const context = await browser.newContext();

  const tab1 = await context.newPage();
  return { tab1, context };
}

async function main() {

  const { tab1, context } = await initialize();

  await tab1.goto('https://www.cfs.gov.hk/tc_chi/nutrient/search1.php');

  await tab1.waitForFunction(async function () {

    const foodGroupIdsOnPage1 = document.querySelector('.tableResponsive').querySelectorAll('td')

    const page1_foodGroupButton = foodGroupIdsOnPage1[0].querySelector('a')
    page1_foodGroupButton.click();

  });

  await tab1.waitForTimeout(2000);
  await tab1.click('[id^=grp] > a');

  await tab1.waitForTimeout(2000);
  const ArrayOfTabs = context.pages();


  let tab2 = ArrayOfTabs[1];
  await tab2.evaluate(function () {
    let tableOfAllFoods = []
    let rowsOnPage3 = document.querySelector(".colorTable2").querySelectorAll("tr");
    for (let row_OnPage3 of rowsOnPage3) {
      let arrayNutritionOfOneFood = []
      let cellsInOneRow = row_OnPage3.querySelectorAll("td");
      for (let cell of cellsInOneRow) {
        arrayNutritionOfOneFood.push(cell.innerText)
      }
      tableOfAllFoods.push(arrayNutritionOfOneFood);
    }
    console.log(tableOfAllFoods)
  });
  tab2.close();
  tab1.goBack();

}

main();

But then, i want to add a for-loop so that it can become a multi-page scraper. And the code breaks. The variable "id" on line 32 is undefined. And i don't know the reason why it is undefined.

const playwright = require('playwright');

async function initialize() {
    const browser = await playwright.chromium.launch({
        headless: false
    });
    const context = await browser.newContext();                        ////create a new browser context, which means no cookies and cache saved
    const tab1 = await context.newPage();
    return { tab1, context };
}


async function main() {

    const { tab1, context } = await initialize();

    await tab1.goto('https://www.cfs.gov.hk/tc_chi/nutrient/search1.php');

    ////const page1_foodGroupsLinksArray = document.querySelector('.tableResponsive').querySelectorAll('td');


    // Node.js (Playwright) Context


    const foodGroupIdsOnPage1 = await tab1.evaluate(async function getFoodGroupsOnPage1() {

        return [...document.querySelector('.tableResponsive').querySelectorAll('td ')].map(e => e.id);
    })


    // Node.js context
    for (let id of foodGroupIdsOnPage1) {

        await tab1.waitForFunction(async function GotoPage2() {     ////wait for function>>>only run the next command after all the commands inside the next bracket run
            // Browser Context

            // for (let page1_foodGroupLink of page1_foodGroupsLinksArray) {

            const page1_foodGroupButton = document.querySelector(id).querySelector('a')         ////beginning of the for loop
            await page1_foodGroupButton.click();
        });

        ////console.log(id);
        await tab1.waitForTimeout(2000);
        await tab1.click('[id^=grp] > a');

        await tab1.waitForTimeout(2000);
        const ArrayOfTabs = context.pages();   ////get how many tabs chromium are

        let tab2 = ArrayOfTabs[1];              ////make active tab to the second tab
        await tab2.evaluate(function extractFoodGroupData() {
            let tableOfAllFoods = [];
            let rowsOnPage3 = document.querySelector(".colorTable2").querySelectorAll("tr");
            for (let row_OnPage3 of rowsOnPage3) {
                let arrayNutritionOfOneFood = [];
                let cellsInOneRow = row_OnPage3.querySelectorAll("td");
                for (let cell of cellsInOneRow) {
                    arrayNutritionOfOneFood.push(cell.innerText);
                }
                ////arrayNutritionOfOneFood.push(group_id);
                ////group_id=group_id+1;
                tableOfAllFoods.push(arrayNutritionOfOneFood);
            }
            console.log(tableOfAllFoods);
        });
        tab2.close();
        tab1.goBack();
        ////}
        ////},[id])

    }

};


main();

Would anyone please tell me how to fix and the reason why it breaks. Thank you for your help in advance!

2 Answers

It is possible that foodGroupIdsOnPage1 is undefined. Could you add a console.log(foodGroupIdsOnPage1) before the for..of loop and see if you're actually getting any results

Updated: console.log(foodGroupIdsOnPage1) works no matter outside or within the for-loop. So I think the problem is playwright(node package) cannot assess the id variable.

Related