Updated: console.log(foodGroupIdsOnPage1) works no matter outside or within the for-loop. So I think the problem is playwright(node package) cannot assess the "id" variable.
I made a data scraper for one page of html which is successful to scrape the data. And the code is as below.
const playwright = require('playwright');
async function initialize() { //nodeJS
const browser = await playwright.chromium.launch({
headless: false
});
const context = await browser.newContext();
const tab1 = await context.newPage();
return { tab1, context };
}
async function main() {
const { tab1, context } = await initialize();
await tab1.goto('https://www.cfs.gov.hk/tc_chi/nutrient/search1.php');
await tab1.waitForFunction(async function () {
const foodGroupIdsOnPage1 = document.querySelector('.tableResponsive').querySelectorAll('td')
const page1_foodGroupButton = foodGroupIdsOnPage1[0].querySelector('a')
page1_foodGroupButton.click();
});
await tab1.waitForTimeout(2000);
await tab1.click('[id^=grp] > a');
await tab1.waitForTimeout(2000);
const ArrayOfTabs = context.pages();
let tab2 = ArrayOfTabs[1];
await tab2.evaluate(function () {
let tableOfAllFoods = []
let rowsOnPage3 = document.querySelector(".colorTable2").querySelectorAll("tr");
for (let row_OnPage3 of rowsOnPage3) {
let arrayNutritionOfOneFood = []
let cellsInOneRow = row_OnPage3.querySelectorAll("td");
for (let cell of cellsInOneRow) {
arrayNutritionOfOneFood.push(cell.innerText)
}
tableOfAllFoods.push(arrayNutritionOfOneFood);
}
console.log(tableOfAllFoods)
});
tab2.close();
tab1.goBack();
}
main();
But then, i want to add a for-loop so that it can become a multi-page scraper. And the code breaks. The variable "id" on line 32 is undefined. And i don't know the reason why it is undefined.
const playwright = require('playwright');
async function initialize() {
const browser = await playwright.chromium.launch({
headless: false
});
const context = await browser.newContext(); ////create a new browser context, which means no cookies and cache saved
const tab1 = await context.newPage();
return { tab1, context };
}
async function main() {
const { tab1, context } = await initialize();
await tab1.goto('https://www.cfs.gov.hk/tc_chi/nutrient/search1.php');
////const page1_foodGroupsLinksArray = document.querySelector('.tableResponsive').querySelectorAll('td');
// Node.js (Playwright) Context
const foodGroupIdsOnPage1 = await tab1.evaluate(async function getFoodGroupsOnPage1() {
return [...document.querySelector('.tableResponsive').querySelectorAll('td ')].map(e => e.id);
})
// Node.js context
for (let id of foodGroupIdsOnPage1) {
await tab1.waitForFunction(async function GotoPage2() { ////wait for function>>>only run the next command after all the commands inside the next bracket run
// Browser Context
// for (let page1_foodGroupLink of page1_foodGroupsLinksArray) {
const page1_foodGroupButton = document.querySelector(id).querySelector('a') ////beginning of the for loop
await page1_foodGroupButton.click();
});
////console.log(id);
await tab1.waitForTimeout(2000);
await tab1.click('[id^=grp] > a');
await tab1.waitForTimeout(2000);
const ArrayOfTabs = context.pages(); ////get how many tabs chromium are
let tab2 = ArrayOfTabs[1]; ////make active tab to the second tab
await tab2.evaluate(function extractFoodGroupData() {
let tableOfAllFoods = [];
let rowsOnPage3 = document.querySelector(".colorTable2").querySelectorAll("tr");
for (let row_OnPage3 of rowsOnPage3) {
let arrayNutritionOfOneFood = [];
let cellsInOneRow = row_OnPage3.querySelectorAll("td");
for (let cell of cellsInOneRow) {
arrayNutritionOfOneFood.push(cell.innerText);
}
////arrayNutritionOfOneFood.push(group_id);
////group_id=group_id+1;
tableOfAllFoods.push(arrayNutritionOfOneFood);
}
console.log(tableOfAllFoods);
});
tab2.close();
tab1.goBack();
////}
////},[id])
}
};
main();
Would anyone please tell me how to fix and the reason why it breaks. Thank you for your help in advance!