'page.evaluate()' inside a loop (in Puppeteer) returning same value

Viewed 920

Overview: This is not the actual problem but I have simplified the problem and converted it into a more understandable one. I am getting same data returned from a page.evaluate() placed inside a for...of loop during every iteration of the loop. I guess it is due to closure, I haven't programmed in JS for long, so I could not implement IIFE with async functions. Also, I don't think we can implement forEach instead of for...if with async functions. I apologize in advance if the question posted is too long, but I want to make it clear.

enter image description here Description: I am trying to scrape a webpage having grocery data. Types of grocery (like vegetables, fruit, dairy, bakery) can be chosen from a select element (dropdown). The grocery list gets displayed depending on type as the option is selected.

  • grocery-choice is the id of select tag that lets us choose options (i.e. select#grocery-choice)
  • grocery-item is the class of the div that contains data for each individual item of the grocery.(i.e. div.grocery-item) For example: For grocery category fruits it contains data on bananas, apples, etc. Each div has an <h4> and two <p> elements for the name, price, and availability respectively.

I am looping over items in the array ([ 'fruit', 'vegetable', 'bakery']) then add each item as JSON into array grocery_info and return grocery_inf from page.evaluate(). grocery_all is the variable where the returned value is assigned.

This is the portion of the code I have an issue with.

async function scrape_and_display(item) {
  await page.select("select#grocery-choice", item);
  await page.waitForSelector("div.grocery-item");
  const grocery_all = await page.evaluate((x) => {
    let grocery_info = [];
    let grocery_list = document.querySelectorAll("div.grocery-item");
    grocery_list.forEach((item) => {
      grocery_info.push({
        name: item.getElementsByTagName("h4")[0].innerText,
        price: item.getElementsByTagName("p")[0].innerText,
        availability: item.getElementsByTagName("p")[1].innerText,
      });
    });
    return grocery_info;
  });
  console.log(grocery_all);
}

for (item of ["fruit", "vegetable", "bakery"]) {
  await scrape_and_display(item);
}

Required Output:

[
  { name: 'Banana', price: '$10 / kg', availability: 'In Stock' },
  { name: 'Apple', price: '$20 / kg', availability: 'In Stock' },
  { name: 'Grape', price: '$45 / kg', availability: 'Out of Stock' },
  { name: 'Orange', price: '$10 / kg', availability: 'In Stock' }
]
[
  { name: 'Brocli', price: '$10 / kg', availability: 'In Stock' },
  { name: 'Pumpkin', price: '$15 / kg', availability: 'In Stock' },
  { name: 'Lettuce', price: '$7 / kg', availability: 'In Stock' }
]
[
  { name: 'Bread', price: '$15 / piece', availability: 'In Stock' },
  { name: 'Apple Pie', price: '$20 / piece', availability: 'In Stock' }
]

Current Output:

[
  { name: 'Bread', price: '$15 / piece', availability: 'In Stock' },
  { name: 'Apple Pie', price: '$20 / piece', availability: 'In Stock' }
]
[
  { name: 'Bread', price: '$15 / piece', availability: 'In Stock' },
  { name: 'Apple Pie', price: '$20 / piece', availability: 'In Stock' }
]
[
  { name: 'Bread', price: '$15 / piece', availability: 'In Stock' },
  { name: 'Apple Pie', price: '$20 / piece', availability: 'In Stock' }
]

Here is the actual problem, instead of grocery category I have states of Nepal, and instead of grocery items, I need schools; and instead of logging to console, I am exporting it as CSV files. If you check CSV files, you will find all the same data, that is the last item of the dropdown, i.e. last option. Here is the working code:

const puppeteer = require("puppeteer");
const createCsvWriter = require("csv-writer").createObjectCsvWriter;

(async () => {
  const browser = await puppeteer.launch({
    headless: false,
    IgnoreHTTPSErrors: true,
  });
  const page = await browser.newPage();
  await page.setViewport({ width: 1366, height: 820 });

  await page.goto("https://www.pabson.org/#!/School-locator");
  const selectOptions = await page.$$eval(
    "select#ddlState > option",
    (options) => {
      return options.map(function (option) {
        return { option: option.innerHTML, value: option.value };
      });
    }
  );
  async function scrape_and_save(item) {
    // let item = obj;
    await page.select("#ddlState", item.value);
    try {
      await page.waitForSelector("div.count-box-info");
      const school_array = await page.evaluate((x) => {
        let school = [];
        let school_list = document.querySelectorAll("div.count-box-info");
        school_list.forEach((item) => {
          school.push({
            name: item.getElementsByTagName("h4")[0].innerText,
            phone: item.getElementsByTagName("p")[0].innerText,
            email: item.getElementsByTagName("p")[1].innerText,
          });
        });
        return school;
      });

      let csvWriter = createCsvWriter({
        path: "school_" + item.option + ".csv",
        header: [
          { id: "name", title: "NAME" },
          { id: "phone", title: "PHONE" },
          { id: "email", title: "EMAIL" },
        ],
      });

      csvWriter
        .writeRecords(school_array) // returns a promise
        .then(() => {
          console.log("...Done");
        });
    } catch (e) {
      console.log(e);
    }
  }
  for (item of selectOptions) {
    await scrape_and_save(item);
  }
  await browser.close();
})();

Any help will be appreciated. Thanks in advance.

1 Answers

It seems re-rendering school list takes some time and you do not wait for this time, so your code just scrape the same data too quickly. You can wait for some change, for example for change of the main info text above the list:

  async function scrape_and_save(item) {
    const previousData = await page.evaluate(
      () => document.querySelector('#divschoolData').innerText
    );

    await page.select("#ddlState", item.value);
    await page.waitForFunction(
      data => { return data !== document.querySelector('#divschoolData').innerText; },
      {},
      previousData,
    );

    try {
      // ...

BTW, try to always add a const/let for loop variable to prevent scope issues:

  for (const item of selectOptions) {

And use strict mode to not forget this)

Related