Looping through an array of links gives navigation timeout error - puppeteer, nodejs

Viewed 39

I have an array of div elements that I want to loop and click the links within it and do the following:

  1. Scrape the images in the clicked linked
  2. store the images in a folder in my local machine
  3. go back to previous/first page
  4. repeat process till last link

While I am trying to do this, I keep getting a timeout error after the first iteration, also the code to go previous/first page executes before all the images are being grabbed. Here is my code:

const puppeteer = require('puppeteer');
const fs = require("fs");

(async () => {
  const browser = await puppeteer.launch({headless: true});
  const page = await browser.newPage();
  await page.setDefaultNavigationTimeout(0);
  await page.goto(
    "https://www.farfetch.com/ng/shopping/men/trainers-2/items.aspx",
  );

  const divLength = await page.$$eval(
    "#slice-container > div:nth-child(3) > div.ltr-8q7uf4.e1vk6kxu2 > div.ltr-8s6hxx.e1vk6kxu0 > div > div.ltr-113ivyq.elu6vcm0 > ul > div", div => div.length
  );

  for (let i = 1; i < divLength; i++) {
    await page.click(
      `#slice-container > div:nth-child(3) > div.ltr-8q7uf4.e1vk6kxu2 > div.ltr-8s6hxx.e1vk6kxu0 > div > div.ltr-113ivyq.elu6vcm0 > ul > div:nth-child(${i}) > a`
    );
    await page.waitForSelector(
      "#content > div > div.ltr-wckt8b > div.ltr-rcjmp3 > div > div > button > img"
    );
    const imageUrl = await page.$$eval(
      "#content > div > div.ltr-wckt8b > div.ltr-rcjmp3 > div > div > button > img", (imgs) => {
      return imgs.map((x) => x.src)}
    );
    console.log(imageUrl);
    await page.goBack({
      timeout: 0,
      waitUntil: "networkidle0",
    });
  }

  // await browser.close();
})();
0 Answers
Related