I have an array of div elements that I want to loop and click the links within it and do the following:
- Scrape the images in the clicked linked
- store the images in a folder in my local machine
- go back to previous/first page
- repeat process till last link
While I am trying to do this, I keep getting a timeout error after the first iteration, also the code to go previous/first page executes before all the images are being grabbed. Here is my code:
const puppeteer = require('puppeteer');
const fs = require("fs");
(async () => {
const browser = await puppeteer.launch({headless: true});
const page = await browser.newPage();
await page.setDefaultNavigationTimeout(0);
await page.goto(
"https://www.farfetch.com/ng/shopping/men/trainers-2/items.aspx",
);
const divLength = await page.$$eval(
"#slice-container > div:nth-child(3) > div.ltr-8q7uf4.e1vk6kxu2 > div.ltr-8s6hxx.e1vk6kxu0 > div > div.ltr-113ivyq.elu6vcm0 > ul > div", div => div.length
);
for (let i = 1; i < divLength; i++) {
await page.click(
`#slice-container > div:nth-child(3) > div.ltr-8q7uf4.e1vk6kxu2 > div.ltr-8s6hxx.e1vk6kxu0 > div > div.ltr-113ivyq.elu6vcm0 > ul > div:nth-child(${i}) > a`
);
await page.waitForSelector(
"#content > div > div.ltr-wckt8b > div.ltr-rcjmp3 > div > div > button > img"
);
const imageUrl = await page.$$eval(
"#content > div > div.ltr-wckt8b > div.ltr-rcjmp3 > div > div > button > img", (imgs) => {
return imgs.map((x) => x.src)}
);
console.log(imageUrl);
await page.goBack({
timeout: 0,
waitUntil: "networkidle0",
});
}
// await browser.close();
})();