So. I have a 'challenge' to track and check if certain IDs are valid with puppeteer in multiple websites. They starts with 3 different IDs, pub ban and mc. Also, some sites doesn't have one or two of those tags Above, my newbie code snippet, since I'm new with JS/Node and I'm really stuck here.
Example:
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<title></title>
<meta name="author" content="">
<meta name="description" content="">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link href="css/style.css" rel="stylesheet">
</head>
<body>
<div id="pub-1">
<p>Hello, world!</p>
</div>
<div id="mc-top">
<p>Hey!</p>
</div>
<script src="js/script.js"></script>
</body>
</html>
And my Node.js code using Puppeteer. Those tags are loaded asyncronously:
const { Logger, Human} = require('caterpillar');
const puppeteer = require("puppeteer");
var fs = require("fs");
const { resolve } = require('path');
const logger = new Logger()
logger.pipe(require('fs').createWriteStream('./debug.log'));
logger.pipe(new Human()).pipe(process.stdout)
var data = fs
//Domains are stored right here
.readFileSync("sites.csv")
.toString()
.split("\n")
.map((e) => e.trim())
.map((e) => e.split(",").map((e) => e.trim()));
(async () => {
for (let i = 0; i < data.length; i++) {
try {
const url = data[i];
const browser = await puppeteer.launch({
headless: false,
args: ["--window-size=1920,1080"],
defaultViewport: null,
});
const page = await browser.newPage();
await page.goto(`${url}`, { waitUntil: "networkidle2" });
console.log("Loading: " + data[i]);
await page.evaluate(scrollToBottom);
await page.waitFor(3000);
```
**My problem lies right here\/**
```
var ids = [];
ids = await page.$$eval('[id^="pub"]', (elms) =>
elms.map((elm) => elm.id));
var spanval = async function Crawl(){
for (let i = 0; i < ids.length; i++) {
const spanVal = await page.$eval("#" + ids[i], (el) => el.innerHTML);
resolve(spanVal);
}
}
if (spanval == false) {
console.log('...');
} else{
logger.log("\n Site: \n"+url);
logger.log("✅Check. IDs caught: \n" +ids+ "\n")
logger.log("IDs parsed:" + ids.length)
}
await browser.close();
} catch (err) {
console.log("Error:" +err.message);
}
}
await browser.close();
})();
// Dealing with lazyloaded content here
async function scrollToBottom() {
await new Promise(resolve => {
const distance = 100;
const delay = 100;
const timer = setInterval(() => {
document.scrollingElement.scrollBy(0, distance);
if (document.scrollingElement.scrollTop + window.innerHeight >= document.scrollingElement.scrollHeight) {
clearInterval(timer);
resolve();
}
}, delay);
});
}
Any other suggestions about my code are appreciated, too!