Is there a way to search multiple IDs in a site using Puppeteer and check if they aren't empty?

Viewed 174

So. I have a 'challenge' to track and check if certain IDs are valid with puppeteer in multiple websites. They starts with 3 different IDs, pub ban and mc. Also, some sites doesn't have one or two of those tags Above, my newbie code snippet, since I'm new with JS/Node and I'm really stuck here.

Example:

<!DOCTYPE html>
<html>

<head>
  <meta charset="utf-8">
  <title></title>
  <meta name="author" content="">
  <meta name="description" content="">
  <meta name="viewport" content="width=device-width, initial-scale=1">

  <link href="css/style.css" rel="stylesheet">
</head>

<body>
  <div id="pub-1">
    <p>Hello, world!</p>
  </div>
   <div id="mc-top">
    <p>Hey!</p>
  </div>
  <script src="js/script.js"></script>
</body>

</html>

And my Node.js code using Puppeteer. Those tags are loaded asyncronously:

const { Logger, Human} = require('caterpillar');
const puppeteer = require("puppeteer");
var fs = require("fs");
const { resolve } = require('path');
const logger = new Logger()

logger.pipe(require('fs').createWriteStream('./debug.log'));
logger.pipe(new Human()).pipe(process.stdout)

var data = fs
    //Domains are stored right here
  .readFileSync("sites.csv")
  .toString()
  .split("\n")
  .map((e) => e.trim()) 
  .map((e) => e.split(",").map((e) => e.trim())); 
(async () => {
  for (let i = 0; i < data.length; i++) {
    try {
      const url = data[i];
      const browser = await puppeteer.launch({
        headless: false,
        args: ["--window-size=1920,1080"],
        defaultViewport: null,
      });
      const page = await browser.newPage();
      await page.goto(`${url}`, { waitUntil: "networkidle2" });
      console.log("Loading: " + data[i]);
      await page.evaluate(scrollToBottom);
      await page.waitFor(3000);
    ```
      **My problem lies right here\/**
    ```
      var ids = [];
      ids = await page.$$eval('[id^="pub"]', (elms) =>
      elms.map((elm) => elm.id));
     var spanval =  async function Crawl(){
      for (let i = 0; i < ids.length; i++) {
        const spanVal = await page.$eval("#" + ids[i], (el) => el.innerHTML);
        
        resolve(spanVal);
    }
}
        if (spanval == false) {
          console.log('...');
        } else{
          logger.log("\n Site: \n"+url);
          logger.log("✅Check. IDs caught: \n" +ids+ "\n")
          logger.log("IDs parsed:" + ids.length)
        }
      

      await browser.close();
    } catch (err) {
      console.log("Error:" +err.message);
    }
  }
  await browser.close();

})();
// Dealing with lazyloaded content here

async function scrollToBottom() {
  await new Promise(resolve => {
    const distance = 100; 
    const delay = 100;
    const timer = setInterval(() => {
      document.scrollingElement.scrollBy(0, distance);
      if (document.scrollingElement.scrollTop + window.innerHeight >= document.scrollingElement.scrollHeight) {
        clearInterval(timer);
        resolve();
      }
    }, delay);
  });
}

Any other suggestions about my code are appreciated, too!

0 Answers
Related