Making Puppeteer goto different links at the same time

You can use puppeteer-cluster to run the tasks in parallel (I'm the author of that library). You can specify how many pages (or browsers) you want to use in parallel. The library then takes care of that to run your task(s).

Code Sample

const { Cluster } = require('puppeteer-cluster');

const videos = [];

(async () => {
    // Setup a cluster with 4 browsers in parallel
    const cluster = await Cluster.launch({
        concurrency: Cluster.CONCURRENCY_BROWSER,
        maxConcurrency: 4,
    });

    // Define your task to be executed
    await cluster.task(async ({ page, data: url }) => {
        await page.goto(url);
        await page.waitForSelector('.music-info');

        var vidInfo = await page.evaluate(/* ... */);
        videos.push(vidInfo);
    });

    // Queue your URLs
    for(var i = 0; i < profile.videoLinks.length; i++){
        cluster.queue(profile.videoLinks[i].video);
    }

    // Wait for the cluster to finish and close it
    await cluster.idle();
    await cluster.close();
})();

I optimize your code by using BrowserContext. and scraping all links in parallel using Promise.all().

Run it and tell me if everything is good.

const puppeteer = require("puppeteer");

const profile = {
  videoLinks: [{ video: "https://example.com/video.mp4" }],
};

const doMyBusiness = async (browser, video) => {
  const context = await browser.createIncognitoBrowserContext();
  const page = await context.newPage();

  await page.goto(video, {
    // waitUntil: 'load'
  });

  await page.waitForSelector(".music-info");

  const vidInfo = await page.evaluate(() => {
    const vidTitle = document.querySelector(".video-meta-title").innerText;
    const musicInfo = document.querySelector(".music-info").innerText;
    const musicLink = document.querySelector(".music-info a").href;
    const likes = document.querySelector(".like-text").innerText;
    const comments = document.querySelector(".comment-text").innerText;

    return {
      vidTitle,
      musicInfo,
      musicLink,
      likes,
      comments,
    };
  });

  await context.close();

  return vidInfo;
};

const main = async () => {
  const browser = await puppeteer.launch();

  const videos = await Promise.all(
    profile.videoLinks.map((video) => doMyBusiness(browser, video))
  );

  console.log(videos);

  await browser.close();
};

Edited my code to can handle a lot of videos by work with small chunks.

NOTE: I'm using puppeteer to scraping title of list of links. you should edit page.waitForSelector() & page.evaluate() to apply your business.

const puppeteer = require('puppeteer');

const profile = {
  videoLinks: [
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
    { video: 'https://linkedin.com' },
    { video: 'https://stackoverflow.com' },
    { video: 'https://facebook.com' },
    { video: 'https://apple.com' },
    { video: 'https://leetcode.com' },
    { video: 'https://codewars.com' },
    { video: 'https://google.com' },
    { video: 'https://google.com' },
  ],
};

const doMyBusiness = async (browser, video) => {
  const context = await browser.createIncognitoBrowserContext();
  const page = await context.newPage();

  await page.goto(video, {
    // waitUntil: 'load'
  });

  await page.waitForSelector('title');

  const vidInfo = await page.evaluate(() => {
    const vidTitle = document.querySelector('title').innerText;

    return {
      vidTitle,
    };
  });

  await context.close();

  return vidInfo;
};

const main = async () => {
  const browser = await puppeteer.launch();
  const videos = [];
  const chunkLength = 10;

  for (let i = 0, len = profile.videoLinks.length; i < len; i += chunkLength) {
    const chunkOfLinks = profile.videoLinks.slice(i, i + chunkLength);

    const chunkOfVideos = await Promise.all(
      chunkOfLinks.map(videoLink =>
        doMyBusiness(browser, videoLink.video).catch(err => err),
      ),
    );

    videos.push(...chunkOfVideos);
  }

  console.log({ videos });

  await browser.close();
};

main();