We are trying web crawl and get contents from multiple pages. I am taking the advantage of async API with Promise ALL which can execute requests in parallel.
Is there a limitation on the number of contexts which can be opened parallel?
const fs = require('fs');
let browser;
const batch_size = 4; // control the number of async parallel calls
(async () => { // main function
let urls = [];
urls = fs.readFileSync('./resources/input_selenium_urls.csv').toString().split("\n");
browser = await chromium.launch();
let context_size = 0;
let processUrls = [];
let total_length = 0;
for (let i=0;i<urls.length;i++,total_length++) {
if ((context_size==batch_size)||(i==urls.length-1)){
await Promise.all(processUrls.map(x => getHTMLPageSource(x)));
context_size = 0;
processUrls = [];
} else {
processUrls.push(urls[i]);
context_size++;
}
}
await browser.close();
})();
async function getHTMLPageSource(url) {
const context = await browser.newContext();
const page = await context.newPage();
let response = {}
try {
await page.goto(url, { waitUntil: 'networkidle' });
response = {
url : url,
content: await page.title(),
error : null
}
console.log(response);
}
catch {
response = {
error : "Timeout error"
}
}
context.close;
return response;
}