Im currently building a scraping app and everything was fine until it was deployed to heroku.
My idea is to have a all my info in a global array so I can simply send the GList as a response back.
The app behaved the way I want it locally but when deployed to heroku it seems to get very slow and get stuck on the for loop part and was not able to reach the browser.close() part.
I've tried adding the buildpacks for puppeteer on heroku but nothing seems to work.
2021-08-12T14:01:13.065554+00:00 app[web.1]: 1
2021-08-12T14:01:13.067475+00:00 app[web.1]: 2
2021-08-12T14:01:16.320617+00:00 app[web.1]: page
2021-08-12T14:01:17.443820+00:00 app[web.1]: page
2021-08-12T14:01:24.567206+00:00 app[web.1]: page
2021-08-12T14:01:43.066815+00:00 app[web.1]: 1
2021-08-12T14:01:43.066920+00:00 app[web.1]: 2
2021-08-12T14:01:46.345629+00:00 app[web.1]: TimeoutError: Navigation timeout of 30000 ms exceeded
2021-08-12T14:01:46.345637+00:00 app[web.1]: at /app/node_modules/puppeteer/lib/cjs/puppeteer/common/LifecycleWatcher.js:106:111
2021-08-12T14:01:46.347614+00:00 app[web.1]: ReferenceError: browser is not defined
2021-08-12T14:01:46.347615+00:00 app[web.1]: at scrapeProduct (file:///app/index.js:50:5)
2021-08-12T14:01:46.347615+00:00 app[web.1]: at async createList (file:///app/index.js:58:16)
2021-08-12T14:01:47.468472+00:00 app[web.1]: TimeoutError: Navigation timeout of 30000 ms exceeded
2021-08-12T14:01:47.468485+00:00 app[web.1]: at
/app/node_modules/puppeteer/lib/cjs/puppeteer/common/LifecycleWatcher.js:106:111
let GList = []
async function scrapeProduct() {
try {
console.log('2')
const array = []
const url = [RTX3090, RTX3080TI, RTX3080, RTX3070TI, RTX3070, RTX3060TI, RTX3060, RTX2060, GTX1660TI]
const browser = await puppeteer.launch({
args: [
'--no-sandbox',
'--disable-setuid-sandbox',
],
});
const page = await browser.newPage();
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36');
process.once('unhandledRejection', () => {
browser.close();
});
for (let i = 0; i < url.length; i++) {
console.log('page')
await page.goto(url[i])
const result = await page.evaluate(() => {
const List = []
allProduct = document.querySelectorAll("div.prod_info")
allProduct.forEach(ele => {
if (ele.querySelector('ul.price_box > li > span.site_btn.orderNow > button') !== null || ele.querySelector('ul.price_box > li > span.site_btn.add24hCart > button') !== null) {
const GraphicCard = {
'name': ele.querySelector('a').innerText,
'price': ele.querySelector('.value').innerText,
'link': ele.querySelector('a').href
}
List.push(GraphicCard)
}
})
return List;
})
array.push(result)
}
await browser.close();
console.log('3')
return array;
} catch (err) {
console.log(err);
await browser.close();
}
}
async function createList() {
try {
console.log('1')
GList = [];
GList.push(await scrapeProduct())
} catch (err) {
console.log(err);
}
}
createList()
setInterval(() => createList(), 30000)
app.get('/', (req, res) => {
res.send(GList);
})