when trying to run the below code the /test and /testp are working fine but the / is getting time out on both Heroku and render services
index.js
const express = require("express");
const app = express();
const port = process.env.PORT || 5000;
const scrapper = require("./scraper");
const scrapper2 = require("./scrapper2");
app.get("/", async (req, res) => {
const link =
"http://yit.maya-tour.co.il/Drop_Report.aspx?client_code=2660&coordinator_code=2669";
console.log(link);
let searchQery = req.query.search;
const scrapp = await scrapper(link);
if (searchQery) {
let station = scrapp.filter((station) =>
station?.location?.includes(searchQery)
);
return res.json(station);
}
return res.json(scrapp);
});
app.get("/test", async (req, res) => {
return res.json("helo");
});
app.get("/testp", async (req, res) => {
const scrapp = await scrapper2();
return res.json(scrapp);
});
app.listen(port, () => {
console.log(`scrapper on${port}`);
});
scrapper.js
const puppeteer = require("puppeteer");
module.exports = async (link) => {
try {
const browserFetcher = puppeteer.createBrowserFetcher();
const revisionInfo = await browserFetcher.download("938248");
//essential boilerpalte
const browser = await puppeteer.launch({
executablePath: revisionInfo.executablePath,
headless: true,
args: ["--no-sandbox"],
});
const page = await browser.newPage();
await page.goto(link);
await page.waitForSelector("#PassListView1 > tbody", { timeout: 3000 });
// getting all the buttons
let buttons = await page.$$eval("a", (links) =>
links.map((a) => ({ name: a.innerText, link: a.href, button: a }))
);
//filtering the non number ones like sorting by name
buttons = buttons.filter((obj) => Number(obj.name));
let array = [];
//getting the first page
const options = await page.$$eval("#PassListView1 > tbody>tr", (options) =>
options.map((option) => ({
driverName: option.innerText.split("\t")[0],
vehicle: option.innerText.split("\t")[1],
station: option.innerText.split("\t")[2],
location: option.innerText.split("\t")[3],
hour: option.innerText.split("\t")[4],
all: option.innerText.split("\t"),
}))
);
array = [...array, ...options];
// iterating on all the buttons -click-> await for page to load -> getting the content of the page
for await (const [i, { name, link, button }] of buttons.entries()) {
await Promise.all([
//waiting page to load
page.waitForNavigation(),
//clicking on button
page.click(
`#PassListView1 > tbody > tr:nth-child(14) > td > table > tbody > tr > td:nth-child(${
i + 2
}) > a `
),
//waiting for a main selector to make sure page loaded
page.waitForSelector("#PassListView1 > tbody", { timeout: 1000 }),
]);
//scraping through data
const options = await page.$$eval(
"#PassListView1 > tbody>tr",
(options) =>
options.map((option) => ({
driverName: option.innerText.split("\t")[0],
vehicle: option.innerText.split("\t")[1],
station: option.innerText.split("\t")[2],
location: option.innerText.split("\t")[3],
hour: option.innerText.split("\t")[4],
all: option.innerText.split("\t"),
}))
);
array = [...array, ...options];
}
await browser.close();
return array;
} catch (error) {
console.log(error);
}
};
scrapper2.js --just to see something is working
this one actually does work
const puppeteer = require("puppeteer");
module.exports = async () => {
try {
const browserFetcher = puppeteer.createBrowserFetcher();
const revisionInfo = await browserFetcher.download("938248");
//essential boilerpalte
const browser = await puppeteer.launch({
executablePath: revisionInfo.executablePath,
headless: true,
args: ["--no-sandbox"],
});
const page = await browser.newPage();
await page.goto("https://www.youtube.com/", {
waitUntil: "load",
// Remove the timeout
timeout: 0,
});
const tilte = await page.title();
await browser.close();
return tilte;
} catch (error) {
console.log(error);
}
};
package.json
{
"name": "webscrapping",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1",
"start": "nodemon index.js"
},
"keywords": [],
"author": "",
"license": "ISC",
"dependencies": {
"express": "^4.17.2",
"nodemon": "^2.0.15",
"puppeteer": "^13.0.1",
"puppeteer-core": "^1.9.0"
}
}
Heroku log
2021-12-28T19:27:18.246133+00:00 app[web.1]: http://yit.maya-tour.co.il/Drop_Report.aspx?client_code=2660&coordinator_code=2669
2021-12-28T19:27:48.246622+00:00 heroku[router]: at=error code=H12 desc="Request timeout" method=GET path="/" host=my-station-server.herokuapp.com request_id=2f68f71e-e593-4124-936d-41d63e68ff17 fwd="77.139.50.119" dyno=web.1 connect=0ms service=30000ms status=503 bytes=0 protocol=http
2021-12-28T19:29:26.522035+00:00 app[web.1]: Error: net::ERR_CONNECTION_TIMED_OUT at http://yit.maya-tour.co.il/Drop_Report.aspx?client_code=2660&coordinator_code=2669
2021-12-28T19:29:26.522043+00:00 app[web.1]: at navigate (/app/node_modules/puppeteer/lib/cjs/puppeteer/common/FrameManager.js:155:23)
2021-12-28T19:29:26.522044+00:00 app[web.1]: at processTicksAndRejections (internal/process/task_queues.js:95:5)
2021-12-28T19:29:26.522044+00:00 app[web.1]: at async FrameManager.navigateFrame (/app/node_modules/puppeteer/lib/cjs/puppeteer/common/FrameManager.js:130:21)
2021-12-28T19:29:26.522045+00:00 app[web.1]: at async Frame.goto (/app/node_modules/puppeteer/lib/cjs/puppeteer/common/FrameManager.js:500:16)
2021-12-28T19:29:26.522045+00:00 app[web.1]: at async Page.goto (/app/node_modules/puppeteer/lib/cjs/puppeteer/common/Page.js:1167:16)
2021-12-28T19:29:26.522046+00:00 app[web.1]: at async module.exports (/app/scraper.js:16:5)
2021-12-28T19:29:26.522047+00:00 app[web.1]: at async /app/index.js:13:18
also, see Heroku build packs installed

https://github.com/CoffeeAndCode/puppeteer-heroku-buildpack.git
https://github.com/jontewks/puppeteer-heroku-buildpack
https://buildpack-registry.s3.amazonaws.com/buildpacks/jontewks/puppeteer.tgz
and heroku/nodejs