node puppeteer working ok on local ,but getting timeout on heroku/render code=H12

Viewed 294

when trying to run the below code the /test and /testp are working fine but the / is getting time out on both Heroku and render services

index.js

const express = require("express");
const app = express();
const port = process.env.PORT || 5000;
const scrapper = require("./scraper");
const scrapper2 = require("./scrapper2");

app.get("/", async (req, res) => {
  const link =
    "http://yit.maya-tour.co.il/Drop_Report.aspx?client_code=2660&coordinator_code=2669";
  console.log(link);
  let searchQery = req.query.search;
  const scrapp = await scrapper(link);
  if (searchQery) {
    let station = scrapp.filter((station) =>
      station?.location?.includes(searchQery)
    );
    return res.json(station);
  }

  return res.json(scrapp);
});
app.get("/test", async (req, res) => {
  return res.json("helo");
});
app.get("/testp", async (req, res) => {
  const scrapp = await scrapper2();
  return res.json(scrapp);
});


app.listen(port, () => {
  console.log(`scrapper on${port}`);
});

scrapper.js

const puppeteer = require("puppeteer");

module.exports = async (link) => {
  try {
    const browserFetcher = puppeteer.createBrowserFetcher();

    const revisionInfo = await browserFetcher.download("938248");

    //essential boilerpalte
    const browser = await puppeteer.launch({
      executablePath: revisionInfo.executablePath,
      headless: true,
      args: ["--no-sandbox"],
    });
    const page = await browser.newPage();
    await page.goto(link);
    await page.waitForSelector("#PassListView1 > tbody", { timeout: 3000 });
    // getting all the buttons
    let buttons = await page.$$eval("a", (links) =>
      links.map((a) => ({ name: a.innerText, link: a.href, button: a }))
    );
    //filtering the non number ones like sorting by name
    buttons = buttons.filter((obj) => Number(obj.name));
    let array = [];
    //getting the first page
    const options = await page.$$eval("#PassListView1 > tbody>tr", (options) =>
      options.map((option) => ({
        driverName: option.innerText.split("\t")[0],
        vehicle: option.innerText.split("\t")[1],
        station: option.innerText.split("\t")[2],
        location: option.innerText.split("\t")[3],
        hour: option.innerText.split("\t")[4],
        all: option.innerText.split("\t"),
      }))
    );
    array = [...array, ...options];
    // iterating on all the buttons -click-> await for page to load -> getting the content of the page
    for await (const [i, { name, link, button }] of buttons.entries()) {
      await Promise.all([
        //waiting page to load
        page.waitForNavigation(),
        //clicking on button
        page.click(
          `#PassListView1 > tbody > tr:nth-child(14) > td > table > tbody > tr > td:nth-child(${
            i + 2
          }) > a `
        ),
        //waiting for a main selector to make sure page loaded
        page.waitForSelector("#PassListView1 > tbody", { timeout: 1000 }),
      ]);
      //scraping through data
      const options = await page.$$eval(
        "#PassListView1 > tbody>tr",
        (options) =>
          options.map((option) => ({
            driverName: option.innerText.split("\t")[0],
            vehicle: option.innerText.split("\t")[1],
            station: option.innerText.split("\t")[2],
            location: option.innerText.split("\t")[3],
            hour: option.innerText.split("\t")[4],
            all: option.innerText.split("\t"),
          }))
      );

      array = [...array, ...options];
    }
    await browser.close();
    return array;
  } catch (error) {
    console.log(error);
  }
};

scrapper2.js --just to see something is working

this one actually does work

const puppeteer = require("puppeteer");

module.exports = async () => {
  try {
    const browserFetcher = puppeteer.createBrowserFetcher();

    const revisionInfo = await browserFetcher.download("938248");

    //essential boilerpalte
    const browser = await puppeteer.launch({
      executablePath: revisionInfo.executablePath,
      headless: true,
      args: ["--no-sandbox"],
    });
    const page = await browser.newPage();
    await page.goto("https://www.youtube.com/", {
      waitUntil: "load",
      // Remove the timeout
      timeout: 0,
    });
    const tilte = await page.title();
    await browser.close();
    return tilte;
  } catch (error) {
    console.log(error);
  }
};

package.json

{
  "name": "webscrapping",
  "version": "1.0.0",
  "description": "",
  "main": "index.js",
  "scripts": {
    "test": "echo \"Error: no test specified\" && exit 1",
    "start": "nodemon index.js"
  },
  "keywords": [],
  "author": "",
  "license": "ISC",
  "dependencies": {
    "express": "^4.17.2",
    "nodemon": "^2.0.15",
    "puppeteer": "^13.0.1",
    "puppeteer-core": "^1.9.0"
  }
}

Heroku log

2021-12-28T19:27:18.246133+00:00 app[web.1]: http://yit.maya-tour.co.il/Drop_Report.aspx?client_code=2660&coordinator_code=2669
2021-12-28T19:27:48.246622+00:00 heroku[router]: at=error code=H12 desc="Request timeout" method=GET path="/" host=my-station-server.herokuapp.com request_id=2f68f71e-e593-4124-936d-41d63e68ff17 fwd="77.139.50.119" dyno=web.1 connect=0ms service=30000ms status=503 bytes=0 protocol=http
2021-12-28T19:29:26.522035+00:00 app[web.1]: Error: net::ERR_CONNECTION_TIMED_OUT at http://yit.maya-tour.co.il/Drop_Report.aspx?client_code=2660&coordinator_code=2669
2021-12-28T19:29:26.522043+00:00 app[web.1]:     at navigate (/app/node_modules/puppeteer/lib/cjs/puppeteer/common/FrameManager.js:155:23)
2021-12-28T19:29:26.522044+00:00 app[web.1]:     at processTicksAndRejections (internal/process/task_queues.js:95:5)
2021-12-28T19:29:26.522044+00:00 app[web.1]:     at async FrameManager.navigateFrame (/app/node_modules/puppeteer/lib/cjs/puppeteer/common/FrameManager.js:130:21)
2021-12-28T19:29:26.522045+00:00 app[web.1]:     at async Frame.goto (/app/node_modules/puppeteer/lib/cjs/puppeteer/common/FrameManager.js:500:16)
2021-12-28T19:29:26.522045+00:00 app[web.1]:     at async Page.goto (/app/node_modules/puppeteer/lib/cjs/puppeteer/common/Page.js:1167:16)
2021-12-28T19:29:26.522046+00:00 app[web.1]:     at async module.exports (/app/scraper.js:16:5)
2021-12-28T19:29:26.522047+00:00 app[web.1]:     at async /app/index.js:13:18

also, see Heroku build packs installed buildpacks

https://github.com/CoffeeAndCode/puppeteer-heroku-buildpack.git
https://github.com/jontewks/puppeteer-heroku-buildpack
https://buildpack-registry.s3.amazonaws.com/buildpacks/jontewks/puppeteer.tgz

and heroku/nodejs

1 Answers

as it turns out the website will only accept Israeli IP address -so needs a proxy for that, still looking for one.

Related