Node js workers should increase outgoing https get requests

Viewed 42

My setup is https module with the use of worker_threads module and redis queue as url list. A single worker can process up to 800 outgoing https get requests per second. My plan was to increase the worker count and increase the outgoing requests. But what actually happened is that the outgoing requests got split up on every worker.

So if I am running 4x workers every one just process 200 outgoing https get requests. I thought it would be 4x multiplied with 800 so in total 2400 outgoing https get requests per second.Ubuntu open files limit is 65000 so I already checked this. Node js script runs without nginx or any other load balancer.

Here is my code please tell me where am I doing wrong.

Main file

const { Worker, isMainThread, parentPort } = require("worker_threads");

function createWorker() {
  var worker = new Worker("./wxr.js", { workerData: {} });
  worker.on("error", (err) => {
    throw err;
  });
  worker.on("message", callback);
  return worker;
}

function callback(data) {}

function spyder(works) {
  var workers = [];

  for (i = 1; i < works + 1; i++) {
    var count = i;
    workers[count] = createWorker();
  }
  var workers = 4;
  spyder(workers);
}

Worker file

const https = require("https");
const { parentPort, workerData } = require("worker_threads");

const redis = require("redis");
const client = redis.createClient();

var max = 0;

function start() {
  var itv = setInterval(function () {
    max++;
    loop();
    if (max == 100) {
      clearInterval(itv);
    }
  }, 0);
}

start();

function loop() {
  client.lpop(["urls"], function (err, url) {
    if (err) {
      return loop();
    } else if (url == null || url == undefined || url == "" || url == " ") {
      return loop();
    } else {
      easyReq(url, function (error, response, body) {
        return loop();
        // easyReq is just a wrapper for the https module get request function I wrote
      });
    }
  });
}
0 Answers
Related