Trying to use async.parallelLimit to make to make successive parallel API calls

user_7994178 0
Viewed 257

I have a data array of objects like [{number:1}, {number:2}, {number:3}... {number:100}]. And want to make parallel API calls in successive batches of 10 until the whole array has been processed.

How would I go about that?

Here's my code. It goes over the first 10, but then it stops.

const async = require("async");
const axios = require("axios");
const calls = require("../model/data"); // [{number:1}, {number:2},{number:3},...{number:100}]

let makeAPICall = function (request, callback) {
  axios
    .post("http://www.api.com/", {
      number: `${request.number}`,
      webhookURL: "http://localhost:8000/",
    })
    .then(function (response) {})
    .catch(function (err) {
      callback(err);
    });
};

const functionArray = calls.map((request) => {
  return (callback) => makeAPICall(request, callback);
});

exports.startCalls = (req, res, next) => {
  async.parallelLimit(functionArray, 10, (err, results) => {
    if (err) {
      console.error("Error: ", err);
    } else {
      console.log("Results: ", results.length, results);
    }
  });
};
async.js express node.js asynchronous api
2 Answers

So, your approach is somewhat backwards as you have something that already returns a promise (Axios) which is the modern way to manage asynchronous operations and now you're trying to convert it back to a plain callback so you can use an old-fashioned library (the async library). It's also slower to run 10, wait for all 10 to finish before starting any more when that is typically not necessary.

Instead, I would suggest you use the Axios promise you already have. You can either use Bluebird's .map() which has a concurrency setting or you can use this bit of code that gives you a function for running promise-producing functions in parallel while controlling the max number that are in-flight at any given time:

// takes an array of items and a function that returns a promise
function mapConcurrent(items, maxConcurrent, fn) {
    let index = 0;
    let inFlightCntr = 0;
    let doneCntr = 0;
    let results = new Array(items.length);
    let stop = false;

    return new Promise(function(resolve, reject) {

        function runNext() {
            let i = index;
            ++inFlightCntr;
            fn(items[index], index++).then(function(val) {
                ++doneCntr;
                --inFlightCntr;
                results[i] = val;
                run();
            }, function(err) {
                // set flag so we don't launch any more requests
                stop = true;
                reject(err);
            });
        }

        function run() {
            // launch as many as we're allowed to
            while (!stop && inflightCntr < maxConcurrent && index < items.length) {
                runNext();
            }
            // if all are done, then resolve parent promise with results
            if (doneCntr === items.length) {
                resolve(results);
            }
        }

        run();
    });
}

You would then use it like this:

let makeAPICall = function(request) {
  return axios.post("http://www.api.com/", {
      number: `${request.number}`,
      webhookURL: "http://localhost:8000/",
    });
};

mapConcurrent(calls, 10, makeAPICall).then(results => {
   // all results in order here
   console.log(results);
}).catch(err => {
    console.log(err);
});

See another similar issue here: Promise.all consumes all my RAM

user_8166200

If you really want to run them in fixed batches where the whole batch finishes before you run any more requests, you could do something like this:

const axios = require("axios");
const calls = require("../model/data"); // [{number:1}, {number:2},{number:3},...{number:100}]

function makeAPICall(request) {
  return axios.post("http://www.api.com/", {
      number: `${request.number}`,
      webhookURL: "http://localhost:8000/",
  });
};

async function runBatches(array, batchSize, fn) {
    let index = 0;
    let results = [];
    while (index < array.length) {
        let promises = [];
        for (let num = 0; num < batchSize && index < array.length; ++num) {
            promises.push(makeAPICall(array[index++]));
        }
        let batchResults = await Promise.all(promises);
        results.push(...batchResults);
    }
    return results;
}

runBatches(calls, 10, makeAPICall).then(results => {
   // all results in order here
   console.log(results);
}).catch(err => {
    console.log(err);
});
user_8166200
Related