Best way to run 1k+ queries on MongoDB and nodejs?

user_394597 0
Viewed 37

I'm trying to run approximately 1,000 queries on MongoDB to check for matches on particular object property.

Please shield your eyes from what is no doubt some extremely amateur code but would appreciate any ways to make this more efficient. It works but takes a long time.

'followers' is an array of Twitter user objects and it is being checked against a database of half a million records.


function checkFollowers(followers){
  // check if followers are in DB

  let matches = [];

  MongoClient.connect(url, function(err, db) {
    if (err) throw err;

    console.log("Connected!");

    async.each(followers, function(item, callback){

      var regex = new RegExp("^" + item.username + "$", 'i')

      dbo.collection("User").findOne({Username : {$regex : regex}},function(err, result) {

        if (err) throw err;

        if(result != null){
          matches.push(result.Username)
        }
,
        callback(null);

      });
    }, function(err){
        if(err) {
            console.log(err);
        } else {
            console.log('All records have been checked');
            db.close();
            console.log("This many matching records " + matches.length)
            console.log(matches)
        }
    });      
  });
}
node.js mongodb javascript
1 Answers

Maybe like this:

const allUsers = dbo.collection("User").find().toArray();
followers.forEach( follower => {
   var regex = new RegExp("^" + follower.username + "$", 'i')      
   if (allUsers.filter( aUser = > regex.test(aUser))) 
      matches.push(aUser.Username);
})

Reading the collection again and again does not make much sense.

Regular expressions are always slow, so you may prefer

dbo.collection("User").aggregate([
   {$set: {username_UC: { $toUpper: "$username" } } }
]).toArray();
followers.forEach( follower => {
   if (allUsers.filter( aUser => aUser.username_UC == follower.toUpperCase() )) 
      matches.push(aUser.Username);
})

Or even faster:

dbo.collection("User").aggregate([
   {$match: {$expr: { $in: [ { $toUpper: "$username" }, followers.map(x => x.toUpperCase()) ] } } }
])
user_30272660
Related