query for range of consecutive values mongo

Viewed 2918

I have a document with a number field. A process adds those documents whose number value is not in the collection, but first, it checks if a document with that number exists.

Consider a collection of documents with number from 0 to 234, number from 653 to 667 and number from 10543 to 22000. Gaps exist for number from 235 to 652 and 668 to 10542 whose documents need to be imported.

Is it possible to build a query which returns the range of consecutive values that exist in the collection? (i.e. 0 to 234 and 653 to 667 and 10543 to 22000)

With this information, I would instantly know to fill missing documents between 235 to 652 and 668 to 10542 and continue at 22001...

2 Answers

If you can accept getting all individual IDs back that are missing as opposed to ranges then this is your query:

collection.aggregate({
    $group: {
        "_id": null, // group all documents into the same bucket
        "numbers":
        {
            $push: "$number" // create an array of all "number" fields
        }
    }
}, {
    $project: {
        "_id": 0, // get rid of the "_id" field - not really needed
        "numbers": {
            $setDifference: [ { // compute the difference between...
                $range: [ 0, 10 ] // ... all numbers from 0 to 10 - adjust this to your needs...
            }, "$numbers" ] // ...and the available values for "number"
        }
    }
})

There are ways to compute the ranges out of this information but I have a feeling this might not even be needed in your case.

UPDATE (based on your comment): Here is a longer version that adds some additional stages to get from the discrete numbers to to the ranges - the code is not exactly pretty and probably not super fast but it should be working at least...

collection.aggregate({
    $sort: {
        "number": 1 // we need to sort in order to find ranges later
    }
},
{
    $group: {
        "_id": null, // group all documents into the same bucket
        "numbers":
        {
            $push: "$number" // create an array of all "number" fields
        }
    }
}, {
    $project: {
        "_id": 0, // get rid of the "_id" field - not really needed
        "numbers": {
            $setDifference: [ { // compute the difference between...
                $range: [ 0, 10 ] // ... all numbers from 0 to 10 - adjust this to your needs...
            }, "$numbers" ] // ...and the available values for "number"
        }
    }
},
{
    $project: {
        "numbers": "$numbers", // ...we create two identical arrays
        "numbers2": "$numbers" // ...by duplicating our missing numbers array
    }
},
{
    $unwind: "$numbers" // this will flatten one of the two created number arrays
},
{
    $project: {
        "number": "$numbers",
        "precedingNumber": {
            $arrayElemAt: [
                "$numbers2", // use the second (remaining) numbers array to find the previous number...
                { $max: [0, { $add: [ { $indexOfArray: [ "$numbers2", "$numbers" ] }, -1 ] } ] } // ...which needs to sit in that sorted array at the position of the element we're looking at right now - 1
            ]
        },
        "followingNumber": {
            $arrayElemAt: [
                "$numbers2", // use the second (remaining) numbers array to find the next number...
                { $add: [ { $indexOfArray: [ "$numbers2", "$numbers" ] }, 1 ] } // ...which needs to sit in that sorted array at the position of the element we're looking at right now + 1
            ]
        }
    }
}, {
    $project: {
        "number": 1, // include number 
        "precedingInRange": { $cond: [ { $eq: [ { $add: [ "$number", -1 ] }, "$precedingNumber" ] }, true, false ] },
        "followingInRange": { $cond: [ { $eq: [ { $add: [ "$number", 1 ] }, "$followingNumber" ] }, true, false ] }
    }
}, {
    $match: {
        $or: [ // filter out all items that are inside a range (or rather: include only the outer items of each range)
            { "precedingInRange": false },
            { "followingInRange": false }
        ]
    }
}, {
    $project: { // some beautification of the ouput to help deal with the data in your application
        "singleNumber": { $cond: [ { $not: { $or: [ "$precedingInRange", "$followingInRange" ] } }, "$number", null ] },
        "startOfRange": { $cond: [ "$followingInRange", "$number", null ] },
        "endOfRange": { $cond: [ "$precedingInRange", "$number", null ] }
    }
})

UPDATE 2:

I have a feeling I found a way better way to nicely get the ranges without too much magic involved:

collection.aggregate({
    $sort: {
        "number": 1 // we need to sort by numbers in order to be able to do the range magic later
    }
}, {
    $group: {
        "_id": null, // group all documents into the same bucket
        "numbers":
        {
            $push: "$number" // create an array of all "number" fields
        }
    }
}, {
    $project: {
        "numbers": {
            $reduce: {
                input: "$numbers",
                initialValue: [],
                in: {
                    "start": { 
                        $concatArrays: [
                            "$$value.start",
                            {
                                $cond: { // if preceding element in array of numbers is not "current element - 1" then add it, otherwise skip
                                    if: { $ne: [ { $add: [ "$$this", -1 ] }, { $arrayElemAt: [ "$numbers", { $add: [ { $indexOfArray: [ "$numbers", "$$this" ] }, -1 ] } ] } ] },
                                    then: [ "$$this" ],
                                    else: []
                                }
                            }
                        ]
                    },
                    "end": { 
                        $concatArrays: [
                            "$$value.end",
                            {
                                $cond: { // if following element in array of numbers is not "current element + 1" then add it, otherwise skip
                                    if: { $ne: [ { $add: [ "$$this", 1 ] }, { $arrayElemAt: [ "$numbers", { $add: [ { $indexOfArray: [ "$numbers", "$$this" ] }, 1 ] } ] } ] },
                                    then: [ "$$this" ],
                                    else: []
                                }
                            }
                        ]
                    }
                }
            }
        }
    }
}, {
    $project: {
        "ranges": {
            $zip: {
                inputs: [ "$numbers.start", "$numbers.end" ],
            }
        }
    }
})

One angle you could approach this is to pre-define the ranges you wish to check for existence, then run an aggregate operation where you can get the counts of the numbers in those ranges.

Take For example, given the pre-defined ranges

var ranges = [
    [0, 99],
    [100, 199],
    [200, 299]
];

and a test collection with just 3 numbers:

db.test.insert([
    { number: 1  },
    { number: 87  },
    { number: 200  }
])

the pipeline to execute would be as follows

db.test.aggregate([
    {
        "$group": {
            "_id": null,
            "range0Count": {
                "$sum": {
                    "$cond": [
                        {
                            "$and": [
                                { "$gte": [ "$number", 0 ] },
                                { "$lte": [ "$number", 99 ] }
                            ]
                        },
                        1,
                        0
                    ]
                }
            },
            "range1Count": {
                "$sum": {
                    "$cond": [
                        {
                            "$and": [
                                { "$gte": [ "$number", 100 ] },
                                { "$lte": [ "$number", 199 ] }
                            ]
                        },
                        1,
                        0
                    ]
                }
            },
            "range2Count": {
                "$sum": {
                    "$cond": [
                        {
                            "$and": [
                                { "$gte": [ "$number", 200 ] },
                                { "$lte": [ "$number", 299 ] }
                            ]
                        },
                        1,
                        0
                    ]
                }
            }
        }
    }
])

which would produce the following result

{
    "_id" : null,
    "range0Count" : 2.0,
    "range1Count" : 0.0,
    "range2Count" : 1.0
}

You can further refactor your pipeline by using the reduce method on the ranges array to extract the group pipeline operator object as follows:

var ranges = [
    [0, 99],
    [100, 199],
    [200, 299]
];
var group = ranges.reduce(function(acc, range, idx) {
    acc["$group"]["range" + idx + "Count"] = {
        "$sum": {
            "$cond": [
                {
                    "$and": [
                        { "$gte": ["$number", range[0] ] },
                        { "$lte": ["$number", range[1] ] }
                    ]
                },
                1,
                0
            ]
        }
    };
    return acc;
}, { "$group": { "_id": null } });

db.test.aggregate([group])

Using the above template you can customise the ranges as you wish and then get from the results the range that has no count.

Related