Process 10 GB JSON file in Node JS

Question

I have a json file and the structure of that file is as follows:

{
"orders":[
 {
  "id": 876876876,
  "app_id":580714,
  "client_details": {},
  "discount_codes": [{}],
  "line_items": [
        {
          "id": 466157049,
          ...
        }
   ],
   ...... 
 },
 {
   "id": 47844583,
   "app_id":580714,
   "client_details": {},
   "discount_codes": [{}],
   "line_items": [
        {
          "id": 466157049,
           ...
        }],
     ....
 },
 {...},
 {...},
 {...}
 ]
}

This array can contains more than 10lakhs (1 million) objects. Currently I need to:

find the object with order id
Total number of orders
get orders with order id and with the number of limit

I am using the following code:

 return new Promise((resolve, reject) => {
        var orders = []
        var getStream = function () {
            var stream = fs.createReadStream(file_path, { encoding: 'utf8' }),
                parser = JSONStream.parse('*');
            return stream.pipe(parser);
        };
    
        getStream()
        .pipe(es.mapSync(function (data) {
            
            orders = data
        })) .on('end', function() {
            
            resolve(orders)

        })
})

But it makes the system hang. Also, I have used the following command as well:

 node --max-old-space-size=8192 index.js

But that also does not worked. Can anyone please help me with processing such big json file.

Edited: Now filesize is 850MB and I am using the following code:

return new Promise((resolve, reject) => {
  var data = ''
        var reader_stream = fs.createReadStream(file_path) 
        reader_stream.setEncoding('UTF8')

        reader_stream.on('data', function(chunk) {
            data += chunk
        })

        reader_stream.on('end',function() {
            try{
                const orders_result = JSON.parse(data)
                var order_count     = (orders_result.orders)

                resolve({
                    "count": order_count.length
                })
            } catch(err) {
                console.log(err)
            }
        })

        reader_stream.on('error', function(err) {
            console.log(err.stack)
            reject(err.stack)
        })
})

and getting the following error

Uncaught Exception: RangeError: Invalid string length

user_431715 · Answer

JSON.parse needs to read the whole file into memory, including the parts that your application does not need. One approach would be to use a SAX-like parser like clarinet. These parsers don't read the whole file into memory, they generate events during the parsing process. You need to handle these events to check whether the data is of interest and only store the information that you actually need.

This will reduce the amount of memory required for the parsing process, but it is not as convenient. Your operation sounds like you don't need all the data, so maybe you are lucky and a stripped down version can fit into memory.

Process 10 GB JSON file in Node JS

1 Answers