Parse large JSON file in Nodejs

user_1141692 0
Viewed 145035

I have a file which stores many JavaScript objects in JSON form and I need to read the file, create each of the objects, and do something with them (insert them into a db in my case). The JavaScript objects can be represented a format:

Format A:

[{name: 'thing1'},
....
{name: 'thing999999999'}]

or Format B:

{name: 'thing1'}         // <== My choice.
...
{name: 'thing999999999'}

Note that the ... indicates a lot of JSON objects. I am aware I could read the entire file into memory and then use JSON.parse() like this:

fs.readFile(filePath, 'utf-8', function (err, fileContents) {
  if (err) throw err;
  console.log(JSON.parse(fileContents));
});

However, the file could be really large, I would prefer to use a stream to accomplish this. The problem I see with a stream is that the file contents could be broken into data chunks at any point, so how can I use JSON.parse() on such objects?

Ideally, each object would be read as a separate data chunk, but I am not sure on how to do that.

var importStream = fs.createReadStream(filePath, {flags: 'r', encoding: 'utf-8'});
importStream.on('data', function(chunk) {

    var pleaseBeAJSObject = JSON.parse(chunk);           
    // insert pleaseBeAJSObject in a database
});
importStream.on('end', function(item) {
   console.log("Woot, imported objects into the database!");
});*/

Note, I wish to prevent reading the entire file into memory. Time efficiency does not matter to me. Yes, I could try to read a number of objects at once and insert them all at once, but that's a performance tweak - I need a way that is guaranteed not to cause a memory overload, not matter how many objects are contained in the file.

I can choose to use FormatA or FormatB or maybe something else, just please specify in your answer. Thanks!

node.js file json javascript
11 Answers
https.get(url1 , function(response) {
  var data = ""; 
  response.on('data', function(chunk) {
    data += chunk.toString(); 
  }) 
  .on('end', function() {
    console.log(data)
  });
});
user_181782450

Using the @josh3736 answer, but for ES2021 and Node.js 16+ with async/await + AirBnb rules:

import fs from 'node:fs';

const file = 'file.json';

/**
 * @callback itemProcessorCb
 * @param {object} item The current item
 */

/**
 * Process each data chunk in a stream.
 *
 * @param {import('fs').ReadStream} readable The readable stream
 * @param {itemProcessorCb} itemProcessor A function to process each item
 */
async function processChunk(readable, itemProcessor) {
  let data = '';
  let total = 0;

  // eslint-disable-next-line no-restricted-syntax
  for await (const chunk of readable) {
    // join with last result, remove CR and get lines
    const lines = (data + chunk).replace('\r', '').split('\n');

    // clear last result
    data = '';

    // process lines
    let line = lines.shift();
    const items = [];

    while (line) {
      // check if isn't a empty line or an array definition
      if (line !== '' && !/[\[\]]+/.test(line)) {
        try {
          // remove the last comma and parse json
          const json = JSON.parse(line.replace(/\s?(,)+\s?$/, ''));
          items.push(json);
        } catch (error) {
          // last line gets only a partial line from chunk
          // so we add this to join at next loop
          data += line;
        }
      }

      // continue
      line = lines.shift();
    }

    total += items.length;

    // Process items in parallel
    await Promise.all(items.map(itemProcessor));
  }

  console.log(`${total} items processed.`);
}

// Process each item
async function processItem(item) {
  console.log(item);
}

// Init
try {
  const readable = fs.createReadStream(file, {
    flags: 'r',
    encoding: 'utf-8',
  });

  processChunk(readable, processItem);
} catch (error) {
  console.error(error.message);
}

For a JSON like:

[
  { "name": "A", "active": true },
  { "name": "B", "active": false },
  ...
]
user_8274720
Related