NodeJS - output json array as mutiple json files to disk

Viewed 96

I try to use NodeJS to read a JSON array from a JSON file, and then output each JSON object multiple JSON files to the disk.

However, I got the Error EMFILE: too many open files

The array has 20,000 objects.

The code:

function main(){
var clusters_statistics=require("cluster_whole_1.json");
for(var i=0; i<clusters_statistics.length; i++){
    var fs=require('fs');
    var outputFilename='cut_json/'+i+'.json';
    fs.writeFile(outputFilename, JSON.stringify(clusters_statistics[i], null, 4), function(err) {
        if(err) {
            console.log(err);
        } else {
            console.log(data);
        }
    }); 
}
}

Update: 1. I tried to use the close() function as suggested by Gustavo, unfortunately, it still says "there are too many files open". ("Open" this time). 2. Then I tried recursion inside the close(), and it works now.

The code:

function main(){
    clusters_statistics=require("cluster_whole_1.json");
    call_close(clusters_statistics.length);
}

function call_close(i){

    var fs = require("fs");
    var path = 'cut_json/'+i+'.json';

    fs.open(path, "w+", function(error, fd) {
        if (error) {
            console.error("open error:  " + error.message);
        }else {
            fs.writeFile(path, JSON.stringify(clusters_statistics[i], null, 4), function(err) {
            if(err) {
              console.log(err);
            } else {

            }
        }); 

        fs.close(fd, function(error) {
            if (error) {
                console.log(err);
            } else {
              if(i<=0){
                  return;
              }else{
                  if(i%100==0){
                      console.log(i);
                  }
                  call_close(i-1);
              }
            }
            });
        }
    });
}
2 Answers
Related