Javascript print to file sequentially

user_595638 0
Viewed 182

Somehow I cannot wrap my head around this problem.

The function runOneCombination below is executed for thousands of files and I would like to extract some information, do some calculations and then print them to a file that I keep appending to.

I don't think the other code of this project is needed, but if it is I can share it.

The problem is indicated in the code below. My expectation is that for each iteration of Promise.all(res.dailyData.map( async (dailyDataRow,index,arr) => { the three statements are executed in order and the fs.writeSync(out,'\r\n'); is executed. Unfortunately, this is not what happens. The output contains all output of writeDaily(out,dailyDataRow) appended without line breaks in between, but followed by one. Then comes everything contained in featureRow[0] (separated by line breaks) then what is in featureRow[1] and featureRow[2] (without line breaks again). My expectation was:

dailyDataRow,featureRow[0],featureRow[1],featureRow[2],\r\n

Previously, this worked, but without append to file so I could only handle one pair of files as input.

function runOneCombination(dailyFile,simFile) {
    return new Promise(async function(resolve, reject) {
        let res = await process(dailyFile,simFile);
        var wh = true;
        if(await fs.existsSync(outFilePath))
            wh = false;
        var out = fs.openSync(outFilePath,'a');            
       // out.once('open', async function(fd) {
            //Write Headers   
            if(wh) {
                await writeHeaders(out);
            }
            await Promise.all(res.dailyData.map( async (dailyDataRow,index,arr) => {
                if(index > 1) { 
                    let samples = await getFeatures(dailyDataRow,res.cdData);
                    await samples.forEach(async (featureRow)=>{
                        //console.log(dailyDataRow,featureRow[0],featureRow[1],featureRow[2]);
                        if(!featureRow) 
                            console.log("fail:",featureRow);
                        else { 
//Problem is in the following lines:
                            writeDaily(out,dailyDataRow).then(async(val)=>{
                                writeFeatures(out,featureRow[0]).then(async(val)=>{
                                    writeFeatures(out,featureRow[1]).then(async(val)=>{
                                        writeFeatures(out,featureRow[2]);
                                    });
                                });
                            });
                        }
                    });
                    fs.writeSync(out,'\r\n');
                }

            })).then((val)=>{
                //out.end(); //close the writing stream 
                resolve('success');
            });
       // });  
    });
}

What am I missing here?

promise async-await node.js file javascript
1 Answers

Why are you even using then when you have await? You need to stop using then and use only async functions and await.

You never need the Promise constructor unless you are wrapping something that takes a callback.

Your issue is definitely that samples.forEach should be:

for(featureRow of samples) { }

doing that in an async function will fix the issue of ordering. As each iteration will complete before the next.

I do not feel like fixing your code to use only await. But here it is anyway:

async function runOneCombination(dailyFile, simFile) {
  let res = await process(dailyFile, simFile);

  let wh = await fs.exists(outFilePath)
  let out = await fs.open(outFilePath, 'a');

  //Write Headers
  if (wh) {
    await writeHeaders(out);
  }

  for (dailyDataRow of res.dailyData.slice(1)) {
    let samples = await getFeatures(dailyDataRow, res.cdData);
    for (featureRow of samples) {
      if (!featureRow) {
        console.log("fail:", featureRow);
        continue;
      }
      //Problem is in the following lines:
      await writeDaily(out, dailyDataRow);
      for (let i = 0; i < 3; i++) {
        await writeFeatures(out, featureRow[i])
      }
      await fs.write(out, '\r\n');
    }
  }  
}

Looks like it could work but it needs some testing.

user_3887870
Related