createReadStream wait for first to finish before starting second createReadStream

user_12284152 0
Viewed 318

From my understanding createReadStream is asynchronous and there is no equivalent synchronous method. This leads us to my problem I'm having today. As you can see in my code, I have two createReadStream, and the second one depends on the first one and it absolutely needs to be run in order so that stationIds gets filled with the desired values. My problem here is that stationIds is empty and so the second createReadStream doesn't do anything at all, how can I run these two createReadStream in the correct way I want?

// Read content of files 
csvDataFiles.forEach(file => {
  const data = []
  fs.createReadStream(directory + file)
  .pipe(csv())
  .on('data', function (row) {
    data.push(row)
  })
  .on('end', function() {
    const object = {}
    let stationId = parseInt(file.substr(0, file.indexOf('.')))
    stationIds.push(stationId)
    object[stationId] = {}
    object[stationId]['length'] = data.length
    object[stationId]['data'] = data
    stationsData.push(object)
  })
})

// Read stations 
fs.createReadStream(directory + 'asfdsfsd.csv')
  .pipe(csv({
    skipLines: 3
  }))
  .on('data', function (row) {
    if (stationIds.includes(parseInt(row['Station ID']))) {
      // console.log(row)
      stations.push(row)
    }
  })
  .on('end', function () {
      // console.log(stations)
  })
node.js asynchronous javascript
1 Answers

You could wrap the processing of the streams in a promise and wait until all data is read. After that you can process the final file. This still needs some error handling, but should give you something to start with:

// Read content of files 
async function processStationsCsv(directory, csvDataFiles) {
    let stationsIds = [];
    let stationsData = [];
    for (const csvFile of csvDataFiles) {
        await readStationDataFromFile(directory + csvFile, stationsIds, stationData);
    }

    let stations = [];
    await new Promise(resolve => {
        // Read stations 
        fs.createReadStream(directory + 'asfdsfsd.csv')
            .pipe(csv({
                skipLines: 3
            }))
            .on('data', row => {
                if (stationsIds.includes(parseInt(row['Station ID']))) {
                    stations.push(row)
                }
            })
            .on('end', () => {
                resolve();
            })
    })

    console.log(stations);

}

function readStationDataFromFile(filePath, stationIds, stationsData) {
    return new Promise(resolve => {
        fs.createReadStream(filePath)
            .pipe(csv())
            .on('data', function (row) {
                data.push(row)
            })
            .on('end', function () {
                const object = {}
                let stationId = parseInt(file.substr(0, file.indexOf('.')))
                stationIds.push(stationId)
                object[stationId] = {}
                object[stationId]['length'] = data.length
                object[stationId]['data'] = data
                stationsData.push(object)
                resolve();
            })
    });
}

// call it with the directory path and the array of csvDataFiles-paths
processStationsCsv(directory, csvDataFiles);
user_37616280
Related