How to make sure all of my file uploads are done on s3?

Viewed 89

I have a use case where I am generating many CSVs' and pushing it to S3 bucket, with meta properties with batch key. example one batch is executing which has 8 CSVs then all 8 CSVs will have same batch key on S3 meta properties.

Now here I want to make sure that all the file is getting generated and uploaded on S3. After that I have to send email of all the files as attached in email of single batch created. With the same example above I will be sending email of 8 attachment from that batch with single email as attachments of 8 CSVs.

How Can I make sure that all the CSVs are generated and stored on S3? Is there anything that can help me to achieve such functionality

Below is my handler function

/**
 * handler function
 */
exports.handler = async (event, context) => {

    var derived = {}

    const store = JSON.parse(event);

    const { data } = store;

    derived.filebatch = filebatchKey.uuid();

   
    derived.report_details = await get_details(data.id);


    const user = derived.user.filters.filter(function(x){return x.origin === "DAILY"});
    
    const teachers = derived.teachers.filters.filter(function(x){return x.origin === "MONTHLY"});
    
    const students = derived.students.filters.filter(function(x){return x.origin === "WEEKLY"});

    if(user.length > 0) {

      for (const row of user) {

            const fileName = `user-report${random.math(4)}`
        
            derived.user = await user(data, row);
            
            // here uploading to S3 with data, file name and batchid that will be stored in meta

            await csv_file_upload(derived.user, fileName, derived.filebatch);
      }

    }

    if(teachers.length > 0) {

      for (const row of teachers) {
      
          const fileName = `teachers-report${random.math(4)}`

          derived.teachers = await teachers(data, row)  

            // here uploading to S3 with data, file name and batchid that will be stored in meta
          await csv_file_upload(derived.teachers, fileName, derived.filebatch);

        }
    } 

    if(students.length > 0){

      for (const row of students) {

          const fileName = `students-report${random.math(4)}`

          derived.students = await students(data, row);

          // here uploading to S3 with data, file name and batchid that will be stored in meta
          await csv_file_upload(derived.students, fileName, derived.filebatch);

        }
    }
    
    return { "meta": { "message": "success" }};

};

CSV convert function

async function csv_file_upload(data, fileName, report_id) {
// function to convert data to csv and upload to s3

  var csv = utils.convertArrayOfObjectsToCSV(data);

  await s3.uploadToS3(process.env.S3_BUCKET, `${fileName}.csv`, csv, filebatchKey, report_id);
  
  return csv;

}

Function to Upload to s3

function uploadToS3(bucketName, filename, filedata, filebatch, report_id) {
    
      var currentDate = new Date().toString();
    
      try {
        var params = {
          Bucket: bucketName,
          Key: filename,
          Body: filedata,
          Metadata: {
            date: currentDate + '-metatag',
            batch_key: filebatch
          },
        };
        return s3.putObject(params).promise().then(() => 'Successfully uploaded');
      } catch (_error) {
        console.error('Error in uploadToS3', _error);
        throw _error;
      }
    }

It could happen that I might have N numbers of CSVs that generate, but the point is I should be getting the last upload or something like that so that I know that all files are executed and uploaded to S3 and Now I can send email with that batch attachment.

0 Answers
Related