I have a use case where I am generating many CSVs' and pushing it to S3 bucket, with meta properties with batch key. example one batch is executing which has 8 CSVs then all 8 CSVs will have same batch key on S3 meta properties.
Now here I want to make sure that all the file is getting generated and uploaded on S3. After that I have to send email of all the files as attached in email of single batch created. With the same example above I will be sending email of 8 attachment from that batch with single email as attachments of 8 CSVs.
How Can I make sure that all the CSVs are generated and stored on S3? Is there anything that can help me to achieve such functionality
Below is my handler function
/**
* handler function
*/
exports.handler = async (event, context) => {
var derived = {}
const store = JSON.parse(event);
const { data } = store;
derived.filebatch = filebatchKey.uuid();
derived.report_details = await get_details(data.id);
const user = derived.user.filters.filter(function(x){return x.origin === "DAILY"});
const teachers = derived.teachers.filters.filter(function(x){return x.origin === "MONTHLY"});
const students = derived.students.filters.filter(function(x){return x.origin === "WEEKLY"});
if(user.length > 0) {
for (const row of user) {
const fileName = `user-report${random.math(4)}`
derived.user = await user(data, row);
// here uploading to S3 with data, file name and batchid that will be stored in meta
await csv_file_upload(derived.user, fileName, derived.filebatch);
}
}
if(teachers.length > 0) {
for (const row of teachers) {
const fileName = `teachers-report${random.math(4)}`
derived.teachers = await teachers(data, row)
// here uploading to S3 with data, file name and batchid that will be stored in meta
await csv_file_upload(derived.teachers, fileName, derived.filebatch);
}
}
if(students.length > 0){
for (const row of students) {
const fileName = `students-report${random.math(4)}`
derived.students = await students(data, row);
// here uploading to S3 with data, file name and batchid that will be stored in meta
await csv_file_upload(derived.students, fileName, derived.filebatch);
}
}
return { "meta": { "message": "success" }};
};
CSV convert function
async function csv_file_upload(data, fileName, report_id) {
// function to convert data to csv and upload to s3
var csv = utils.convertArrayOfObjectsToCSV(data);
await s3.uploadToS3(process.env.S3_BUCKET, `${fileName}.csv`, csv, filebatchKey, report_id);
return csv;
}
Function to Upload to s3
function uploadToS3(bucketName, filename, filedata, filebatch, report_id) {
var currentDate = new Date().toString();
try {
var params = {
Bucket: bucketName,
Key: filename,
Body: filedata,
Metadata: {
date: currentDate + '-metatag',
batch_key: filebatch
},
};
return s3.putObject(params).promise().then(() => 'Successfully uploaded');
} catch (_error) {
console.error('Error in uploadToS3', _error);
throw _error;
}
}
It could happen that I might have N numbers of CSVs that generate, but the point is I should be getting the last upload or something like that so that I know that all files are executed and uploaded to S3 and Now I can send email with that batch attachment.