Node : Wait the python script to write the file then upload it to s3

user_12646403 0
Viewed 344

I have done the following code. Where I create a file by a python script then upload it to S3 then give the user the ability to download it.

 exports.createFeature = async (req, res, next) => {
  
  let retourUrl = await uploadFile(req.body)

  res.status(201).json(retourUrl)

 };

function uploadFile(feature) {

  return new Promise(async (resolve, reject) => {
    
    let options = {
      scriptPath: 'pathToDcript',
      args: [arg1, arg2, arg3]
  };
   
  PythonShell.run('script.py', options, function (err) {
    if (err) throw err;
    console.log('file has been created !');

    //read the file 
    let contents = fs.readFileSync('pathToFile', {encoding:'utf8', flag:'r'});

    
    //convert it to buffer
    const fileContent = Buffer.from(contents, "utf-8");

    // Setting up S3 upload parameters
    let key = keyUserData+feature.userId+'/fileName'
    const params = {
        Bucket: bucket,
        Key:  key, // File name you want to save as in S3
        Body: fileContent
    };

    // Uploading files to the bucket
    s3.upload(params, function(err, data) {
        if (err) {
            throw err;
        }
        //console.log(`File uploaded successfully. ${data.Location}`);
    });
    
    // delete the file 
    fs.unlinkSync('pathToFile');
    
    //get url for download
    const presignedURL = s3.getSignedUrl('getObject', {
      Bucket: bucket,
      Key: key,
      Expires: 60*5
    })
    resolve(presignedURL)
  })
  });
    
}

But I have the download url before the file is been uploaded to S3, any idea on how I can make it wait till all finish ?

express node.js amazon-s3
2 Answers

if you want to use s3.upload with a callback. you need to change your code as mentioned below.

exports.createFeature = async (req, res, next) => {

  let retourUrl = await uploadFile(req.body)

  res.status(201).json(retourUrl)

};

function uploadFile(feature) {

  return new Promise((resolve, reject) => {

    let options = {
      scriptPath: 'pathToDcript',
      args: [arg1, arg2, arg3]
    };

    PythonShell.run('script.py', options, function (err) {
      if (err) throw err;
      console.log('file has been created !');

      //read the file 
      let contents = fs.readFileSync('pathToFile', { encoding: 'utf8', flag: 'r' });


      //convert it to buffer
      const fileContent = Buffer.from(contents, "utf-8");

      // Setting up S3 upload parameters
      let key = keyUserData + feature.userId + '/fileName'
      const params = {
        Bucket: bucket,
        Key: key, // File name you want to save as in S3
        Body: fileContent
      };

      // Uploading files to the bucket
      s3.upload(params, function (err, data) {
        if (err) {
          throw err;
        }
        // delete the file 
        fs.unlinkSync('pathToFile');
  
        //get url for download
        const presignedURL = s3.getSignedUrl('getObject', {
          Bucket: bucket,
          Key: key,
          Expires: 60 * 5
        })
        //console.log(`File uploaded successfully. ${data.Location}`);
        resolve(presignedURL)
      });
    })
  });

}
user_71166440

The S3 upload method of the AWS SDK returns a Promise which can be awaited on.

For example:

await s3.upload(...)

Note that in this case the callback function to the Python script should be changed to an async function, in order to allow the await syntax. For example:

PythonShell.run('script.py', options, async function (err)
user_18892730
Related