express.js: pass an uploaded image to s3

user_16645990 0
Viewed 533

I am trying to pass an image uploaded from a react app through express to a managed s3 bucket. The platform/host I am using creates and manages the s3 bucket and generates upload and access urls. This all works fine (I have tested a generated upload url in postman with an image in a binary body and it worked perfectly).

My problem is passing the image through express. I am using multer to get the image from the form but I am assuming multer is turning that image into some kind of file object and s3 is expecting some sort of blob or stream.

In following code, the image in req.file exists, I get a 200 response from s3 with no errors and when I visit the asset url the url works, but the image itself is missing.

const router = Router();
const upload = multer()

router.post('/', upload.single('file'), async (req, res) => {
    console.log(req.file)
    const asset = req.file
    const assetPath = req.headers['asset-path']

    let s3URLs = await getPresignedURLS(assetPath)
  
    const sendAsset = await fetch(
      s3URLs.urls[0].upload_url, // the s3 upload url 
      {
        method: 'PUT',
        headers: {
            "Content-Type": asset.mimetype
        },
        body: asset,
        redirect: 'follow'
      }
    )
  console.log("s3 response", sendAsset)

  res.status(200).json({"url": s3URLs.urls[0].access_url });
  
});

export default router;

I am not sure what to do to convert what multer gives me to something that aws s3 will accept. I am also open to getting rid of multer if there is an easier way to upload binary files to express.

multer express node.js amazon-s3 javascript
2 Answers

Instead of multer, you can use multiparty to get file data from request object. And to upload file to s3 bucket you can use aws-sdk.

const AWS = require("aws-sdk");
const multiparty = require("multiparty");

/**
 * Helper method which takes the request object and returns a promise with a data.
 */
const getDataFromRequest = (req) =>
  new Promise(async(resolve, reject) => {
    const form = new multiparty.Form();
    await form.parse(req, (err, fields, files) => {
      if (err) reject(err);
      const bucketname = fields.bucketname[0];
      const subfoldername = fields.subfoldername[0];
      const file = files["file"][0]; // get the file from the returned files object
      if (!file) reject("File was not found in form data.");
      else resolve({
        file,
        bucketname,
        subfoldername
      });
    });
  });

/**
 * Helper method which takes the request object and returns a promise with the AWS S3 object details.
 */
const uploadFileToS3Bucket = (
  file,
  bucketname,
  subfoldername,
  options = {}
) => {
  const s3 = new AWS.S3();

  // turn the file into a buffer for uploading
  const buffer = readFileSync(file.path);

  var originalname = file.originalFilename;
  var attach_split = originalname.split(".");
  var name = attach_split[0];
  // generate a new random file name
  const fileName = name;

  // the extension of your file
  const extension = extname(file.path);

  console.log(`${fileName}${extension}`);

  const params = {
    Bucket: bucketname, //Bucketname
    ACL: "private", //Permission
    Key: join(`${subfoldername}/`, `${fileName}${extension}`), // File name you want to save as in S3
    Body: buffer, // Content of file
  };

  // return a promise
  return new Promise((resolve, reject) => {
    return s3.upload(params, (err, result) => {
      if (err) reject(err);
      else resolve(result); // return the values of the successful AWS S3 request
    });
  });
};

router.post('/', upload.single('file'), async(req, res) => {
  try {
    // extract the file from the request object
    const {
      file,
      bucketname,
      subfoldername
    } = await getDataFromRequest(req);

    // Upload File to specified bucket
    const {
      Location,
      ETag,
      Bucket,
      Key
    } = await uploadFileToS3Bucket(
      file,
      bucketname,
      subfoldername
    );

    let response = {};
    res["Location"] = Location;
    response["ETag"] = ETag;
    response["Bucket"] = Bucket;
    response["Key"] = Key;

    res.status(200).json(response);
  } catch (error) {
    throw error;
  }

});

Request body will be form data with following fields

  1. bucketname:
  2. subfoldername:
  3. file: FileData
user_126891140

For anyone that ever stumbles across this question the solution was to create an custom multer storage engine. Inside the engine you get access to the file with a stream property that s3 accepted (with the correct headers).

user_166459900
Related