fetch image from URL to S3

One common issue in those cases is a permission error. To allow AWS Lambda to PutObject in S3, you need to set such permissions in the Lambda Execution Role.

Here is an example of a policy that will allow Lambda to do anything in S3:

{
    "Version": "2012-10-17",
    "Statement": [
{
            "Effect": "Allow",
            "Action": [
                "s3:ListAllMyBuckets",
                "s3:GetBucketLocation"
            ],
            "Resource": "*"
        },
        {
            "Effect": "Allow",
            "Action": "s3:*",
            "Resource": [
                "arn:aws:s3:::ExampleBucketName",
                "arn:aws:s3:::ExampleBucketName/*"
            ]
        }
    ]
}

For security purposes, it is important to restrict the action to only the ones you actually need, such as PutObject, and you can get more information on this link.

Here's an example of how to stream a file from an HTTP URL into S3. It uses promises/async/await rather than callbacks and it dispenses with the Jimp package, about which I know little, in favor of the more traditional fetch API:

Note: if you do not explicitly supply a content-type when uploading to S3 then it will be set to application/octet-stream which will be problematic when clients are downloading the object. So this code determines the content type of the file first, and sets it when streaming to S3.

const AWS = require('aws-sdk');
const fetch = require('node-fetch');
const stream = require('stream');
const s3 = new AWS.S3();

const uploadStream = ({ Bucket, Key, ContentType }) => {
  const pass = new stream.PassThrough();
  return {
    writeStream: pass,
    promise: s3.upload({ Bucket, Key, ContentType, Body: pass }).promise(),
  };
}

const uploadFetch = async ({ url, Bucket, Key, ContentType }) => {
  const response = await fetch(url);
  const { writeStream, promise } = uploadStream({Bucket, Key, ContentType});
  response.body.pipe(writeStream);
  return promise;
}

exports.handler = async (_event, _context) => {
  const source_jpeg = {
    Key: 'audi.jpeg',
    Bucket: 'mybucket',
    url: 'https://upload.wikimedia.org/wikipedia/commons/0/08/Audi_A3_2015.jpeg',
  };

  // HEAD the source image to get content type
  const rc_head = await fetch(source_jpeg.url, {method: 'HEAD'});
  const content_type = rc_head.headers.get('content-type');
  console.log('head:', rc_head.status, rc_head.statusText, content_type);

  try {
    // GET the source image and stream it to S3
    const parms = {...source_jpeg, ContentType: content_type};
    const rc_upload = await uploadFetch(parms);
    console.log('get/upload jpeg:', rc_upload);
  } catch(e) {
    console.log(e);
  }
};

Also, be sure that your Lambda function is configured with a reasonable timeout (the default timeout is 3 seconds).

If this helps anyone else the image needed to be written to a buffer. The following line fixed it:

const buffer = await image.getBufferAsync(imageType);

Then buffer is used for the S3 Body parameter. so the full script is:

    "use strict";

    const AWS = require("aws-sdk");
    const Jimp = require("jimp");
    const s3 = new AWS.S3();
    const imageType = "image/jpeg";
    const bucket = 'bucketxzy';

    exports.handler = async (event, context) => {
        let objectKey = 'sampleimage.jpeg';
        const image = await Jimp.read('https://www.sampleurl.com/sampleimage.jpg/');
        const buffer = await image.getBufferAsync(imageType);

        return s3.putObject({
            Bucket: bucket,
            Key: objectKey,
            Body: buffer,
            ContentType: imageType
        }).promise();


    };