print s3 object content through lambda

Viewed 25

I have a notification on an S3 bucket upload to place a message in an SQS queue. The SQS queue triggers a lambda function. I am trying to extract the content of the S3 object after a notification message is sent to SQS message. when the notification message arrives on SQS, a lambda is triggered and it should extract the contents of the file.

Following is the SQS response

{
  "Records": [
    {
      "messageId": "19dd0b57-b21e-4ac1-bd88-01bbb068cb78",
      "receiptHandle": "MessageReceiptHandle",
      "body": {
        "Records": [
          {
            "eventVersion": "2.0",
            "eventSource": "aws:s3",
            "awsRegion": "us-east-1",
            "eventTime": "1970-01-01T00:00:00.000Z",
            "eventName": "ObjectCreated:Put",
            "userIdentity": {
              "principalId": "EXAMPLE"
            },
            "requestParameters": {
              "sourceIPAddress": "127.0.0.1"
            },
            "responseElements": {
              "x-amz-request-id": "EXAMPLE123456789",
              "x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH"
            },
            "s3": {
              "s3SchemaVersion": "1.0",
              "configurationId": "testConfigRule",
              "bucket": {
                "name": "example-bucket",
                "ownerIdentity": {
                  "principalId": "EXAMPLE"
                },
                "arn": "arn:aws:s3:::example-bucket"
              },
              "object": {
                "key": "access_log_20220908-102154.txt",
                "size": 1024,
                "eTag": "0123456789abcdef0123456789abcdef",
                "sequencer": "0A1B2C3D4E5F678901"
              }
            }
          }
        ]
      },
      "attributes": {
        "ApproximateReceiveCount": "1",
        "SentTimestamp": "1523232000000",
        "SenderId": "123456789012",
        "ApproximateFirstReceiveTimestamp": "1523232000001"
      },
      "messageAttributes": {},
      "md5OfBody": "{{{md5_of_body}}}",
      "eventSource": "aws:sqs",
      "eventSourceARN": "arn:aws:sqs:us-east-1:XXXX:s3-OS-queue",
      "awsRegion": "us-east-1"
    }
  ]
}

Here is the code:

def lambda_handler(event, context):
    
    #Loops through every file uploaded
    for record in event['Records']:
        #pull the body out & json load it
        json_record=(record['body'])

                
        try:
            print(json_record)
        except botocore.exceptions.ClientError as error:
            print("raised error")
            raise error
        

        bucket = json_record['Records'][0]["s3"]["bucket"]["name"]
        print(bucket)
        key=json_record["Records"][0]["s3"]["object"]["key"]
        print(key)
        
        
        obj = s3.get_object(Bucket=bucket, Key=key)
        print(obj)  
        
        #following part fails
        body = obj['Body'].read()
        print(body)
        lines = body.splitlines()
        print(lines)
      ```

printing the object returns a response metadata
```{'ResponseMetadata': {'RequestId': 'T2SAR2GW7WRP4B13', 'HostId': 'QVkghmfbKfXfiAP08SOeo9VDu9+HJxb+gs39/pBxDskJwcYVzdajDRWqhYdvJOo1/rNhMicWFvQ=', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amz-id-2': 'QVkghmfbKfXfiAP08SOeo9VDu9+HJxb+gs39/pBxDskJwcYVzdajDRWqhYdvJOo1/rNhMicWFvQ=', 'x-amz-request-id': 'T2SAR2GW7WRP4B13', 'date': 'Thu, 15 Sep 2022 23:49:15 GMT', 'last-modified': 'Thu, 08 Sep 2022 22:49:18 GMT', 'etag': '"c9bfa5ac6d2c2a2d19ec1544d564c277"', 'accept-ranges': 'bytes', 'content-type': 'text/plain', 'server': 'AmazonS3', 'content-length': '355891'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2022, 9, 8, 22, 49, 18, tzinfo=tzutc()), 'ContentLength': 355891, 'ETag': '"c9bfa5ac6d2c2a2d19ec1544d564c277"', 'ContentType': 'text/plain', 'Metadata': {}, 'Body': <botocore.response.StreamingBody object at 0x7fd20edbf250>}

error message

TypeError: string indices must be integers
Traceback (most recent call last):
  File "/var/task/sample-bulk.py", line 42, in lambda_handler
    bucket = json_record['Records'][0]["s3"]["bucket"]["name"]

It looks like get_object returns a object and key as a string which is not being accepted. Just for context, the object is a file that contains Apache access log. the goal is to eventually ingest the contents of that file to an index in OpenSearch

0 Answers
Related