I have a notification on an S3 bucket upload to place a message in an SQS queue. The SQS queue triggers a lambda function. I am trying to extract the content of the S3 object after a notification message is sent to SQS message. when the notification message arrives on SQS, a lambda is triggered and it should extract the contents of the file.
Following is the SQS response
{
"Records": [
{
"messageId": "19dd0b57-b21e-4ac1-bd88-01bbb068cb78",
"receiptHandle": "MessageReceiptHandle",
"body": {
"Records": [
{
"eventVersion": "2.0",
"eventSource": "aws:s3",
"awsRegion": "us-east-1",
"eventTime": "1970-01-01T00:00:00.000Z",
"eventName": "ObjectCreated:Put",
"userIdentity": {
"principalId": "EXAMPLE"
},
"requestParameters": {
"sourceIPAddress": "127.0.0.1"
},
"responseElements": {
"x-amz-request-id": "EXAMPLE123456789",
"x-amz-id-2": "EXAMPLE123/5678abcdefghijklambdaisawesome/mnopqrstuvwxyzABCDEFGH"
},
"s3": {
"s3SchemaVersion": "1.0",
"configurationId": "testConfigRule",
"bucket": {
"name": "example-bucket",
"ownerIdentity": {
"principalId": "EXAMPLE"
},
"arn": "arn:aws:s3:::example-bucket"
},
"object": {
"key": "access_log_20220908-102154.txt",
"size": 1024,
"eTag": "0123456789abcdef0123456789abcdef",
"sequencer": "0A1B2C3D4E5F678901"
}
}
}
]
},
"attributes": {
"ApproximateReceiveCount": "1",
"SentTimestamp": "1523232000000",
"SenderId": "123456789012",
"ApproximateFirstReceiveTimestamp": "1523232000001"
},
"messageAttributes": {},
"md5OfBody": "{{{md5_of_body}}}",
"eventSource": "aws:sqs",
"eventSourceARN": "arn:aws:sqs:us-east-1:XXXX:s3-OS-queue",
"awsRegion": "us-east-1"
}
]
}
Here is the code:
def lambda_handler(event, context):
#Loops through every file uploaded
for record in event['Records']:
#pull the body out & json load it
json_record=(record['body'])
try:
print(json_record)
except botocore.exceptions.ClientError as error:
print("raised error")
raise error
bucket = json_record['Records'][0]["s3"]["bucket"]["name"]
print(bucket)
key=json_record["Records"][0]["s3"]["object"]["key"]
print(key)
obj = s3.get_object(Bucket=bucket, Key=key)
print(obj)
#following part fails
body = obj['Body'].read()
print(body)
lines = body.splitlines()
print(lines)
```
printing the object returns a response metadata
```{'ResponseMetadata': {'RequestId': 'T2SAR2GW7WRP4B13', 'HostId': 'QVkghmfbKfXfiAP08SOeo9VDu9+HJxb+gs39/pBxDskJwcYVzdajDRWqhYdvJOo1/rNhMicWFvQ=', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amz-id-2': 'QVkghmfbKfXfiAP08SOeo9VDu9+HJxb+gs39/pBxDskJwcYVzdajDRWqhYdvJOo1/rNhMicWFvQ=', 'x-amz-request-id': 'T2SAR2GW7WRP4B13', 'date': 'Thu, 15 Sep 2022 23:49:15 GMT', 'last-modified': 'Thu, 08 Sep 2022 22:49:18 GMT', 'etag': '"c9bfa5ac6d2c2a2d19ec1544d564c277"', 'accept-ranges': 'bytes', 'content-type': 'text/plain', 'server': 'AmazonS3', 'content-length': '355891'}, 'RetryAttempts': 0}, 'AcceptRanges': 'bytes', 'LastModified': datetime.datetime(2022, 9, 8, 22, 49, 18, tzinfo=tzutc()), 'ContentLength': 355891, 'ETag': '"c9bfa5ac6d2c2a2d19ec1544d564c277"', 'ContentType': 'text/plain', 'Metadata': {}, 'Body': <botocore.response.StreamingBody object at 0x7fd20edbf250>}
error message
TypeError: string indices must be integers
Traceback (most recent call last):
File "/var/task/sample-bulk.py", line 42, in lambda_handler
bucket = json_record['Records'][0]["s3"]["bucket"]["name"]
It looks like get_object returns a object and key as a string which is not being accepted. Just for context, the object is a file that contains Apache access log. the goal is to eventually ingest the contents of that file to an index in OpenSearch