python json serialization error on returning the output result

user_662285 0
Viewed 278

I am getting error on last line of this code. I am doing data transformation using python in AWS using lambda function. Weird part is it works absolutely fine in my local AWS account but fails on client system on the last line with below error.

import json
import boto3
import base64
import string
from typing import Optional, Iterable, Union

delete_dict = {sp_character: '' for sp_character in string.punctuation}
PUNCT_TABLE = str.maketrans(delete_dict)
output = []

def lambda_handler(event, context):
    
    for record in event['records']:
        payload = base64.b64decode(record['data']).decode('utf-8')
        
        remove_special_char = json.loads(payload, object_pairs_hook=clean_keys)
        row_w_newline = str(remove_special_char) + "\n"
        row_w_newline = base64.b64encode(row_w_newline.encode('utf-8'))
        
        
        output_record = {
            'recordId': record['recordId'],
            'result': 'Ok',
            'data': row_w_newline
        }
        output.append(output_record)

    print('Processed {} records.'.format(len(event['records'])))
    
    return {'records': output}

This line gives below error :

return {'records': output}

Error:

An error occurred during JSON serialization of response: 

Traceback (most recent call last):
  File "/var/lang/lib/python3.6/json/__init__.py", line 238, in dumps
    **kw).encode(obj)
  File "/var/lang/lib/python3.6/json/encoder.py", line 199, in encode
    chunks = self.iterencode(o, _one_shot=True)
  File "/var/lang/lib/python3.6/json/encoder.py", line 257, in iterencode
    return _iterencode(o, 0)
  File "/var/runtime/bootstrap.py", line 135, in decimal_serializer
    raise TypeError(repr(o) + " is not JSON serializable")

Output:

[{'recordId': '49624207526699472828556941664751397737727737843643580418000000', 'result': 'Ok', 'data': b'eyd2ZXJzaW9uJzogJzAnLCAnaWQnOiAnMTExMTE3NzItZjMwYi00OTFjLTZiYmItMWYzYmFkM2FmM2Y5JywgJ2RldGFpbHR5cGUnOiAnQVdTIEFQSSBDYWxsIHZpYSBDbG91ZFRyYWlsJywgJ3NvdXJjZSc6ICdhd3Muc2VjdXJpdHlodWInLCAnYWNjb3VudCc6ICc5MzQ3NTU5ODkxNzYnLCAndGltZSc6ICcyMDIxLTExLTIzVDIwOjQ2OjI2WicsICdyZWdpb24nOiAndXMtZWFzdC0xJywgJ3Jlc291cmNlcyc6IFtdLCAnZGV0YWlsJzogeydldmVudFZlcnNpb24nOiAnMS4wOCcsICd1c2VySWRlbnRpdHknOiB7J3R5cGUnOiAnQXNzdW1lZFJvbGUnLCAncHJpbmNpcGFsSWQnOiAnQVJPQTVUSTZNTzI0SFVLQUJFTTc2OnNndXB0YTE2JywgJ2Fybic6ICdhcm46YXdzOnN0czo6OTM0NzU1OTg5MTc2OmFzc3VtZWQtcm9sZS9VRU5UQVdTR1RTU0VDVVJJVFlIVUJFWS9zZ3VwdGExNicsICdhY2NvdW50SWQnOiAnOTM0NzU1OTg5MTc2JywgJ2FjY2Vzc0tleUlkJzogJ0FTSUE1VEk2TU8yNExSSENPWEdEJywgJ3Nlc3Npb25Db250ZXh0JzogeydzZXNzaW9uSXNzdWVyJzogeyd0eXBlJzogJ1JvbGUnLCAncHJpbmNpcGFsSWQnOiAnQVJPQTVUSTZNTzI0SFVLQUJFTTc2JywgJ2Fybic6ICdhcm46YXdzOmlhbTo6OTM0NzU1OTg5MTc2OnJvbGUvVUVOVEFXU0dUU1NFQ1VSSVRZSFVCRVknLCAnYWNjb3VudElkJzogJzkzNDc1NTk4OTE3NicsICd1c2VyTmFtZSc6ICdVRU5UQVdTR1RTU0VDVVJJVFlIVUJFWSd9LCAnd2ViSWRGZWRlcmF0aW9uRGF0YSc6IHt9LCAnYXR0cmlidXRlcyc6IHsnY3JlYXRpb25EYXRlJzogJzIwMjEtMTEtMjNUMTg6NDk6MDVaJywgJ21mYUF1dGhlbnRpY2F0ZWQnOiAnZmFsc2UnfX19LCAnZXZlbnRUaW1lJzogJzIwMjEtMTEtMjNUMjA6NDY6MjZaJywgJ2V2ZW50U291cmNlJzogJ3NlY3VyaXR5aHViLmFtYXpvbmF3cy5jb20nLCAnZXZlbnROYW1lJzogJ0JhdGNoRGlzYWJsZVN0YW5kYXJkcycsICdhd3NSZWdpb24nOiAndXMtZWFzdC0xJywgJ3NvdXJjZUlQQWRkcmVzcyc6ICcxMzcuMTE3LjQzLjE0NCcsICd1c2VyQWdlbnQnOiAnYXdzLWludGVybmFsLzMgYXdzLXNkay1qYXZhLzEuMTIuMTEyIExpbnV4LzUuNC4xNDctODMuMjU5LmFtem4yaW50Lng4Nl82NCBPcGVuSkRLXzY0LUJpdF9TZXJ2ZXJfVk0vMjUuMzEyLWIwNyBqYXZhLzEuOC4wXzMxMiB2ZW5kb3IvT3JhY2xlX0NvcnBvcmF0aW9uIGNmZy9yZXRyeS1tb2RlL3N0YW5kYXJkJywgJ3JlcXVlc3RQYXJhbWV0ZXJzJzogeydTdGFuZGFyZHNTdWJzY3JpcHRpb25Bcm5zJzogWydhcm46YXdzOnNlY3VyaXR5aHViOnVzLWVhc3QtMTo5MzQ3NTU5ODkxNzY6c3Vic2NyaXB0aW9uL2F3cy1mb3VuZGF0aW9uYWwtc2VjdXJpdHktYmVzdC1wcmFjdGljZXMvdi8xLjAuMCddfSwgJ3Jlc3BvbnNlRWxlbWVudHMnOiB7J1N0YW5kYXJkc1N1YnNjcmlwdGlvbnMnOiBbeydTdGFuZGFyZHNBcm4nOiAnYXJuOmF3czpzZWN1cml0eWh1Yjp1cy1lYXN0LTE6OnN0YW5kYXJkcy9hd3MtZm91bmRhdGlvbmFsLXNlY3VyaXR5LWJlc3QtcHJhY3RpY2VzL3YvMS4wLjAnLCAnU3RhbmRhcmRzSW5wdXQnOiB7fSwgJ1N0YW5kYXJkc1N0YXR1cyc6ICdERUxFVElORycsICdTdGFuZGFyZHNTdWJzY3JpcHRpb25Bcm4nOiAnYXJuOmF3czpzZWN1cml0eWh1Yjp1cy1lYXN0LTE6OTM0NzU1OTg5MTc2OnN1YnNjcmlwdGlvbi9hd3MtZm91bmRhdGlvbmFsLXNlY3VyaXR5LWJlc3QtcHJhY3RpY2VzL3YvMS4wLjAnLCAnU3RhbmRhcmRzU3RhdHVzUmVhc29uJzogeydTdGF0dXNSZWFzb25Db2RlJzogJ0lOVEVSTkFMX0VSUk9SJ319XX0sICdyZXF1ZXN0SUQnOiAnMDAyYzQ1YzItNmYxYy00YTA5LWE5YTctZjQ1OTY5MWFiYzRjJywgJ2V2ZW50SUQnOiAnYTUzZTA1NjktYjYwOC00Nzg5LThlNzctNGNlYjhlNTg2YzliJywgJ3JlYWRPbmx5JzogRmFsc2UsICdldmVudFR5cGUnOiAnQXdzQXBpQ2FsbCcsICdtYW5hZ2VtZW50RXZlbnQnOiBUcnVlLCAncmVjaXBpZW50QWNjb3VudElkJzogJzkzNDc1NTk4OTE3NicsICdldmVudENhdGVnb3J5JzogJ01hbmFnZW1lbnQnfX0K'}

Edit : Updated

I have updated my code as below and now i don't receive any error but the output has backward slash () after each and every key/value. how to avoid this ?

for record in event['records']:
        payload = base64.b64decode(record['data']).decode('utf-8')       
        output_record = {
            'recordId': record['recordId'],
            'result': 'Ok',
            'data': base64.b64encode(json.dumps(payload).encode('utf-8') + b'\n').decode('utf-8')
        }
        output.append(output_record)  
 
    return {'records': output}

Output :

"{\"version\":\"0\",\"id\":\"bcdfaff3-ff56-b1ea-a341-0f0ed44380c6\",\"detail-type\":\"Security Hub Findings - Imported\",\"source\":\"aws.securityhub\",\"account\":\"934755989176\",\"time\":\"2021-11-24T16:47:00Z\",\"region\":\"us-east-1\",\"resources\":[\"arn:aws:securityhub:us-east-1::product/aws/securityhub/arn:aws:securityhub:us-east-1:934755989176:subscription/aws-foundational-security-best-practices/v/1.0.0/IAM.7/finding/3fbf88ba-b228-4caa-a4aa-5bee36b1b007\"],\"detail\":{\"findings\":[{\"ProductArn\":\"arn:aws:securityhub:us-east-1::product/aws/securityhub\",\"Types\":[\"Software and Configuration Checks/Industry and Regulatory Standards/AWS-Foundational-Security-Best-Practices\"],\"Description\":\"This AWS control checks whether the account password policy for IAM users uses the following recommended configurations: RequireUppercaseCharacters: true, RequireLowercaseCharacters: true, RequireSymbols: true, RequireNumbers: true, MinimumPasswordLength: 8.\",\"Compliance\":{\"Status\":\"PASSED\"},\"ProductName\":\"Security Hub\",\"FirstObservedAt\":\"2021-11-18T14:03:23.999Z\",\"CreatedAt\":\"2021-11-18T14:03:23.999Z\",\"LastObservedAt\":\"2021-11-24T16:46:54.212Z\",\"CompanyName\":\"AWS\",\"FindingProviderFields\":{\"Types\":[\"Software and Configuration Checks/Industry and Regulatory Standards/AWS-Foundational-Security-Best-Practices\"],\"Severity\":{\"Normalized\":0,\"Label\":\"INFORMATIONAL\",\"Product\":0,\"Original\":\"INFORMATIONAL\"}},\"ProductFields\":{\"StandardsArn\":\"arn:aws:securityhub:::standards/aws-foundational-security-best-practices/v/1.0.0\",\"StandardsSubscriptionArn\":\"arn:aws:securityhub:us-east-1:934755989176:subscription/aws-foundational-security-best-practices/v/1.0.0\",\"ControlId\":\"IAM.7\",\"RecommendationUrl\":\"https://docs.aws.amazon.com/console/securityhub/IAM.7/remediation\",\"RelatedAWSResources:0/name\":\"securityhub-iam-password-policy-recommended-defaults-35610a74\",\"RelatedAWSResources:0/type\":\"AWS::Config::ConfigRule\",\"StandardsControlArn\":\"arn:aws:securityhub:us-east-1:934755989176:control/aws-foundational-security-best-practices/v/1.0.0/IAM.7\",\"aws/securityhub/ProductName\":\"Security Hub\",\"aws/securityhub/CompanyName\":\"AWS\",\"Resources:0/Id\":\"arn:aws:iam::934755989176:root\",\"aws/securityhub/FindingId\":\"arn:aws:securityhub:us-east-1::product/aws/securityhub/arn:aws:securityhub:us-east-1:934755989176:subscription/aws-foundational-security-best-practices/v/1.0.0/IAM.7/finding/3fbf88ba-b228-4caa-a4aa-5bee36b1b007\"},\"Remediation\":{\"Recommendation\":{\"Text\":\"For directions on how to fix this issue, consult the AWS Security Hub Foundational Security Best Practices documentation.\",\"Url\":\"https://docs.aws.amazon.com/console/securityhub/IAM.7/remediation\"}},\"SchemaVersion\":\"2018-10-08\",\"GeneratorId\":\"aws-foundational-security-best-practices/v/1.0.0/IAM.7\",\"RecordState\":\"ACTIVE\",\"Title\":\"IAM.7 Password policies for IAM users should have strong configurations\",\"Workflow\":{\"Status\":\"RESOLVED\"},\"Severity\":{\"Normalized\":0,\"Label\":\"INFORMATIONAL\",\"Product\":0,\"Original\":\"INFORMATIONAL\"},\"UpdatedAt\":\"2021-11-24T16:46:49.775Z\",\"WorkflowState\":\"NEW\",\"AwsAccountId\":\"934755989176\",\"Region\":\"us-east-1\",\"Id\":\"arn:aws:securityhub:us-east-1:934755989176:subscription/aws-foundational-security-best-practices/v/1.0.0/IAM.7/finding/3fbf88ba-b228-4caa-a4aa-5bee36b1b007\",\"Resources\":[{\"Partition\":\"aws\",\"Type\":\"AwsAccount\",\"Region\":\"us-east-1\",\"Id\":\"AWS::::Account:934755989176\"}]}]}}"
python
2 Answers

This error occurs because you are sending a binary string in your json output when the method json.dumps is called it complains that doesn't know how to convert the string as text. I reproduced your error wiht this code:

import json
a = {'text':b'foovar'}
json.dumps(a)

Maybe you could just decode or encode your data when you send it or recieve it something like this

import json
a = {'text':b'foovar'.decode('ascii')}
json.dumps(a)

and in your client encode it and use base64 decode.

Your lambda function transform under the hood your response dictionary to json to do that it calls json.dumps.

Final solution

just to replicate your actual case i created this code snippet:

test = {'recordId': '49624207526699472828556941664751397737727737843643580418000000', 'result': 'Ok', 'data': b'eyd2ZXJzaW9uJzogJzAnLCAnaWQnOiAnMTExMTE3NzItZjMwYi00OTFjLTZiYmItMWYzYmFkM2FmM2Y5JywgJ2RldGFpbHR5cGUnOiAnQVdTIEFQSSBDYWxsIHZpYSBDbG91ZFRyYWlsJywgJ3NvdXJjZSc6ICdhd3Muc2VjdXJpdHlodWInLCAnYWNjb3VudCc6ICc5MzQ3NTU5ODkxNzYnLCAndGltZSc6ICcyMDIxLTExLTIzVDIwOjQ2OjI2WicsICdyZWdpb24nOiAndXMtZWFzdC0xJywgJ3Jlc291cmNlcyc6IFtdLCAnZGV0YWlsJzogeydldmVudFZlcnNpb24nOiAnMS4wOCcsICd1c2VySWRlbnRpdHknOiB7J3R5cGUnOiAnQXNzdW1lZFJvbGUnLCAncHJpbmNpcGFsSWQnOiAnQVJPQTVUSTZNTzI0SFVLQUJFTTc2OnNndXB0YTE2JywgJ2Fybic6ICdhcm46YXdzOnN0czo6OTM0NzU1OTg5MTc2OmFzc3VtZWQtcm9sZS9VRU5UQVdTR1RTU0VDVVJJVFlIVUJFWS9zZ3VwdGExNicsICdhY2NvdW50SWQnOiAnOTM0NzU1OTg5MTc2JywgJ2FjY2Vzc0tleUlkJzogJ0FTSUE1VEk2TU8yNExSSENPWEdEJywgJ3Nlc3Npb25Db250ZXh0JzogeydzZXNzaW9uSXNzdWVyJzogeyd0eXBlJzogJ1JvbGUnLCAncHJpbmNpcGFsSWQnOiAnQVJPQTVUSTZNTzI0SFVLQUJFTTc2JywgJ2Fybic6ICdhcm46YXdzOmlhbTo6OTM0NzU1OTg5MTc2OnJvbGUvVUVOVEFXU0dUU1NFQ1VSSVRZSFVCRVknLCAnYWNjb3VudElkJzogJzkzNDc1NTk4OTE3NicsICd1c2VyTmFtZSc6ICdVRU5UQVdTR1RTU0VDVVJJVFlIVUJFWSd9LCAnd2ViSWRGZWRlcmF0aW9uRGF0YSc6IHt9LCAnYXR0cmlidXRlcyc6IHsnY3JlYXRpb25EYXRlJzogJzIwMjEtMTEtMjNUMTg6NDk6MDVaJywgJ21mYUF1dGhlbnRpY2F0ZWQnOiAnZmFsc2UnfX19LCAnZXZlbnRUaW1lJzogJzIwMjEtMTEtMjNUMjA6NDY6MjZaJywgJ2V2ZW50U291cmNlJzogJ3NlY3VyaXR5aHViLmFtYXpvbmF3cy5jb20nLCAnZXZlbnROYW1lJzogJ0JhdGNoRGlzYWJsZVN0YW5kYXJkcycsICdhd3NSZWdpb24nOiAndXMtZWFzdC0xJywgJ3NvdXJjZUlQQWRkcmVzcyc6ICcxMzcuMTE3LjQzLjE0NCcsICd1c2VyQWdlbnQnOiAnYXdzLWludGVybmFsLzMgYXdzLXNkay1qYXZhLzEuMTIuMTEyIExpbnV4LzUuNC4xNDctODMuMjU5LmFtem4yaW50Lng4Nl82NCBPcGVuSkRLXzY0LUJpdF9TZXJ2ZXJfVk0vMjUuMzEyLWIwNyBqYXZhLzEuOC4wXzMxMiB2ZW5kb3IvT3JhY2xlX0NvcnBvcmF0aW9uIGNmZy9yZXRyeS1tb2RlL3N0YW5kYXJkJywgJ3JlcXVlc3RQYXJhbWV0ZXJzJzogeydTdGFuZGFyZHNTdWJzY3JpcHRpb25Bcm5zJzogWydhcm46YXdzOnNlY3VyaXR5aHViOnVzLWVhc3QtMTo5MzQ3NTU5ODkxNzY6c3Vic2NyaXB0aW9uL2F3cy1mb3VuZGF0aW9uYWwtc2VjdXJpdHktYmVzdC1wcmFjdGljZXMvdi8xLjAuMCddfSwgJ3Jlc3BvbnNlRWxlbWVudHMnOiB7J1N0YW5kYXJkc1N1YnNjcmlwdGlvbnMnOiBbeydTdGFuZGFyZHNBcm4nOiAnYXJuOmF3czpzZWN1cml0eWh1Yjp1cy1lYXN0LTE6OnN0YW5kYXJkcy9hd3MtZm91bmRhdGlvbmFsLXNlY3VyaXR5LWJlc3QtcHJhY3RpY2VzL3YvMS4wLjAnLCAnU3RhbmRhcmRzSW5wdXQnOiB7fSwgJ1N0YW5kYXJkc1N0YXR1cyc6ICdERUxFVElORycsICdTdGFuZGFyZHNTdWJzY3JpcHRpb25Bcm4nOiAnYXJuOmF3czpzZWN1cml0eWh1Yjp1cy1lYXN0LTE6OTM0NzU1OTg5MTc2OnN1YnNjcmlwdGlvbi9hd3MtZm91bmRhdGlvbmFsLXNlY3VyaXR5LWJlc3QtcHJhY3RpY2VzL3YvMS4wLjAnLCAnU3RhbmRhcmRzU3RhdHVzUmVhc29uJzogeydTdGF0dXNSZWFzb25Db2RlJzogJ0lOVEVSTkFMX0VSUk9SJ319XX0sICdyZXF1ZXN0SUQnOiAnMDAyYzQ1YzItNmYxYy00YTA5LWE5YTctZjQ1OTY5MWFiYzRjJywgJ2V2ZW50SUQnOiAnYTUzZTA1NjktYjYwOC00Nzg5LThlNzctNGNlYjhlNTg2YzliJywgJ3JlYWRPbmx5JzogRmFsc2UsICdldmVudFR5cGUnOiAnQXdzQXBpQ2FsbCcsICdtYW5hZ2VtZW50RXZlbnQnOiBUcnVlLCAncmVjaXBpZW50QWNjb3VudElkJzogJzkzNDc1NTk4OTE3NicsICdldmVudENhdGVnb3J5JzogJ01hbmFnZW1lbnQnfX0K'}
test['data'] = test['data'].decode('utf-8')
json.dumps(test)

works fine. I would recommend to read some examples on serialization if you are struggling with this

user_30585090

This code helped to resolve above issue

def lambda_handler(event, context):
    
    for record in event['records']:
        payload = base64.b64decode(record['data']).decode('utf-8')
        remove_special_char = json.loads(payload, object_pairs_hook=clean_keys)
        output_record = {
            'recordId': record['recordId'],
            'result': 'Ok',
            'data': base64.b64encode(json.dumps(remove_special_char).encode('utf-8') + b'\n').decode('utf-8')
        }
        output.append(output_record)

    print('Processed {} records.'.format(len(event['records'])))
    return {'records': output}
user_6622850
Related