Dynamodb - batchWrite not working(nodejs)

Viewed 1769

I am using lambda, API gateway and dynamodb

I am creating a batchWrite function to upload more than 25 items in one single function.

But I find that when I upload 35 items, the first 25 items are uploaded successfully to dynamodb, but last 10 items fail but no error is logged.

The function is written in node.js

'use strict'
const AWS = require('aws-sdk');
const sha1 = require('sha1');
const documentClient = new AWS.DynamoDB.DocumentClient();

exports.handler = async function (event, context, callback) {

    let responseBody = "";
    let statusCode = 0;

    const { excelObject } = JSON.parse(event.body);

    if(excelObject){
        await uploadFileByBatch(excelObject)
    }

    const response = {
        statusCode: statusCode,
        headers:{
            "Content-Type": "application/json",
            "access-control-allow-origin": "*"
        },
        body: responseBody
    }

    console.log(response)
    return response
}

let uploadFileByBatch = async function (payload) {

    var items = [];
    var params={};

    if(payload.length > 25){
        console.log("> 25")
        let newObj = payload.slice(0,25)
        payload.splice(0,25)

        console.log("new Payload length")
        console.log(payload.length)

        newObj.forEach(obj =>{
            let hash = sha1(Buffer.from(new Date().toString()+ Math.random()));
            items.push(
                {
                    PutRequest:{
                        Item: {
                            id: obj.id?obj.id:hash,
                            organization_EN: obj.organization_EN,
                            email: obj.email,
                            isActive: obj.isActive
                        }
                    }
                }
            )
        })

        params = {
            RequestItems:{
                "Community": items
            }
        }
        console.log(params)
        try{
            const data = await documentClient.batchWrite(params).promise();
        }catch(err){
            console.log(err)
        }
        uploadFileByBatch(payload)
    }else{
        console.log("less than 25")
        payload.forEach(obj =>{
            let hash = sha1(Buffer.from(new Date().toString()+ Math.random()));
            items.push(
                {
                    PutRequest:{
                        Item: {
                            id: obj.id?obj.id:hash,
                            organization_EN: obj.organization_EN,
                            email: obj.email,
                            isActive: obj.isActive
                        }
                    }
                }
            )
        })
        params = {
            RequestItems:{
                "Community": items
            }
        }
        console.log(params)
        try{
            console.log("first BatchWrite")
            const data = await documentClient.batchWrite(params).promise();
            console.log("first Data")
        }catch(err){
            console.log(err)
        }
    }

}

The log is like below.

>25
New Payload length
10

{
  RequestItems: {
    'Community': [
      [Object], [Object], [Object],
      [Object], [Object], [Object],
      [Object], [Object], [Object],
      [Object], [Object], [Object],
      [Object], [Object], [Object],
      [Object], [Object], [Object],
      [Object], [Object], [Object],
      [Object], [Object], [Object],
      [Object]
    ]
  }
}

first BatchWrite
Less than 25
{
  RequestItems: {
    'Community': [
      [Object], [Object],
      [Object], [Object],
      [Object], [Object],
      [Object], [Object],
      [Object], [Object]
    ]
  }
}

{
  statusCode: 0,
  headers: {
    'Content-Type': 'application/json',
    'access-control-allow-origin': '*'
  },
  body: ''
}
1 Answers

You cannot upload more than 25 in a single request for batchWrite function with DynamoDB

If one or more of the following is true, DynamoDB rejects the entire batch write operation:

  • One or more tables specified in the BatchWriteItem request does not exist.
  • Primary key attributes specified on an item in the request do not match those in the corresponding table's primary key schema.
  • You try to perform multiple operations on the same item in the same BatchWriteItem request. For example, you cannot put and delete the same item in the same BatchWriteItem request.
  • Your request contains at least two items with identical hash and range keys (which essentially is two put operations).
  • There are more than 25 requests in the batch.
  • Any individual item in a batch exceeds 400 KB.
  • The total request size exceeds 16 MB.

You can retrieve informations about this here https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_BatchWriteItem.html

There are ways to handle big batch operations, you can maybe do parallels requests of 25, but you should be careful with it.

Related