I am trying to createReadStream and concat into as Buffer, following by feeding into aws-sdk to do a multipart upload to S3 Glacier, however I faced the error with a large filesize, any size > 30GB - 2.5TB.
RangeError [ERR_INVALID_ARG_VALUE]: The argument 'size' is invalid. Received 32212254720
at Function.allocUnsafe (node:buffer:373:3)
at Function.concat (node:buffer:552:25)
My code
'use strict';
const AWS = require("aws-sdk");
const path = require("path");
var fs = require('fs');
// var buffer = fs.readFileSync(filePath);
const readFile = async (path) => {
const stream = fs.createReadStream(path);
return new Promise((resolve, reject) => {
const bufferParts = [];
stream.on('data', (data) => {
bufferParts.push(data);
});
stream.on('end', () => {
resolve(Buffer.concat(bufferParts))
});
})
}
async function uploadToAws() {
const filePath = path.join(__dirname, 'sample.txt');
// Create a new service object and some supporting variables
var glacier = new AWS.Glacier({ apiVersion: '2012-06-01', region: 'ap-southeast-1' }),
vaultName = '{vaultName}',
buffer = await readFile(filePath),
partSize = 4 * 1024 * 1024 * 1024, // 4GB chunks,
numPartsLeft = Math.ceil(buffer.length / partSize),
startTime = new Date(),
params = { vaultName: vaultName, partSize: partSize.toString() };
.
.
.
I searched a bit and seems that it's due to Js limitation. I changed from readFileSync to createReadStream due to big file size used. but error still occurs
What could be the cause of this? how do i resolve this?
thanks