How to download large files from aws s3 with restart on network loss

user_14337509 0
Viewed 325

I am trying to implement a "network safe" downloader from aws s3 bucket. the downloader should be able to download a single .zip file from s3 and write it to a local .zip file.

My current approach is using node with readStream and writeStream ass follows

const download = async () => {
    AWS.config.update(
        {
            accessKeyId: "",
            secretAccessKey: "",
            region: ""
        }
    );

    const s3 = new AWS.S3();

    const params = {
        Bucket: '',
        Key: ''
    };

    const { ContentLength: contentLength } = await s3.headObject(params).promise();

    const rs = s3.getObject(params).createReadStream()
    const ws = fs.createWriteStream(path.join('./', 'file.zip'));

    let progress = 0;

    rs.on('data', function (chunk) {
        progress += chunk.length;
        console.log(`Progress: ${progress / contentLength * 100}%`);
    });

    rs.pipe(ws);
}

what i need is a way to catch/create an event regarding network errors that will allow me to pause and restart the download when network is back on. or even better - auto restart the download when network is restored. currently i couldn't find any events regarding network errors and seems that network loss while download is in process does not triggers the 'error' event.

any solutions in node/python will be very appriciated

node-streams aws-sdk-nodejs node.js amazon-web-services amazon-s3
1 Answers

Short update: found a work around using wget to download a preSignedUrl to the object i want to download. still not the native experience i wanted to have. share your thoughts

export class Downloader {
    constructor(dest_dir, file_name_to_save) {
        this.dest_dir = dest_dir;
        this.file_name = file_name_to_save;
        this.progress = "61%"
    }

    async download_file(on_progress, on_done, socket, io) {

        let client = new S3Client({
            region: 'eu-west-1',
            credentials: {
                accessKeyId: "",
                secretAccessKey: "",
            }
        });

        const command = new GetObjectCommand({ Bucket: "s", Key: "" });
        const url = await getSignedUrl(client, command, { expiresIn: 3600 });



        let child = spawn('wget', ['-c', '-O', this.dest_dir + this.file_name, url]);

        socket.on('cancel', () => {
            child.kill()
            console.log("killed child")
        })

        child.stderr.on('data', (e) => {
            let stdout = e.toString();

            let p = stdout.match(/([0-9]+?\%)+/g);

            if (p && p.length > 0 && p[0] != this.progress) {
                on_progress && on_progress(p[0])
                console.log(p[0])
                this.progress = p[0]
            }
        });


        child.stdout.on('end', function (data) {
            on_done && on_done("end");
            console.log("end: ", data)
            child.kill();
        });

        child.on('exit', function (code) {
            if (code != 0) console.log('Failed: ' + code);
            else console.log("success!", code)

            on_done && on_done("exit");
        });
    }
}
user_143375090
Related