Listing and getting S3 files from an AWS node 4.3 lambda

Here’s how you can efficiently list all files in a bucket, and then download them all at once. If you use the node4.3 runtime on lambda, you can use promises natively. Beware, Promise.all will attempt to download all the files at once. If you have many files in your bucket, this could cause problems (i.e. throttling).

Don’t forget to give your lambda a role that has permission to listObjects and getObject

var AWS = require('aws-sdk');
var s3 = new AWS.S3();

var bucketName = 'myBucketName';

function main(callback) {
    s3.listObjects({
        Bucket: bucketName,
        Delimiter: '/'
    }).promise()
        .then(data => {
            return data.Contents.map((object) => {
                return s3.getObject({
                    Bucket: bucketName,
                    Key: object.Key
                }).promise();
            });
        })
        .then((data) => Promise.all(data))
        .then(console.log)
        .then(callback)
        .catch(console.log)

}

exports.handler = function(event, context, callback){
    return main(callback);
};

Leave a Reply

Fill in your details below or click an icon to log in:

WordPress.com Logo

You are commenting using your WordPress.com account. Log Out / Change )

Twitter picture

You are commenting using your Twitter account. Log Out / Change )

Facebook photo

You are commenting using your Facebook account. Log Out / Change )

Google+ photo

You are commenting using your Google+ account. Log Out / Change )

Connecting to %s