How to copy/move all objects in Amazon S3 from one prefix to other using the AWS SDK for Node.js
You will need to make one AWS.S3.listObjects() to list your objects with a specific prefix. But you are correct in that you will need to make one call for every object that you want to copy from one bucket/prefix to the same or another bucket/prefix.
You can also use a utility library like async to manage your requests.
var AWS = require('aws-sdk');var async = require('async');var bucketName = 'foo';var oldPrefix = 'abc/';var newPrefix = 'xyz/';var s3 = new AWS.S3({params: {Bucket: bucketName}, region: 'us-west-2'});var done = function(err, data) { if (err) console.log(err); else console.log(data);};s3.listObjects({Prefix: oldPrefix}, function(err, data) { if (data.Contents.length) { async.each(data.Contents, function(file, cb) { var params = { Bucket: bucketName, CopySource: bucketName + '/' + file.Key, Key: file.Key.replace(oldPrefix, newPrefix) }; s3.copyObject(params, function(copyErr, copyData){ if (copyErr) { console.log(copyErr); } else { console.log('Copied: ', params.Key); cb(); } }); }, done); }});Hope this helps!
Here is a code snippet that do it in the "async await" way:
const AWS = require('aws-sdk');AWS.config.update({ credentials: new AWS.Credentials(....), // credential parameters});AWS.config.setPromisesDependency(require('bluebird'));const s3 = new AWS.S3();... ...const bucketName = 'bucketName'; // example bucketconst folderToMove = 'folderToMove/'; // old folder nameconst destinationFolder = 'destinationFolder/'; // new destination folder try { const listObjectsResponse = await s3.listObjects({ Bucket: bucketName, Prefix: folderToMove, Delimiter: '/', }).promise(); const folderContentInfo = listObjectsResponse.Contents; const folderPrefix = listObjectsResponse.Prefix; await Promise.all( folderContentInfo.map(async (fileInfo) => { await s3.copyObject({ Bucket: bucketName, CopySource: `${bucketName}/${fileInfo.Key}`, // old file Key Key: `${destinationFolder}/${fileInfo.Key.replace(folderPrefix, '')}`, // new file Key }).promise(); await s3.deleteObject({ Bucket: bucketName, Key: fileInfo.Key, }).promise(); }) );} catch (err) { console.error(err); // error handling}
A small change to the code of Aditya Manohar that improves the error handling in the s3.copyObject function and will actually finish the "move" request by removing the source files after the copy requests have been executed:
const AWS = require('aws-sdk');const async = require('async');const bucketName = 'foo';const oldPrefix = 'abc/';const newPrefix = 'xyz/';const s3 = new AWS.S3({ params: { Bucket: bucketName }, region: 'us-west-2'});// 1) List all the objects in the source "directory"s3.listObjects({ Prefix: oldPrefix}, function (err, data) { if (data.Contents.length) { // Build up the paramters for the delete statement let paramsS3Delete = { Bucket: bucketName, Delete: { Objects: [] } }; // Expand the array with all the keys that we have found in the ListObjects function call, so that we can remove all the keys at once after we have copied all the keys data.Contents.forEach(function (content) { paramsS3Delete.Delete.Objects.push({ Key: content.Key }); }); // 2) Copy all the source files to the destination async.each(data.Contents, function (file, cb) { var params = { CopySource: bucketName + '/' + file.Key, Key: file.Key.replace(oldPrefix, newPrefix) }; s3.copyObject(params, function (copyErr, copyData) { if (copyErr) { console.log(err); } else { console.log('Copied: ', params.Key); } cb(); }); }, function (asyncError, asyncData) { // All the requests for the file copy have finished if (asyncError) { return console.log(asyncError); } else { console.log(asyncData); // 3) Now remove the source files - that way we effectively moved all the content s3.deleteObjects(paramsS3Delete, (deleteError, deleteData) => { if (deleteError) return console.log(deleteError); return console.log(deleteData); }) } }); }});Note that I have moved the cb() callback function outside the if-then-else loop. That way even when an error occurs the async module will fire the done() function.