How to create zipfile in S3 with Boto3 Python?
python's in-memory zip library is perfect for this. Here's an example from one of my projects:
import ioimport zipfilezip_buffer = io.BytesIO()with zipfile.ZipFile(zip_buffer, "a", zipfile.ZIP_DEFLATED, False) as zipper: infile_object = s3.get_object(Bucket=bucket, Key=object_key) infile_content = infile_object['Body'].read() zipper.writestr(file_name, infile_content)s3.put_object(Bucket=bucket, Key=PREFIX + zip_name, Body=zip_buffer.getvalue())
So I managed to get it to work in my Heroku flask app.Hope it helps anyone who is struggling.PS subfolder = blog_folderSo structure is, Bucket/blog_folder/resourcesBucket/blog_folder/zipped
import tempfile, zipfile, os, boto3AWS_ACCESS_KEY_ID = "some access key"AWS_ACCESS_SECRET_ACCESS_KEY = "some secret key"AWS_STORAGE_BUCKET_NAME = "some bucket"def make_zipfile(output_filename, source_dir): relroot = os.path.abspath(os.path.join(source_dir, os.pardir)) with zipfile.ZipFile(output_filename, "w", zipfile.ZIP_DEFLATED) as zip: for root, dirs, files in os.walk(source_dir): # add directory (needed for empty dirs) zip.write(root, os.path.relpath(root, relroot)) for file in files: filename = os.path.join(root, file) if os.path.isfile(filename): # regular files only arcname = os.path.join(os.path.relpath(root, relroot), file) zip.write(filename, arcname)aws_session = boto3.Session(aws_access_key_id = AWS_ACCESS_KEY_ID, aws_secret_access_key = AWS_SECRET_ACCESS_KEY)s3 = aws_session.resource("s3")current_path = os.getcwd()temp = tempfile.TemporaryDirectory(suffix="_tmp", prefix="basic_", dir=current_path)### AT TOP OF YOUR APP.PY file ^^^^^^^^^^@app_blog.route("/download_blog_res_zipfile/<int:blog_id>", methods = ["GET", "POST"])def download_blog_res_zipfile(blog_id): current_path = os.getcwd() blog = Blog.query.filter_by(id = blog_id).first() print(blog) print("DOWNLOAD COUNT!!!") print(blog.download_count) blog.download_count += 1 db.session.commit() del_folders = os.listdir(os.getcwd() + "/BLOG_ZIPPED_FOLDER") for folder in del_folders: zipp_path = os.getcwd() + "/BLOG_ZIPPED_FOLDER/" + folder print(folder) print("DELETING ZIPPING!") shutil.rmtree(os.getcwd() + "/BLOG_ZIPPED_FOLDER/" + folder) temp_zipp = tempfile.TemporaryDirectory(suffix="_tmp", prefix="zipping_", dir=current_path + "/BLOG_ZIPPED_FOLDER") s3 = boto3.client("s3", region_name = REGION_NAME) s3_resource = boto3.resource("s3") my_bucket = s3_resource.Bucket(AWS_STORAGE_BUCKET_NAME) paginator = s3.get_paginator("list_objects") folder = "blogs/blog_{}/resources".format(blog.id) file_list = [page for page in paginator.paginate(Bucket = AWS_STORAGE_BUCKET_NAME)\ .search("Contents[?Size >`0`][]") if folder in page["Key"]] for key in file_list: file_name = key["Key"].split("/")[-1] print(file_name) file_obj = my_bucket.Object(key["Key"]).get()["Body"] with open(os.getcwd() + "/" + BLOG_FOLDER + "/" + file_name, "wb") as w: w.write(file_obj.read()) make_zipfile(temp_zipp.name + "/blog_res_{}.zip".format(blog_id), current_path + "/" + BLOG_FOLDER) try: for key in file_list: file_name = key["Key"].split("/")[-1] file_path = current_path + "/" + BLOG_FOLDER +"/" + file_name os.remove(file_path) print("TRYY!!") print("REMOVED!!!") except: for key in file_list: file_name = key["Key"].split("/")[-1] file_path = current_path + "/" + BLOG_FOLDER + "/" + file_name os.remove(file_path) print("EXCEPT!!!") print("REMOVED!!!") return send_from_directory(temp_zipp.name, "blog_res_{}.zip".format(blog_id), as_attachment = True)