Your question is extremely complex, because solving it can send you down lots of rabbit holes.
I believe that Rahul Iyer is on the right track, because IMHO it would be easier to initiate a new EC2 instance and compress the files on this instance and move them back to a S3 bucket that only serves zip files to the client.
If your files were smaller you could use AWS Cloudfront to handling the zipping when a client requests a file.
During my research I did note that other languages, such as .Net and Java had APIs that handle streaming into zip files. I also looked at zipstream, which has been forked several times. It's unclear how zipstream can be used to stream a file for zipping.
The code below will chunk a file and write the chucks to a zip file. The input files were close to 12Gbs and the output file was almost 5Gbs.
During testing I didn't see any major issues with memory usage or big spikes.
I did add some pseudo S3 code to one of the posts below. I think more testing is required to understand how this code works on files in S3.
from io import RawIOBase
from zipfile import ZipFile
from zipfile import ZipInfo
from zipfile import ZIP_DEFLATED
# This module is needed for ZIP_DEFLATED
import zlib
class UnseekableStream(RawIOBase):
def __init__(self):
self._buffer = b''
def writable(self):
return True
def write(self, b):
if self.closed:
raise ValueError('The stream was closed!')
self._buffer += b
return len(b)
def get(self):
chunk = self._buffer
self._buffer = b''
return chunk
def zipfile_generator(path, stream):
with ZipFile(stream, mode='w') as zip_archive:
z_info = ZipInfo.from_file(path)
z_info.compress_type = ZIP_DEFLATED
with open(path, 'rb') as entry, zip_archive.open(z_info, mode='w') as dest:
for chunk in iter(lambda: entry.read(16384), b''): # 16384 is the maximum size of an SSL/TLS buffer.
dest.write(chunk)
yield stream.get()
yield stream.get()
stream = UnseekableStream()
# each on the input files was 4gb
files = ['input.txt', 'input2.txt', 'input3.txt']
with open("test.zip", "wb") as f:
for item in files:
for i in zipfile_generator(item, stream):
f.write(i)
f.flush()
stream.close()
f.close()
pseudocode s3/zip code
This code is strictly hypothetical, because it needs testing.
from io import RawIOBase
from zipfile import ZipFile
from zipfile import ZipInfo
from zipfile import ZIP_DEFLATED
import os
import boto3
# This module is needed for ZIP_DEFLATED
import zlib
session = boto3.Session(
aws_access_key_id='XXXXXXXXXXXXXXXXXXXXXXX',
aws_secret_access_key='XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX',
region_name='XXXXXXXXXX')
s3 = session.resource('s3')
bucket_name = s3.Bucket('bucket name')
class UnseekableStream(RawIOBase):
def __init__(self):
self._buffer = b''
def writable(self):
return True
def write(self, b):
if self.closed:
raise ValueError('The stream was closed!')
self._buffer += b
return len(b)
def get(self):
chunk = self._buffer
self._buffer = b''
return chunk
def zipfile_generator(path, stream):
with ZipFile(stream, mode='w') as zip_archive:
z_info = ZipInfo.from_file(path)
z_info.compress_type = ZIP_DEFLATED
with open(path, 'rb') as entry, zip_archive.open(z_info, mode='w') as dest:
for chunk in iter(lambda: entry.read(16384), b''):
dest.write(chunk)
yield stream.get()
yield stream.get()
stream = UnseekableStream()
with open("test.zip", "wb") as f:
for file in bucket_name.objects.all():
obj = s3.get_object(Bucket=bucket_name, Key=file.key)
for i in zipfile_generator(obj.get(), stream):
f.write(i)
f.flush()
stream.close()
f.close()
