create big zip archives with lot of small files in memory (on the fly) with python

Ok, finally I created that zip's Frankenstein:

from io import BytesIO
from zipfile import ZipFile, ZIP_DEFLATED
import sys
import gc

files = (
    'input/i_1.docx',  # one file size is about ~580KB
    'input/i_2.docx',
    'input/i_3.docx',
    'input/i_4.docx',
    'input/i_5.docx',
    'input/i_6.docx',
    'input/i_7.docx',
    'input/i_8.docx',
    'input/i_9.docx',
    'input/i_10.docx',
    'input/i_11.docx',
    'input/i_12.docx',
    'input/i_13.docx',
    'input/i_14.docx',
    'input/i_21.docx'
)


# this function allow to get size of in-memory object
# add only for debug purposes
def _get_size(input_obj):
    memory_size = 0
    ids = set()
    objects = [input_obj]
    while objects:
        new = []
        for obj in objects:
            if id(obj) not in ids:
                ids.add(id(obj))
                memory_size += sys.getsizeof(obj)
                new.append(obj)
        objects = gc.get_referents(*new)
    return memory_size


class CustomizedZipFile(ZipFile):

    # add customized BytesIO to be able return faked offset
    class _CustomizedBytesIO(BytesIO):

        def __init__(self, fake_offset: int):
            self.fake_offset = fake_offset
            self.temporary_switch_to_faked_offset = False
            super().__init__()

        def tell(self):
            if self.temporary_switch_to_faked_offset:
                # revert tell method to normal mode to minimize faked behaviour
                self.temporary_switch_to_faked_offset = False
                return super().tell() + self.fake_offset
            else:
                return super().tell()

    def __init__(self, *args, **kwargs):
        # create empty file to write if fake offset is set
        if 'fake_offset' in kwargs and kwargs['fake_offset'] is not None and kwargs['fake_offset'] > 0:
            self._fake_offset = kwargs['fake_offset']
            del kwargs['fake_offset']
            if 'file' in kwargs:
                kwargs['file'] = self._CustomizedBytesIO(self._fake_offset)
            else:
                args = list(args)
                args[0] = self._CustomizedBytesIO(self._fake_offset)
        else:
            self._fake_offset = 0
        super().__init__(*args, **kwargs)

    # finalize zip (should be run only on last chunk)
    def force_write_end_record(self):
        self._write_end_record(False)

    # don't write end record by default to be able get not ended chunks
    # ZipFile writing end metainfo on close by default
    def _write_end_record(self, skip_write_end=True):
        if not skip_write_end:
            if self._fake_offset > 0:
                self.start_dir = self._fake_offset
                self.fp.temporary_switch_to_faked_offset = True
            super()._write_end_record()


def archive(files):

    compression_type = ZIP_DEFLATED
    CHUNK_SIZE = 1048576  # 1MB

    with open('tmp.zip', 'wb') as resulted_file:
        offset = 0
        filelist = []
        with BytesIO() as chunk:
            for f in files:
                with BytesIO() as tmp:
                    with CustomizedZipFile(tmp, 'w', compression=compression_type) as zf:
                        with open(f, 'rb') as b:
                            zf.writestr(
                                zinfo_or_arcname=f.replace('input/', 'output/'),
                                data=b.read()
                            )
                        zf.filelist[0].header_offset = offset
                        data = tmp.getvalue()
                        offset = offset + len(data)
                    filelist.append(zf.filelist[0])
                chunk.write(data)
                print('size of zipfile:', _get_size(zf))
                print('size of chunk:', _get_size(chunk))
                if len(chunk.getvalue()) > CHUNK_SIZE:
                    resulted_file.write(chunk.getvalue())
                    chunk.seek(0)
                    chunk.truncate()
            # write last chunk
            resulted_file.write(chunk.getvalue())
        # file parameter may be skipped it we using fake_offset
        # because empty _CustomizedBytesIO will be initialized at constructor
        with CustomizedZipFile(None, 'w', compression=compression_type, fake_offset=offset) as zf:
            zf.filelist = filelist
            zf.force_write_end_record()
            end_data = zf.fp.getvalue()
        resulted_file.write(end_data)


archive(files)

Output is:

size of zipfile: 2182955
size of chunk: 582336
size of zipfile: 2182979
size of chunk: 1164533
size of zipfile: 2182983
size of chunk: 582342
size of zipfile: 2182979
size of chunk: 1164562
size of zipfile: 2182983
size of chunk: 582343
size of zipfile: 2182979
size of chunk: 1164568
size of zipfile: 2182983
size of chunk: 582337
size of zipfile: 2182983
size of chunk: 1164556
size of zipfile: 2182983
size of chunk: 582329
size of zipfile: 2182984
size of chunk: 1164543
size of zipfile: 2182984
size of chunk: 582355
size of zipfile: 2182984
size of chunk: 1164586
size of zipfile: 2182984
size of chunk: 582338
size of zipfile: 2182984
size of chunk: 1164545
size of zipfile: 2182980
size of chunk: 582320

So we can see that chunk is always dumped on storage and truncated when reach the maximum chunk size (1MB in my case)

Resulted archive tested with MacOS The Unarchiver v4.2.4 and Windows 10 default unarchiver and 7-zip

Note!

Size of created by chunks archive is 16 bytes bigger than archive created by common zipfile library. Probably some extra zero bytes is written somewhere. I didn't check why it happens

_{^{zipfile is worstest python library I've ever seen before. Looks like it supposed to be used as non-extendable binary-like file}}