Splitting a large file into multiple other smaller files using a generator

Question

I have a large file that is 143mb in size. I want to split the file into smaller files that are 2.5mb in size, put them into a directory and return the file names. The way I'm attempting to do this is with a generator:

def gen_read(filename, chunk=1024*8):
    with open(filename, "rb") as f:
        for part in iter(lambda: f.read(chunk), b''):
            yield part

The goal is to take this generator and read the file into parts from there write each part into a temporary filename until the file is 2.5mb in size and add to the extension of the temporary file to have a sort of list of them. I'm trying to do so via this function:

API_TEMP_FILE_PATH = "/tmp"

def random_filename(length=10):
    s = ""
    acceptable = string.ascii_letters
    for _ in range(length):
        s += random.choice(acceptable)
    return s

def split_file(filename, bytes_limit=2621440):
    split_files = []
    file_ext_number = 1
    tmp_filename = random_filename(length=32)
    do_break = False
    while not do_break:
        file_path = "{}/{}_split_file.part_{}".format(API_TEMP_FILE_PATH, tmp_filename, file_ext_number)
        stream = gen_read(filename)
        for part in next(stream):
            if not part:
                do_break = True
            if os.path.exists(file_path):
                size = os.stat(file_path).st_size
                if size > bytes_limit:
                    file_ext_number += 1
            with open(file_path, 'wb') as dest:
                dest.write(part)
    searcher = re.compile('{}\_split\_file\.part\_\d(\d+)?'.format(tmp_filename))
    for filename in os.listdir(API_TEMP_FILE_PATH):
        if searcher.search(filename) is not None:
            split_files.append("{}/{}".format(API_TEMP_FILE_PATH, filename))
    return split_files

The issue I'm running into is that my generator is only producing 1 "character" at a time (can be seen by adding print(repr(part)) right underneath the for part):

...
'\x10'
'\x00'
'\x00'
'\x00'
'\x00'
'\x05'
'\x00'
'\x00'
'\x10'
...

As for that, the file size never changes from 1. What am I doing wrong to where this file split function isn't working as expected?

user_12969777 · Answer

I figured it out, instead of using a generator I just read into the file a certain amount:

def split_file(filename, bytes_limit=2621440):
    split_files = []
    file_ext_number = 1
    tmp_filename = random_filename(None, length=32, is_pcap=False)
    file_path = "{}/{}_split_file.part_".format(API_TEMP_FILE_PATH, tmp_filename)
    with open(filename, "rb") as source:
        byte = source.read(bytes_limit)
        while byte:
            open(file_path + "{}".format(file_ext_number), 'wb').write(byte)
            byte = source.read(bytes_limit)
            file_ext_number += 1
    searcher = re.compile('{}\_split\_file\.part\_\d(\d+)?'.format(tmp_filename))
    for filename in os.listdir(API_TEMP_FILE_PATH):
        if searcher.search(filename) is not None:
            split_files.append("{}/{}".format(API_TEMP_FILE_PATH, filename))
    return split_files

It produces all the correct files

Splitting a large file into multiple other smaller files using a generator

1 Answers