python multiprocessing comparison

Guillermo 0
Viewed 79

I am using multiprocessing for training a neural network where one process construct the batch samples and puts them in a queue and the parent process reads from the queue and trains the network with pytorch.

I noticed that the total time of training using multiprocessing was not shorter than using a single process, and when investigating further, I discovered that although reading from the queue in the multiprocess is faster than constructing the queue in the single process (as expected), the process of training (which is the same code for both multiprocessing and single processing) takes longer in the multiprocess.

I made up a simple script exemplifying. See script below:

import multiprocessing as mp
import numpy as np
import time

n = 200

def get_sample():
    local_loop = 400
    # data
    x = np.random.rand(n,n)
    p = np.random.rand(n,n)
    y = 0
    for i in range(local_loop):
        y += np.power(x, p)
    return y

def new_process(q_data, total_loops):
    for i in range(total_loops):
        q_data.put(get_sample())
    print('finish new process')

def main(multi_proc=False):
    st = time.time()
    total_loops = 100
    
    local_loop = 2500
    mt = 0
    other_t = 0

    st_multi = time.time()
    if multi_proc:
        q_data = mp.Queue()
        new_proc = mp.Process(target=new_process,args=(q_data, total_loops))
        new_proc.start()
    mt += time.time() - st_multi

    for i in range(total_loops):
        st_multi = time.time()
        if multi_proc:
            y = q_data.get()
        else:
            y = get_sample()
        mt += time.time() - st_multi

        other_st = time.time()
        for j in range(local_loop):
            y += np.random.rand(n,n)
        other_t += time.time() - other_st

    st_multi = time.time()
    if multi_proc:
        assert q_data.empty()
        new_proc.join()
    mt += time.time() - st_multi

    print('\nmulti_proc', multi_proc)
    print('multi_proc_time', mt)
    print('other_time', other_t)

    print(f'total time: {time.time()-st}')

if __name__ == '__main__':
    main(multi_proc=False)
    main(multi_proc=True)

When I run it, I get the result:

multi_proc False
multi_proc_time 36.44150114059448
other_time 39.08155846595764
total time: 75.5232412815094
finish new process

multi_proc True
multi_proc_time 0.4313678741455078
other_time 40.54900646209717
total time: 40.980711460113525

other_time is more than 1 second longer when multi_process=True (when they should be the same). This seems to be consistent across platforms/multiple experiments and in my real example it is longer than the gain from using multiprocessing, which is causing a big problem.

Any hint of what is happening?

multiprocessing python
1 Answers

Your results are what I would expect. But is your benchmark a true representation of reality?

In the multiprocessing case you have 2 processes:

  1. The main process, which creates "batch samples" and retrieves results generated by a child process.
  2. A get_sample child process that gets the sample created by the main process and puts a result to a queue for the main process to retrieve.

Both processes are running in parallel but the main process described above is very trivial requiring very little CPU processing compared to the child process. So any gains you achieve by running both processes in parallel are defeated by the additional overhead required in moving samples and results from one address space to another.

But what if creating a new batch sample were not so trivial? In the revamped benchmark below, I am ensuring that we spin some CPU cycles in producing a new sample by calling spin_cycles. I have arranged things so that the code for the multiprocessing benchmark and sequential processing benchmark are kept separate for clarity:

import multiprocessing as mp
import numpy as np
import time

n = 200
total_loops = 20
local_loops = 400

def spin_cycles():
    # simulate real processing time:
    x = 0
    for i in range(10_000_000):
        x += i * i
    return x

########### Sequential Benchmark: #######################

def process_sequential(sample):
    # data
    x = np.random.rand(n,n)
    p = np.random.rand(n,n)
    y = 0
    for i in range(local_loops):
        y += np.power(x, p)
    return y

def sequential_processing():
    results = []
    for sample in range(total_loops):
        # simulate real processing time:
        spin_cycles()
        results.append(process_sequential(sample))

def main_sequential():
    st = time.time()
    results = sequential_processing()
    et = time.time()
    print('Sequential time:', et-st)

########## Multiprocessing Benchmark ################

def process_multi(in_q, out_q):
    for _ in range(total_loops):
        sample = in_q.get()
        # data
        x = np.random.rand(n,n)
        p = np.random.rand(n,n)
        y = 0
        for i in range(local_loops):
            y += np.power(x, p)
        out_q.put(y)

def construct_batch_samples_multi(in_q):
    for sample in range(total_loops):
        # simulate real processing time:
        spin_cycles()
        in_q.put(sample)

def main_multi():
    st = time.time()
    in_q, out_q = mp.Queue(), mp.Queue()
    p = mp.Process(target=process_multi, args=(in_q, out_q))
    p.start()
    construct_batch_samples_multi(in_q)
    results = [out_q.get() for _ in range(total_loops)]
    et = time.time()
    p.join()
    print('Multiprocessing time:', et-st)

########### Run Benchmarks #######################
if __name__ == '__main__':
    main_multi()
    main_sequential()

Prints:

Multiprocessing time: 18.932964086532593
Sequential time: 28.5939993858337

Update

This is to demonstrate that the time to actually process a sample is identical for the multiprocessing and sequential benchmarks. Specifically, I am measuring in the multiprocessing case the total time processing samples not including the time to read and write to queues for a more direct comparison with the sequential processing case. I have also used time.process_time() to measure actual CPU time and have removed the call to spin_cycles so as to not confuse the issue.

import multiprocessing as mp
import numpy as np
import time

n = 200
total_loops = 20
local_loops = 400

########### Sequential Benchmark: #######################

def sequential_processing():
    results = []
    start_time = time.process_time()
    for sample in range(total_loops):
        # data
        x = np.random.rand(n,n)
        p = np.random.rand(n,n)
        y = 0
        for i in range(local_loops):
            y += np.power(x, p)
        results.append(y)
    processing_time = time.process_time() - start_time
    print("Sequential processing time:", processing_time)
    return results

def main_sequential():
    st = time.time()
    results = sequential_processing()
    et = time.time()
    print('Total elapsed time:', et-st)

########## Multiprocessing Benchmark ################

def process_multi(q):
    samples = [q.get() for _ in range(total_loops)]
    results = []
    t = time.process_time()
    for sample in samples:
        # data
        x = np.random.rand(n,n)
        p = np.random.rand(n,n)
        y = 0
        for i in range(local_loops):
            y += np.power(x, p)
        results.append(y)
    processing_time = time.process_time() - t
    print('Multiprocessing processing time:', processing_time)
    return results

def construct_batch_samples_multi(q):
    for sample in range(total_loops):
        q.put(sample)

def main_multi():
    st = time.time()
    q  = mp.Queue()
    p = mp.Process(target=construct_batch_samples_multi, args=(q,))
    p.start()
    process_multi(q)
    et = time.time()
    p.join()
    print('Total multiprocessing elapsed time:', et-st)

########### Run Benchmarks #######################
if __name__ == '__main__':
    main_multi()
    main_sequential()

Prints:

Multiprocessing processing time: 10.796875
Total multiprocessing elapsed time: 11.060487270355225
Sequential processing time: 10.96875
Total elapsed time: 11.052014112472534

There is essentially no difference in the two processing times (in this run the multiprocessing processing was a bit faster).

Booboo0
Related