Python Multiprocessing w/ Map: Running Entire Script, Not Just Sub-function

Viewed 1419

I'm trying to use Python's multiprocessing map function. I have placed the map call inside a sub-function, as I need to loop through a larger data-set to divide it up and call map on the smaller chunks.

My problem is that the time.sleep(5) line is being called multiple times and 'Test!' is printing 5 times (which seems to equal once at the start, and then 2*2 for the number of loops * number of processes), even though it is at a higher level than the multiprocessing calls. At the same time, though, the CSV output is what I expect, so runParallel() is running as expected and being called the expected number of times.

from multiprocessing import Pool
import numpy as np
import os,csv,copy,time
from AuxFuncs import *

def master():    
    time.sleep(5)
    print('Test!')
    for mult in [1,10]:
        runParallel(mult)

def runParallel(mult):
    randIntInputs = list()
    for i in range(5): randIntInputs.append((np.random.randint(10)*mult,mult))
    if __name__=='__main__':
        p = Pool(processes=2)
        results = p.map(testFunc,randIntInputs)
        p.close()
        p.join()
        valsToSave = [list(result[0]) for result in results]
        write2dListToCSV(valsToSave,'output' + str(mult) + '.csv')

def testFunc(inputs):
    return np.random.randint(1,10,5) * inputs[0],inputs[1]

master()

And the output is:

Test!
Test!
Test!
Test!
Test!

I thought the problem might be that I put the Pool call in a function, but even if I move it out of a function, I have the same issue ("Test!" is printed 3 times by the below code.)

from multiprocessing import Pool
import numpy as np
import os,csv,copy,time
from AuxFuncs import *

def testFunc(inputs):
    return np.random.randint(1,10,5) * inputs[0],inputs[1]

print('Test!')
mult,randIntInputs = 5,list()
for i in range(5): randIntInputs.append((np.random.randint(10)*mult,mult))
if __name__=='__main__':
    p = Pool(processes=2)
    results = p.map(testFunc,randIntInputs)
    p.close()
    p.join()
    valsToSave = [list(result[0]) for result in results]
    write2dListToCSV(valsToSave,'output' + str(mult) + '.csv')

EDIT: Thanks for the help. Looks like this works:

from multiprocessing import Pool import numpy as np import os,csv,copy,time from AuxFuncs import *

def master():    
    if __name__=='__main__':
        time.sleep(5)
        print('Test!')
        for mult in [1,10]:
            runParallel(mult)

def runParallel(mult):
    randIntInputs = list()
    for i in range(5): randIntInputs.append((np.random.randint(10)*mult,mult))
    # if __name__=='__main__':
    p = Pool(processes=2)
    results = p.map(testFunc,randIntInputs)
    p.close()
    p.join()
    valsToSave = [list(result[0]) for result in results]
    write2dListToCSV(valsToSave,'output' + str(mult) + '.csv')

def testFunc(inputs):
    return np.random.randint(1,10,5) * inputs[0],inputs[1]

master()
1 Answers
Related