Combining data from multiple processes

user_5936720 0
Viewed 46

For this multithreaded python code, I'd like to combine the data from each process variable "work_output" into a global variable "workOutput". I'm not sure how to access variables outside individual processes or how to combine the data from multiple processes.

#edited code based on comment that answered the question

from multiprocessing import Pool, cpu_count
import time
import psutil
import os
import math

work =(["process1", 1,2], ["process2", 2,3], ["process3", 3,4], ["process4", 4,5], ["process5", 5,6], ["process6", 6,7])
workOutputAll=[]

def calculateOutput(work_data):
    #timeToSleep=int(100)
    #time.sleep(timeToSleep)
    work_output=work_data[1]*work_data[2]
    print(f"Process {work_data[0]} {work_data[1]}*{work_data[2]}={work_output}\n")
    #workOutput.append(work_output)
    return work_output

def work_log(work_data):
    print(F"Process {work_data[0]} started\n")
    y=calculateOutput(work_data)
    print(f"Process {work_data[0]} Finished.\n")
    return y

def pool_handler():
    p=Pool(4)
    z=p.map(work_log, work)
    workOutputAll.append(z)

if __name__ == '__main__':
    pool_handler()
    print(workOutputAll)
multiprocessing python
1 Answers

Write functions that return the result (like a math problem y = f(x)). Then map() will take all the processes and return them as a list.

from multiprocessing import Pool, cpu_count
import time
import psutil
import os
import math

work =(["process1", 1,2], ["process2", 2,3], ["process3", 3,4], ["process4", 4,5], ["process5", 5,6], ["process6", 6,7])


def calculate_output(work_data):
    work_output=work_data[1]*work_data[2]
    print(f"Process {work_data[0]} {work_data[1]}*{work_data[2]}={work_output}\n")
    return work_output

def work_log(work_data):
    print(F"Process {work_data[0]} started\n")
    result = calculate_output(work_data)
    print(f"Process {work_data[0]} Finished.\n")
    return result

def pool_handler():
    p=Pool(4)
    results = p.map(work_log, work)
    print(results)

if __name__ == '__main__':
    pool_handler()

stdout:

Process process1 started

Process process1 1*2=2

Process process1 Finished.

Process process2 started

Process process2 2*3=6

Process process2 Finished.

Process process4 started

Process process4 4*5=20

Process process4 Finished.

Process process3 started

Process process6 started

Process process6 6*7=42

Process process3 3*4=12

Process process6 Finished.
Process process3 Finished.


Process process5 started

Process process5 5*6=30

Process process5 Finished.

[2, 6, 12, 20, 30, 42]
user_10611930
Related