Python 3.9: multiprocessing process start() got an error| TypeError: cannot pickle 'weakref' object

user_12277738 0
Viewed 96

I'm trying to decrease running time by using multiprocessing. I got a weird error TypeError: cannot pickle 'weakref' object I'm not quite sure why this error occurs because I also use this approach to run another program but it run normally. Can someone explain why this error occurs. I already follow this Solution but it did not work for me.

import multiprocessing
from scipy import stats
import numpy as np
import pandas as pd
class T_TestFeature:
    def __init__(self, data, classes):
        self.data = data
        self.classes = classes 
        self.manager = multiprocessing.Manager()
        self.pval = self.manager.list()
        
    def preform(self):
        process = []
        for i in range(10):
            process.append(multiprocessing.Process(target=self.t_test, args=(i,)))

        for p in process:
            p.start()

        for p in process:
            p.join()

    def t_test(self, k):
        index_samples = np.array(self.data)[:,k]
        rs1 = [index_samples[i] for i in range(len(index_samples)) if self.classes[i] == "Virginia"]
        rs2 = [index_samples[i] for i in range(len(index_samples)) if self.classes[i] != "Virginia"]
        self.pval.append(stats.ttest_ind(rs1, rs2, equal_var=False).pvalue)

def main():
    df = pd.read_excel("/Users/xxx/Documents/Project/src/flattened.xlsx")
    flattened = df.values.T
    y = df.columns
    result = T_TestFeature(flattened, y)
    result.preform()
    print(result.pval)

if __name__ == "__main__":
    main()

Traceback (most recent call last):
  File "/Users/xxx/Documents/Project/src/t_test.py", line 41, in <module>
    main()
  File "/Users/xxx/Documents/Project/src/t_test.py", line 37, in main
    result.preform()
  File "/Users/xxx/Documents/Project/src/t_test.py", line 21, in preform
    p.start()
  File "/Users/xxx/opt/anaconda3/lib/python3.9/multiprocessing/process.py", line 121, in start
    self._popen = self._Popen(self)
  File "/Users/xxx/opt/anaconda3/lib/python3.9/multiprocessing/context.py", line 284, in _Popen
    return Popen(process_obj)
  File "/Users/xxx/opt/anaconda3/lib/python3.9/multiprocessing/popen_spawn_posix.py", line 32, in __init__
    super().__init__(process_obj)
  File "/Users/xxx/opt/anaconda3/lib/python3.9/multiprocessing/popen_fork.py", line 19, in __init__
    self._launch(process_obj)
  File "/Users/x/opt/anaconda3/lib/python3.9/multiprocessing/popen_spawn_posix.py", line 47, in _xxlaunch
    reduction.dump(process_obj, fp)
  File "/Users/xxx/opt/anaconda3/lib/python3.9/multiprocessing/reduction.py", line 60, in dump
    ForkingPickler(file, protocol).dump(obj)
TypeError: cannot pickle 'weakref' object
multiprocessing python
1 Answers

Here is a simpler way to reproduce your issue:

from multiprocessing import Manager, Process


class A:
    def __init__(self):
        self.manager = Manager()

    def start(self):
        print("started")

if __name__ == "__main__":
    a = A()
    proc = Process(target=a.start)
    proc.start()
    proc.join()

You cannot pickle instances containing manager objects, because they contain reference to the manager process they started (therefore, in general you can't pickle instances containing objects of class Process).

A simple fix would be to not store the manager. It will automatically be garbage collected once no references to the managed list remains:

def __init__(self, data, classes):
    self.data = data
    self.classes = classes
    manager = multiprocessing.Manager()
    self.pval = manager.list()
user_163107410
Related