Please find the code and exception below, I tried digging stackoverflow but couldn't identify why pool.map is throwing this exception.
import multiprocessing
import pandas as pd
import numpy as np
from multiprocessing import Pool
from transformers import pipeline
import os
def parallelize_dataframe(df, func,num_partitions,num_cores):
a,b,c,d,e = np.array_split(df, num_partitions)
pool = Pool(num_cores)
df = pd.concat(pool.map(func, [a,b,c,d,e]))
pool.close()
pool.join()
return df
def get_doc(text,**kwargs):
nlp = kwargs['nlp_obj']
result = nlp(question="Which type of doctor does the patient consult?", context=text)
return result['answer']
def square(x):
return x**2
def test_func(data):
nlp = pipeline("question-answering")
data["HCP"] = data["text"].apply(get_doc,nlp_obj = nlp)
return data
if __name__ == '__main__':
num_partitions = 5
num_cores = multiprocessing.cpu_count()
data = pd.read_csv('./data.csv')
data = data[['text']]
data = data[data['text'].isna() == False]
data.reset_index(inplace=True,drop=True)
data = data.loc[:10]
test = parallelize_dataframe(data, test_func,num_partitions,num_cores)
print(test)
in parallelize_dataframe(df, func, num_partitions, num_cores) 10 a,b,c,d,e = np.array_split(df, num_partitions) 11 pool = Pool(num_cores) ---> 12 df = pd.concat(pool.map(func, [a,b,c,d,e])) 13 pool.close() 14 pool.join()
/usr/lib/python3.6/multiprocessing/pool.py in map(self, func, iterable, chunksize) 264 in a list that is returned. 265 ''' --> 266 return self._map_async(func, iterable, mapstar, chunksize).get() 267 268 def starmap(self, func, iterable, chunksize=None):
/usr/lib/python3.6/multiprocessing/pool.py in get(self, timeout) 642 return self._value 643 else: --> 644 raise self._value 645 646 def _set(self, i, obj):
AssertionError: daemonic processes are not allowed to have children