Python Multiprocessing cannot Join for Large Data Set

I am trying compute cosine similarity of 260774x96 data matrix. I was using full size Dataframe to compute the cosine similarity, and using this code. similarities = cdist(Dataframe, Dataframe, metric='cosine').

However, the Jupyter Notebook run out memory(64GB) and crash the kernel. So that I decide to compute single row with Dataframe each time, and then sort, trim and save it each loop. The issue is it takes too much times, about 12 hours.

So I applied multiprocessing to speed up, for small size data it works, single row compute with 500 rows Dataframe. However, computing single row with full size Dataframe, the process can work and put data to the queue, but the join() just not work. The main function keep running even the child process was end with no error. I try to print the queue in another cell, it has full data.

Anther problem is input data queue cannot take full size Dataframe, it show Full error.

I don't know what is the problem the program just stuck on join() step. Or is other practical way to have parallel computing.

from multiprocess import Lock, Process, Queue, current_process,Array,Lock
from time import time
import queue  # imported for using queue.Empty exception
from scipy.spatial.distance import cdist
from scipy.spatial import distance
# from cos_process import cos_process
from sklearn.metrics import pairwise_distances



def cos_process(idx, result_queue, index_queue,df_data_only_numpy):
    print(f"process start {current_process().name}")
    # for shrik the data size, so that the multiprocessing Join can work.
    # df_data_only_numpy = df_data_only_numpy[:500]
    while True:

        try:
            task_index_list = index_queue.get_nowait()


        except queue.Empty:

            break
        else:
            cosine_dict = {}
            for i in task_index_list:


                similarities = cdist([df_data_only_numpy[i]], df_data_only_numpy, metric='cosine')
                sorted_save_data = np.sort(similarities[0])[:20] # output the cosine simlarities data after sorted
                sorted_save_key = (np.argsort(similarities[0])[:20]) #output the index of sorted simlarities
                # make a dictionary {index:{index:cosine,index:cosine},....}
                cosine_dict[i]={int(i):float(data) if data==data else (data) for i,data in zip(sorted_save_key,sorted_save_data) }
#             print(cosine_dict)
            try:
                # put the dictionary data to queue
                result_queue.put_nowait(cosine_dict)
            except queue.Full:
                print("queue full in process")


    print(f"end process {current_process().name}")
    

    return True


def main():

    number_of_processes = 10 # create 10 processes for computer
    process_compute_time = 1000  # each process run 1000 rows in the while loop
    df_data_only_numpy = df_data_only.to_numpy()
    index_queue = Queue()  # index of the df
    result_queue = Queue()  # store the output of the similarity
    processes = []


    # for idx in df_data_only.T:
    # only test 1000 rows data in this case, the dataframe has 260774 rows and 96 columns
    for idx in range(0, 1000, process_compute_time):
        try:
            index_queue.put_nowait(list(range(idx, idx+process_compute_time)))  
        except queue.Full:
            print("full")
            
    print("creating process")
    # creating processes
    for w in range(number_of_processes):
        p = Process(target=cos_process, args=(index_queue, result_queue, index_queue,df_data_only_numpy))
        processes.append(p)
        p.start()

        
    # completing process 
    for p in processes:
        print("before join")
        p.join()
        print("finish join")

        # print the output

    while not result_queue.empty():
        print(result_queue.get())

    return True


if __name__ == '__main__':
    main()