Concurrent processes hang sometimes

Viewed 11

I am trying to extract faces using retinaface. To speed up the process, I try to run function concurrently. What the concurrent task function basically takes a list of all the directories, breaks them into smaller directories and then runs then runs the respective function on each list.

The problem is sometimes i can use more number of processes sometimes i can only use 2. Whats more troubling is that at times the functions just hang while other times, they work. Like say i start with 3 processes, after sometime, the terminal prints nothing, just a blank screen and when I go to the activity monitor, my ram usage has returned to normal at around 4gb(when i start, the usage spikes up to 15ish gb, my total ram is 16gb). I think my processes are getting stuck, i just dont know why.

def concurent_tasks(targets, num_processes, dir_list, *args):
    processes = {}
    list2D = split_dir_to_2Dlists(dir_list, num_processes)

    for i in range(num_processes):
        for j in targets:
            new_args = [list2D[i]] + list(args)
            processes["process_" + str(j) +str(i)] = multiprocessing.Process(target=j, args = new_args)
    for key in processes:
        processes[key].start()
    
    for key in processes:
        processes[key].join()

    for key in processes:
        processes[key].terminate()

def extract_faces(org_dir, new_dir_path):
    #org_dir is list of all folders in a directory
    for dir in org_dir: # gets a subfolder in the current directory
        counter = 0
        folder_name = dir.split('/')[-1] # name of the sub folder

        start =time.time()
        for entry in os.scandir(dir): # gets an image from the subfolder

            #print("directory: ", dir)
            print("entry: ", entry.path)
            #print("folder name: ", folder_name)

            try:
                #Extracting the face
                faces = RetinaFace.extract_faces(entry.path, align=True)
                if faces is not None: #returns 1 if a face is detected:

                    new_folder_path = os.path.join(new_dir_path,folder_name)
                    if not os.path.exists(new_folder_path): #creating a sub folder in the new directory if it does not exist
                        os.makedirs(new_folder_path)

                    #_, file_extention = os.path.splitext(entry.path)
                    for i in range(len(faces)):
                        new_file_name = folder_name + str(counter) +'_'+ str(i) + '.jpg'
                        new_file_path = os.path.join(new_folder_path,new_file_name)
                        cv2.imwrite(new_file_path, cv2.cvtColor(faces[i], cv2.COLOR_RGB2BGR))
                        
                        gc.collect()
                        gc.collect()
                        gc.collect()
                        gc.collect()
                        
                    counter+=1
            except:
                pass
        
        stop = time.time()
        gc.collect()
        gc.collect()
        gc.collect()
        gc.collect()
        print("Time for subfolder ", entry, " : ", stop-start)
0 Answers
Related