I am trying to extract faces using retinaface. To speed up the process, I try to run function concurrently. What the concurrent task function basically takes a list of all the directories, breaks them into smaller directories and then runs then runs the respective function on each list.
The problem is sometimes i can use more number of processes sometimes i can only use 2. Whats more troubling is that at times the functions just hang while other times, they work. Like say i start with 3 processes, after sometime, the terminal prints nothing, just a blank screen and when I go to the activity monitor, my ram usage has returned to normal at around 4gb(when i start, the usage spikes up to 15ish gb, my total ram is 16gb). I think my processes are getting stuck, i just dont know why.
def concurent_tasks(targets, num_processes, dir_list, *args):
processes = {}
list2D = split_dir_to_2Dlists(dir_list, num_processes)
for i in range(num_processes):
for j in targets:
new_args = [list2D[i]] + list(args)
processes["process_" + str(j) +str(i)] = multiprocessing.Process(target=j, args = new_args)
for key in processes:
processes[key].start()
for key in processes:
processes[key].join()
for key in processes:
processes[key].terminate()
def extract_faces(org_dir, new_dir_path):
#org_dir is list of all folders in a directory
for dir in org_dir: # gets a subfolder in the current directory
counter = 0
folder_name = dir.split('/')[-1] # name of the sub folder
start =time.time()
for entry in os.scandir(dir): # gets an image from the subfolder
#print("directory: ", dir)
print("entry: ", entry.path)
#print("folder name: ", folder_name)
try:
#Extracting the face
faces = RetinaFace.extract_faces(entry.path, align=True)
if faces is not None: #returns 1 if a face is detected:
new_folder_path = os.path.join(new_dir_path,folder_name)
if not os.path.exists(new_folder_path): #creating a sub folder in the new directory if it does not exist
os.makedirs(new_folder_path)
#_, file_extention = os.path.splitext(entry.path)
for i in range(len(faces)):
new_file_name = folder_name + str(counter) +'_'+ str(i) + '.jpg'
new_file_path = os.path.join(new_folder_path,new_file_name)
cv2.imwrite(new_file_path, cv2.cvtColor(faces[i], cv2.COLOR_RGB2BGR))
gc.collect()
gc.collect()
gc.collect()
gc.collect()
counter+=1
except:
pass
stop = time.time()
gc.collect()
gc.collect()
gc.collect()
gc.collect()
print("Time for subfolder ", entry, " : ", stop-start)