Multi-Object Tracking using Opencv Python: Delete trackers for the objects that moved out of frame

I am working on a project where I am using cv2 for multi-object detection and tracking. I was able to detect objects in a live feed and I was also able to track them with in the frame.

For example:

Two Persons got detected in the frame and we assigned two trackers for each one of them say Person0, Person1

Till here it is working fine.

Challenge is when first person leaves, corresponding tracker gets deleted and second person tracker ID gets changed to Person 0, But I dont want that to be chnaged to Person 0, it should be Person1

Below is the code I've tried:

from imutils.video import VideoStream
from imutils.video import FPS
import numpy as np
import argparse
import imutils
import time
import cv2
from math import floor

# initialize the list of class labels MobileNet SSD was trained to
# detect, then generate a set of bounding box colors for each class
CLASSES = ["background", "aeroplane", "bicycle", "bird", "boat",
    "bottle", "bus", "car", "cat", "chair", "cow", "diningtable",
    "dog", "horse", "motorbike", "person", "pottedplant", "sheep",
    "sofa", "train", "tvmonitor"]
COLORS = np.random.uniform(0, 255, size=(len(CLASSES), 3))

# load our serialized model from disk
print("[INFO] loading model...")
net = cv2.dnn.readNetFromCaffe('./models/MobileNetSSD_deploy.prototxt.txt', './models/MobileNetSSD_deploy.caffemodel')

# initialize the video stream, allow the cammera sensor to warmup,
# and initialize the FPS counter
print("[INFO] starting video stream...")
vs = VideoStream(src=1).start()
time.sleep(2.0)
fps = FPS().start()
init_once = []
oklist = []
tracker_lst = []
newbox = []
# loop over the frames from the video stream
while True:
    # grab the frame from the threaded video stream and resize it
    # to have a maximum width of 400 pixels
    frame = vs.read()
    frame = imutils.resize(frame, width=900,height = 600)


    (h, w) = frame.shape[:2]
    blob = cv2.dnn.blobFromImage(cv2.resize(frame, (300, 300)),0.007843, (300, 300), 127.5)
    net.setInput(blob)
    detections = net.forward()
    bounding_boxes = []
    for i in np.arange(0,detections.shape[2]):
        confidence = detections[0,0,i,2]
        if confidence > 0.2:
            idx = int(detections[0,0,i,1])
            if CLASSES[idx] == 'person':
                box = detections[0,0,i,3:7]*np.array([w,h,w,h])
                bounding_boxes.append(box)
                label = "{}: {:.2f}%".format(CLASSES[idx], confidence * 100)
                print("[INFO] {}".format(label))
                (startX, startY, endX, endY) = box.astype(int)
                cv2.rectangle(frame, (startX, startY), (endX, endY),COLORS[idx], 2)
                y = startY - 15 if startY - 15 > 15 else startY + 15
                cv2.putText(frame, label, (startX, y),cv2.FONT_HERSHEY_SIMPLEX, 0.5, COLORS[idx], 2)
#    if len(init_once) < 1:
#        bbox = bounding_boxes
    print("@@@@Detected bounding boxes:", bounding_boxes)
    # loop over the detections
    newbbox = []
    bbox = bounding_boxes
    for (i,boxie) in enumerate(bbox):
        (startX,startY,endX,endY) = boxie.astype(int)
        boxie = (int(startX),int(startY),int(endX-startX),int(endY-startY))
        newbbox.append(boxie)
        #print (newbbox)
        if len(bounding_boxes):
            tracker_lst.append(cv2.TrackerKCF_create())
            okinit = tracker_lst[i].init(frame, boxie)
            init_once.append(1)
            oklist.append(okinit)
        print (len(tracker_lst))
        oklist[i], newbbox[i] = tracker_lst[i].update(frame)
    for (i,newbox) in enumerate (newbbox) :
         if oklist[i]:
             Tracker_ID = "{}: {:.2f}%".format('person'+str(i), confidence * 100)
             cv2.rectangle(frame,(int(floor(newbox[0])),int(floor(newbox[1]))), (int(floor(newbox[0]))+int(floor(newbox[2])),int(floor(newbox[1]))+int(floor(newbox[3]))),(255,255,255), 1)          
             new_y = floor(newbox[1]) - 15 if floor(newbox[1]) - 15 > 15 else floor(newbox[1]) + 15
             print (newbox[0])
             print (newbox[1])
             cv2.putText(frame, Tracker_ID, (int(floor(newbox[0])), int(floor(newbox[1]))),cv2.FONT_HERSHEY_SIMPLEX, 0.4, (255,255,255), 1)
         else :
             cv2.putText(frame, "Tracking failure object {0}".format(i), (100,(80+(20*i))), cv2.FONT_HERSHEY_SIMPLEX, 0.4,(255,255,255),1)
             if len(bounding_boxes)>0:
                 del tracker_lst[i]
                 del oklist[i]

             #oklist[i], newbbox[i] = tracker_lst[i].update(frame)                


    # show the output frame
    cv2.imshow("Frame", frame)
    key = cv2.waitKey(1) & 0xFF

    # if the `q` key was pressed, break from the loop
    if key == ord("q"):
        break

    # update the FPS counte
    fps.update()

# stop the timer and display FPS information
fps.stop()
print("[INFO] elapsed time: {:.2f}".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))

# do a bit of cleanup
cv2.destroyAllWindows()
vs.release()

How do I know, which objects got dropped from the frame and remove the tracker only for that, with out changing tracker ID of retained images in the frame