Contours in the same row are couple pixels apart

I was trying to get contours sorted from a sudoku image, generating ROIs from the bottom in the right-left direction. It goes nicely in most images, but in some (as shown bellow) it just skips a part then goes to the other part, messing with the order.

I tried to sort as follows:

def sort_contours(contours):
    bounding_boxes = [cv2.boundingRect(c) for c in contours]
    (contours, bounding_boxes) = zip(
        *sorted(zip(contours, bounding_boxes), key=lambda b: b[1][1], reverse=True))
    return contours

It didn't work, because i noticed that in some cases the squares in the same row were 1-3 pixels apart in the y coordinate.Then, I tried to change the bounding_boxes manually, as follows, but it didn't worked:

def sort_contours(contours):
    bounding_boxes = [cv2.boundingRect(c) for c in contours]
    for i in range(len(bounding_boxes) - 2):
        if (abs(bounding_boxes[i][1] - bounding_boxes[i+1][1]) < 10) and (bounding_boxes[i][1] != bounding_boxes[i+1][1]):
            bounding_boxes[i] = (bounding_boxes[i][0], bounding_boxes[i]
                                 [1], bounding_boxes[i][2], bounding_boxes[i][3])
            bounding_boxes[i+1] = ((bounding_boxes[i+1][0], bounding_boxes[i]
                                   [1], bounding_boxes[i+1][2], bounding_boxes[i+1][3]))
    (contours, bounding_boxes) = zip(
        *sorted(zip(contours, bounding_boxes), key=lambda b: b[1][1], reverse=True))
    return contours

Func for setting the contours:

def contours_ocr(image):
    cnts, hierarchy = cv2.findContours(
        image, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    hierarchy = hierarchy[0]
    inner_countours = [c[0] for c in zip(cnts, hierarchy) if c[1][3] == 0]
    inner_countours = sort_contours(inner_countours)
    image_count = -1
    for contour in inner_countours:
        x, y, w, h = cv2.boundingRect(contour)
        cv2.rectangle(image, (x, y), (x+w, y+h), (0, 255, 0), 6)
        roi = image[y:y+h, x:x+w]
        cv2.imwrite('temp/temp_image.jpg', roi)
        roi = "temp/temp_image.jpg"
        ocr = pytesseract.image_to_string(roi, config='--psm 6')
        if ocr != '':
            ocr = ocr[0]
            sudoku[image_count] = ocr
        cv2.imwrite('temp/full_image.jpg', image)
        image_count -= 1

Is there a way to fix that?