Handwritten in order digit recognition and extraction with Python-OpenCV

user_9884511 0
Viewed 1586

I want to extract the digits inside the boxes in order.

Original image

enter image description here

I used the watershed algorithm to separate digits connected to boxes but it wont't contour digits properly, instead it selects only some part of digits.

#To get in big box that contain smaller boxes from the image
img = cv2.imread('1_6.png',0)
img = cv2.GaussianBlur(img,(3,3),1)
_,img  = cv2.threshold(img,240,255,cv2.THRESH_BINARY)
img = cv2.GaussianBlur(img,(11,11),1)
edges = cv2.Canny(img,100,200)
_,c,h = cv2.findContours(edges.copy(),cv2.RETR_CCOMP,cv2.CHAIN_APPROX_NONE)
img = cv2.imread('1_6.png')
temp_c = sorted(c,key=cv2.contourArea,reverse=True)

#Select the big box
epsilon = 0.0001*cv2.arcLength(temp_c[0],True)
approx = cv2.approxPolyDP(temp_c[0],epsilon,True)

#Crop big box
pts = approx.copy()
rect = cv2.boundingRect(pts)
x,y,w,h = rect
croped = img[y:y+h, x:x+w].copy()

## (2) make mask
pts = pts - pts.min(axis=0)

mask = np.ones(croped.shape[:2], np.uint8)
cv2.drawContours(mask, [pts], -1, (255, 255, 255), -1, cv2.LINE_AA)

## (3) do bit-op
dst = cv2.bitwise_and(croped, croped, mask=mask)


gray = cv2.cvtColor(dst,cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(gray,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)
kernel = np.ones((1,1),np.uint8)
opening = cv2.morphologyEx(thresh,cv2.MORPH_CLOSE,kernel, iterations = 2)

sure_bg = cv2.dilate(opening,kernel,iterations=1)

dist_transform = cv2.distanceTransform(opening,cv2.DIST_L2,5)

ret, sure_fg = cv2.threshold(dist_transform,0.3*dist_transform.max(),255,0)

sure_fg = np.uint8(sure_fg)

unknown = cv2.subtract(sure_bg,sure_fg)

ret, markers = cv2.connectedComponents(sure_fg)

# Add one to all labels so that sure background is not 0, but 1
markers = markers+1

# Now, mark the region of unknown with zero
markers[unknown==255] = 0

plt.imshow(markers,cmap="gray")

img = dst.copy()
markers = cv2.watershed(dst,markers)
img[markers == -1] = [0,0,255]

Current result

enter image description here

computer-vision opencv image-processing image python
1 Answers

Here's my approach. I will try to be as detailed as possible:

  • Convert image to grayscale
  • Perform canny edge detection
  • Remove horizontal and vertical lines to isolate characters
  • Perform morphological operations to enhance letters
  • Find contours
  • Filter contours using contour area and aspect ratio
  • Sort contours from left to right to extract the digits in order
  • Iterate through sorted contours and extract ROI

First we perform Canny edge detection using cv2.Canny()

enter image description here

Next the goal is to remove the vertical and horizontal lines we can isolate the digits. We begin by creating various kernels, each targeted at a horizontal, vertical, or general orientation

vertical_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1,2))
horizontal_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (2,1))
kernel = cv2.getStructuringElement(cv2.MORPH_CROSS, (3,3))

We begin by removing the horizontal lines with cv2.erode()

enter image description here

Now we dilate the vertical lines with cv2.dilate()

enter image description here

Next we remove the vertical lines

enter image description here

Now note we have almost nothing left, so we have to restore the digits by dilating

enter image description here

From here we find contours using cv2.findContours(). We filter using cv2.contourArea() and by aspect ratio to obtain bounding boxes.

enter image description here

Now to extract the digits in order, we use imutils.contours.sort_contours()

Finally, we extract the ROI for each digit and save the image. Here's a screenshot of the saved ROI's in order

enter image description here

import cv2
import numpy as np
from imutils import contours

image = cv2.imread('1.png')
original = image.copy()
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
canny = cv2.Canny(gray, 130, 255, 1)

vertical_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1,2))
horizontal_kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (2,1))
kernel = cv2.getStructuringElement(cv2.MORPH_CROSS, (3,3))
erode = cv2.erode(canny, vertical_kernel)
cv2.imshow('remove horizontal', erode)
dilate = cv2.dilate(erode, vertical_kernel, iterations=5)
cv2.imshow('dilate vertical', dilate)
erode = cv2.erode(dilate, horizontal_kernel, iterations=1)
cv2.imshow('remove vertical', erode)
dilate = cv2.dilate(erode, kernel, iterations=4)
cv2.imshow('dilate horizontal', dilate)

cnts = cv2.findContours(dilate, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
cnts = cnts[0] if len(cnts) == 2 else cnts[1]

digit_contours = []
for c in cnts:
    area = cv2.contourArea(c)
    peri = cv2.arcLength(c, True)
    approx = cv2.approxPolyDP(c, 0.01 * peri, True)
    x,y,w,h = cv2.boundingRect(approx)
    aspect_ratio = w / float(h)

    if (aspect_ratio >= 0.4 and aspect_ratio <= 1.3):
        if area > 150:
            ROI = original[y:y+h, x:x+w]
            cv2.rectangle(image,(x,y),(x+w,y+h),(0,255,0),2)
            digit_contours.append(c)

sorted_digit_contours = contours.sort_contours(digit_contours, method='left-to-right')[0]
contour_number = 0
for c in sorted_digit_contours:
    x,y,w,h = cv2.boundingRect(c)
    ROI = original[y:y+h, x:x+w]
    cv2.imwrite('ROI_{}.png'.format(contour_number), ROI)
    contour_number += 1

cv2.imshow('canny', canny)
cv2.imshow('image', image)
cv2.waitKey(0)
user_111621650
Related