Detecting zone with some color and text with openCV in python

Question

What I want :

When the user clicks on a button, the program takes the screnshot in the clipboard 'no problem with this part), detects Zone Of Interest (ZOI), cuts the zone and extracts the text of multiple zones for my programme. I try to extract information from a prescription. I know the zones of interests will always be in the same rectangle of color but I'm not sure the prescription will be in fullsize each time I need to extract information. You can see the original image and what I want the ZOI are in red.

What I tried:

First time I tried making the contour of the zone using the color in the font. The program can make a zone around the grey specific zone but the isn't what I want (green box). You can see above the original image, post-process image and the code

     import numpy as np
            import cv2
    
        # Read input image
        img = cv2.imread('test_image.png')
    
        gray = np.all(img == (227,227,227), 2)
    
        # Convert logical matrix to uint8
        gray = gray.astype(np.uint8)*255
    
        # Find contours
        cnts = cv2.findContours(gray, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_NONE)[-2]  # Use index [-2] to be compatible to OpenCV 3 and 4
    
        # Get contour with maximum area
        c = max(cnts, key=cv2.contourArea)
    
        x, y, w, h = cv2.boundingRect(c)
    
        # Draw green rectangle for testing
        cv2.rectangle(img, (x, y), (x+w, y+h), (0, 255, 0), thickness = 2)
    
        # Show result
        cv2.imshow('gray', gray)
        cv2.imwrite('niveau_gris.jpg', gray)
        cv2.imshow('img', img)
        cv2.imwrite('test_image_resultat.jpg', img)
        cv2.waitKey(0)
        cv2.destroyAllWindows()

Second time I tried making the contour of the zone using two parametres in the aim to select the zone with the text. The program doesn't make a zone at all. You can see the code below

    import numpy as np
    import cv2
    
    frame = cv2.imread('test_image_constrasate.jpg')
    # Convert BGR to HSV
    hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
    # define range of red color in HSV
    lower_red = np.array([189,189,189])
    upper_red = np.array([204,203,204])
    
    mask = cv2.inRange (hsv, lower_red, upper_red)
    contours = cv2.findContours(mask.copy(),
                               cv2.RETR_TREE,
                               cv2.CHAIN_APPROX_SIMPLE)[-2]
    
    if len(contours) > 0:
        red_area = max(contours, key=cv2.contourArea)
        x, y, w, h = cv2.boundingRect(red_area)
        cv2.rectangle(frame,(x, y),(x+w, y+h),(0, 0, 255), 2)
    
    
    cv2.imshow('frame', frame)
    cv2.imshow('mask', mask)
    
    cv2.waitKey(0)

Third time I try using Hough Line Transform but I'm not experimented enough with working with image I'm no sure of the repeatability of the process because the screenshot maybe not be fullsize and I'm not sure how to use the line to make boxes.

provide research

I sought on StackoverFlow with the following terms : "OpenCv select rectangle", "OpenCV select area base on the color", "OpenCV how to select a zone base on his color"....

Can you help me ? Thank you for your future help

user_2602877 · Answer

Approach for the title:

find the buttons that sit to the left and right of it, using matchTemplate
title is a rectangle relative to those

Approach for the table:

inRange on color of table header
connectedComponentsWithStats
filter by height to find only table header cells
find the widest cell
use striped background to separate rows

entire thing: https://gist.github.com/crackwitz/54a2a8ed3fdb2d07b969ef5aeae9dfcf

utility functions:

def crop(im, x, y, w, h):
    (height, width) = im.shape[:2]
    assert w > 0 and h > 0
    assert x >= 0 and y >= 0
    assert (x+w <= width) and (y+h <= height)
    return im[y:y+h, x:x+w]

def find_template(haystack, needle):
    (nw, nh) = needle.shape[:2]
    scores = cv.matchTemplate(haystack, needle, method=cv.TM_SQDIFF)
    (minval, maxval, minloc, maxloc) = cv.minMaxLoc(scores)
    #print(minval, minloc)
    # minval ought to be 0... bug?
    assert minval <= nw*nh*3 * 1**2, "can't find template"
    (x,y) = minloc
    return (x, y, nw, nh)

load:

im = cv.imread("YebIa.png")#, cv.IMREAD_GRAYSCALE)
(imh, imw) = im.shape[:2]
print("size:", imw, 'x', imh)
imshow(im)

extract button templates from hand-picked coordinates in this specific picture. best to save those and imread instead:

button1 = crop(im, 214, 88, 24, 24)
imshow(button1)
button2 = crop(im, 672, 88, 24, 24)
imshow(button2)

find buttons, get title:

button1_rect = find_template(im, button1)
button2_rect = find_template(im, button2)
b1x, b1y, b1w, b1h = button1_rect
b2x, b2y, b2w, b2h = button2_rect

top = b1y
bottom = b1y + b1h
left = b1x + b1w
right = b2x
title = crop(im, left, top, right-left, bottom-top)
imshow(title)

inRange:

# table header, first cell is largest
header_color = (194, 142, 93)
mask = cv.inRange(im, header_color, header_color)

connected components:

(nlabels, labels, stats, centroids) = cv.connectedComponentsWithStats(mask)
# print(stats) # x, y, w, h, area (ConnectedComponentsTypes)

filter and sort components:

comps = [(label, *stat) for label, stat in enumerate(stats)]
# (label, x, y, w, h, area)
comps = [comp for comp in comps if comp[4] == 25] # height: exactly 25 pixels
comps.sort(key=lambda comp: comp[5], reverse=True) # area, descending... or simply max(key=)
header_comp = comps[0] # largest area
header_rect = header_comp[1:5]
(hx,hy,hw,hh) = header_rect
header = crop(im, *header_rect)
imshow(header)

find table body and rows:

# table body
# pixel column just before the header cell (B) contains striped background but no text
# column to the left of that (A) contains only white, until the end
bx = hx
by = hy+hh + 1
bw = hw

columnA = crop(im, bx-2, by, 1, imh-by)[:,0,1]
(I,) = np.where(columnA != 255)
bh = I.min() # table body height

columnB = crop(im, bx-1, by, 1, bh)[:,0,1]
rowmask = (columnB == 255)#.astype(np.int8)
(I,) = np.where(np.diff(rowmask))
I += 1 # diff shifts things back, edge is on the second pixel, not the first

row_tops = np.concatenate(([0], I[:-1]))
row_bottoms = I
# np.vstack([row_tops, row_bottoms]).T

extract each row:

print((bx, by, bw, bh))
for i,(top,bottom) in enumerate(zip(row_tops, row_bottoms)):
    print(f"row {i+1}:")
    imshow(crop(im, bx, by+top, bw, bottom-top))
    print()

Detecting zone with some color and text with openCV in python

1 Answers