So, I am trying to use pythons OpenCV library to detect player coordinates in a video game by matching the minimap image to the full-size game map. for testing, I've pared down the full-size map to something smaller.
I'm pretty new to both Python and the OpenCV library, so please forgive my ignorance.
I started by converting my images to grayscale, I resized my minimap image to be the same scale as the large image (so that the features can match exactly, I was hoping) I used Canny edge detection on both since the maps use lots of white lines for walls, and figured this would help simplify everything. I did also try without doing this but it was no better.
my issue is honestly just finding an accurate match, I get matches all over the full-size world map and nothing specific. here is my code:
from cv2 import cv2
import numpy as np
import matplotlib.pyplot as pyplot
import matplotlib.image as mpimg
COORDS = [0,0]
WORLD_MAP = cv2.imread('world_map_gray.png')
#WORLD_MAP = cv2.cvtColor(WORLD_MAP,cv2.COLOR_BGR2GRAY)
WORLD_MAP = cv2.Canny(WORLD_MAP,200,255)
#WORLD_MAP = cv2.Laplacian(WORLD_MAP,cv2.CV_8U)
MINI_MAP = cv2.imread('screenshot.png')
MINI_MAP = MINI_MAP[110:315,1540:1767]
MINI_MAP = cv2.cvtColor(MINI_MAP,cv2.COLOR_BGR2GRAY)
MINI_MAP = cv2.resize(MINI_MAP,(87,79),interpolation=cv2.INTER_NEAREST)
MINI_MAP = cv2.Canny(MINI_MAP,200,255)
#MINI_MAP = cv2.Laplacian(MINI_MAP,cv2.CV_8U)
#resize to 87,79
########COPY FROM WEB####################################################
MIN_MATCH_COUNT = 10
img1 = WORLD_MAP # queryImage
img2 = MINI_MAP # trainImage
# Initiate SIFT detector
sift = cv2.SIFT_create(sigma=1)
# find the keypoints and descriptors with SIFT
kp1, des1 = sift.detectAndCompute(img1,None)
kp2, des2 = sift.detectAndCompute(img2,None)
FLANN_INDEX_KDTREE = 0
index_params = dict(algorithm = FLANN_INDEX_KDTREE, trees = 10)
search_params = dict(checks = 50)
flann = cv2.FlannBasedMatcher(index_params, search_params)
matches = flann.knnMatch(des1,des2,k=2)
# store all the good matches as per Lowe's ratio test.
good = []
for m,n in matches:
if m.distance < 0.5*n.distance:
good.append(m)
#good = sorted(good,key = lambda x:x.distance)
if len(good)>MIN_MATCH_COUNT:
src_pts = np.float32([ kp1[m.queryIdx].pt for m in good ]).reshape(-1,1,2)
dst_pts = np.float32([ kp2[m.trainIdx].pt for m in good ]).reshape(-1,1,2)
M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RHO,2)
matchesMask = mask.ravel().tolist()
h,w = img1.shape
pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2)
dst = cv2.perspectiveTransform(pts,M)
COORDS = [np.abs(dst[0][0][0]),np.abs(dst[0][0][1])]
COORDS[0] = COORDS[0] +img2.shape[0]/2
COORDS[1] = COORDS[1] +img2.shape[1]/2
img2 = cv2.polylines(img2,[np.int32(dst)],True,255,3, cv2.LINE_AA)
else:
print ("Not enough matches are found - %d/%d" % (len(good),MIN_MATCH_COUNT))
matchesMask = None
draw_params = dict(matchColor = (0,255,0), # draw matches in green color
singlePointColor = None,
matchesMask = matchesMask, # draw only inliers
flags = 2)
img3 = cv2.drawMatches(img1,kp1,img2,kp2,good,None,**draw_params)
WORLD_MAP_KP_IMG = cv2.drawKeypoints(img1,kp1,None)
cv2.imwrite('results\\WORLD_MAP_KEYPOINTS.png',WORLD_MAP_KP_IMG)
MINI_MAP_KP_IMG = cv2.drawKeypoints(img2,kp1,None)
cv2.imwrite('results\\MINI_MAP_KEYPOINTS.png',MINI_MAP_KP_IMG)
cv2.imwrite('results\\MATCHES.png',img3)
print("COMPLETE")
print("COORDS: ("+str(COORDS[0])+","+str(COORDS[1])+")")
here are the results, the Mini-map with its keypoints on it:
and of course, here is where the actual location of it is:
As far as I can tell there just aren't enough key points in the minimap image to match with - but I'm not sure how to remedy this. I've been reading through websites and documentation all day, I can't figure it out.
Thanks!
--EDIT
as requested here are the raw source images:
