How can I make the cursor move in game using python (ex: First shooter shooting game)

Viewed 3759

I’m trying to build an automation program with python using Opencv to detect the given color and pyautogui for automation. I tried using pyautogui to move the cursor in game but didn’t work. Used another library pynput had some advantages then stopped. I’d need your help!!!

This is my code:

import pyautogui as pg
import time
from PIL import ImageGrab
import numpy as np
import cv2
import keyboard
from pynput.mouse import Controller, Button

def clickAt(Mouse, x, y):
    Mouse.position = (x, y)
    Mouse.press(Button.right)
    Mouse.click(Button.left)
    time.sleep(2)
    Mouse.release(Button.right)


#screen 
x1, y1 = (200, 290)
x2, y2 = (1525, 800)

for i in range(50):

   img = ImageGrab.grab(bbox=(x1, y1, x2, y2)) #bbox specifies specific region (bbox= x,y,width,height)
   img_np = np.array(img)
   frame = cv2.cvtColor(img_np, cv2.COLOR_BGR2HSV)

   #lower and higher of the color
   lower = np.array([86, 179, 7])
   upper = np.array([255, 255, 255])

   mask = cv2.inRange(frame, lower, upper)

   croped = cv2.bitwise_and(frame, frame, mask=mask)

   res = cv2.bitwise_and(frame,frame, mask= mask)

   #cv2.imshow("res",res)
   cv2.imshow("mask", mask)
   #cv2.imshow("frame", frame)

   van = np.where(mask > 0)
   #print(van)
   x_van = van[1]
   y_van = van[0]
   time.sleep(5)

   Mouse = Controller()
   #clickAt(Mouse,100,100)

   x_axis = van[1] + x1
   y_axis = van[0] + y1

   print(x_axis[5])
   print(y_axis[5])

   hit_pos = (x_van[0] + x1, y_van[1] + y1)

   pg.moveTo(hit_pos)
   clickAt(Mouse,100,100)
   clickAt(Mouse,x_axis[5], y_axis[5])

   #click the mouse
   pg.click(hit_pos)
   print('BIRD HIT at', hit_pos)

   time.sleep(1)

  cv2.waitKey(25) 
  time.sleep(3)
  #cv2.destroyAllWindows()
1 Answers

Im working on the exact same project as you and I found the below method works however, the movement is the coordinate points, instead the x,y parameters are move RELATIVE to your current mouse position

import win32api, win32con

win32api.mouse_event(win32con.MOUSEEVENTF_MOVE, x, y, 0, 0)
Related