After experimenting with the Cart Pole from openai gym, I wanted to try something a bit more advanced reinforcement learning and went on with Super Mario Bros NES. Using a similar setup doesn't seem to yield the results I was expecting though, and Mario doesn't seem to learn anything.
I have been using retro gym to emulate the environment and writing some custom wrappers for actions, observations and reward (code for these are at the bottom) and these seem to yield what I was expecting. For reinforcement learning I have been using TensorFlow for the network model and the DQN agent from Keras-RL with an epsilon greedy Q policy, and after many hours of training Mario is still unable to jump high enough to get over the initial obstacles.
Code for reinforcement learning:
import retro
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Flatten
from tensorflow.keras.optimizers import Adam
from rl.agents import DQNAgent
from rl.policy import EpsGreedyQPolicy
from rl.memory import SequentialMemory
from wrapper import wrapper
env = retro.make('SuperMarioBros-Nes', use_restricted_actions=retro.Actions.DISCRETE)
env = wrapper(env)
states = 70
actions = 4
train_size = 5000000
def build_model(states, actions):
model = Sequential()
model.add(Flatten(input_shape=(1,10,7)))
model.add(Dense(100, activation='relu'))
model.add(Dense(100, activation='relu'))
model.add(Dense(actions, activation='linear'))
return model
def build_agent(model, actions):
policy = EpsGreedyQPolicy()
memory = SequentialMemory(limit=train_size, window_length=1)
dqn = DQNAgent(model=model, memory=memory, policy=policy, nb_actions=actions, nb_steps_warmup=100000, target_model_update=1e-2)
return dqn
model = build_model(states, actions)
dqn = build_agent(model, actions)
dqn.compile(Adam(lr=1e-3), metrics=['mae'])
dqn.fit(env, nb_steps=train_size, visualize=True, verbose=1)
dqn.save_weights('weights/MarioWeights.h5f', overwrite=True)
I have tried tuning network architecture and target update rate, different policies, exploration vs. exploitation, observation space, reward and target update rate, but little seems to change and reward converges at very low values after a while.
Code for wrappers:
class ObservationWrapper(gym.ObservationWrapper):
def __init__(self, env):
super().__init__(env)
self.env = env
def observation(self, obs):
### STATE MODIFICATION ###
ram = self.env.get_ram() # Grab ram data
tiles = SMB.get_tiles(ram) # Grab dict with all tiles on the screen
t_list = list(tiles.values()) # Convert to list
ct_list = np.asarray(self.chunkify(t_list)) # Convert to array with image dimensions
try:
m_x = int(np.where(ct_list == 170)[1])
except:
m_x = 3
return ct_list[4:14, m_x:m_x+7] # Crop to ROI
def chunkify(self, list, n=16): # Function to extract rows, or chunks, of 16 values from the image
c_list = []
row = []
count = 0
for i in list:
row.append(i.value)
count += 1
if count%16 == 0:
c_list.append(row)
row = []
return c_list
class ActionWrapper(gym.ActionWrapper):
def __init__(self, env):
super().__init__(env)
self.action_list = [0, 6, 18, 24]#, 3] # NOOP, RIGHT, JUMP, RIGHT + JUMP, LEFT
def action(self, act):
return self.action_list[act]
class BasicWrapper(gym.Wrapper):
def __init__(self, env):
super().__init__(env)
self.env = env
self.total_reward = 0
self.max_reward = 0
self.frame = 0
self.mario_old_pos = [40,176] # Starting position of Mario on env.reset()
self.t_old = 400 # Counting down from 400
def reset_init(self):
self.total_reward = 0
self.max_reward = 0
self.frame = 0
self.mario_old_pos = [40,176]
self.t_old = 400
def check_alive(self, ram): # Check if dying or dead
if SMB._is_alive(ram):
return 0
else:
return -15
def step(self, action):
next_state, reward, done, info = self.env.step(action)
ram = self.env.get_ram()
done = SMB._got_flag(ram) # Check if Mario finished level and got flag
mario_pos = SMB.get_mario_location_in_level(ram) # Grab Mario location in level
vx = (int(mario_pos[0]) - int(self.mario_old_pos[0])) * 2
if vx > 10:
vx = 0
self.mario_old_pos = mario_pos
d = self.check_alive(ram)
timer = info.get('time')
c = timer - self.t_old
self.t_old = timer
reward = vx + vy + c + d
self.total_reward += reward
if self.total_reward > self.max_reward:
self.max_reward = self.total_reward
# Reset if Mario is stuck
if self.frame > 100:
if d < 0 or (self.total_reward < self.max_reward - 0.1*self.total_reward):
self.reset_init()
self.env.reset()
self.frame += 1
return next_state, reward, done, info
The reward here is stolen from https://github.com/Kautenja/gym-super-mario-bros, while the observations are inspired by https://chrispresso.io/AI_Learns_To_Play_SMB_Using_GA_And_NN.
I feel like I am missing something essential. Any suggestions for how to proceed?