Error: DQN expects a model that has one dimension for each action

user_13739397 0
Viewed 1405

I am building an RL agent for which the model is defined:

def build_model(height,width,channels,actions):
    model = Sequential()
    model.add(Conv2D(32,(8,8),strides=(4,4),activation='relu',input_shape=(3,height,width,channels)))
    model.add(Conv2D(64,(4,4),strides=(2,2),activation='relu'))
    model.add(Dense(512,activation='relu'))
    model.add(Dense(256,activation='relu'))
    model.add(Dense(actions,activation='linear'))
    return model

Then created a model:

env = gym.make('SpaceInvaders-v0')
height,width,channels = env.observation_space.shape
actions = env.action_space.n
model = build_model(height,width,channels,actions)

Then define Agent:

def build_agent(model,actions):
    policy = LinearAnnealedPolicy(EpsGreedyQPolicy(),attr='eps',value_max=1,value_min=.1,value_test=-.2, nb_steps=10000)
    memory = SequentialMemory(limit=2000,window_length=3)
    dqn = DQNAgent(model=model,memory=memory,policy=policy,dueling_type='avg',nb_actions=actions,nb_steps_warmup=1000)

Whine I try to build the Agent:

dqn = build_agent(model,actions)

I get this error:

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-100-320eef5d391d> in <module>
----> 1 dqn = build_agent(model,actions)

<ipython-input-95-97801a497208> in build_agent(model, actions)
      2     policy = LinearAnnealedPolicy(EpsGreedyQPolicy(),attr='eps',value_max=1,value_min=.1,value_test=-.2, nb_steps=10000)
      3     memory = SequentialMemory(limit=2000,window_length=3)
----> 4     dqn = DQNAgent(model=model,memory=memory,policy=policy,dueling_type='avg',nb_actions=actions,nb_steps_warmup=1000)

~/Documents/RL/pRL/lib/python3.8/site-packages/rl/agents/dqn.py in __init__(self, model, policy, test_policy, enable_double_dqn, enable_dueling_network, dueling_type, *args, **kwargs)
    105         # Validate (important) input.
    106         if list(model.output.shape) != list((None, self.nb_actions)):
--> 107             raise ValueError(f'Model output "{model.output}" has invalid shape. DQN expects a model that has one dimension for each action, in this case {self.nb_actions}.')
    108 
    109         # Parameters.

ValueError: Model output "Tensor("dense_30/BiasAdd:0", shape=(None, 3, 24, 18, 6), dtype=float32)" has invalid shape. DQN expects a model that has one dimension for each action, in this case 6.

I checked the value of model.output.shape and self.nb_actions:

print(model.output_shape)
# (None, 3, 24, 18, 6)
print(actions) # This is self.nb_action
# 6

I don't know what should be the shape of 'model.output'. In This error it looks like it should be '(None,6)' but I don't know what change I have to make to get this shape.

openai-gym reinforcement-learning tensorflow python
2 Answers

Convert model.output.shape to tuple and then compare if tuple(model.output.shape) != (None, 4)

user_123395130

Just tried everything and looked everywhere for answers, notice in the model summary the input is already in 3 dimensions. Therefore add a flatten layer solved the problems for me

def build_model(height,width,channels,actions):
    model = Sequential()
    model.add(Conv2D(32,(8,8),strides=(4,4),activation='relu',input_shape=(3,height,width,channels)))
    model.add(Conv2D(64,(4,4),strides=(2,2),activation='relu'))
    model.add(Dense(512,activation='relu'))
    model.add(Dense(256,activation='relu'))
    model.add(Flatten())    #add it anywhere you want tbh <<<<<<<<<<<<<<<<<<<<
    model.add(Dense(actions,activation='linear'))
   return model
user_198220050
Related