I'm hoping for some help with my DDPG algorithm. Everything runs and the Q values from the critic network are created correctly but the result for the action ends up pegged at either the maximum or minimum allowed value. Such behavior seems to imply that something is wrong with my gradient computations but I can't quite figure it out.
For testing purposes I'm using a very simple input that the agent should be able to learn and to which I know the correct answer:
state | action | reward
A | 100 | 100
A | 100 | 100
A | 100 | 100
A | 100 | 100
A | 200 | 200
A | 200 | 0
A | 200 | 0
A | 200 | 0
Since 200 is accepted once it has an expected value of 50 reward while 100 is accepted every time, meaning that it has an expected value of 100 reward. Thus, given that the state is the same, the result should always be 100. I should point out that future rewards are currently being ignored - this is a session-maximizing model only.
The code is a bit long so I've edited out that parts that are irrelevant:
def build_actor_net(s, scope, trainable,a_dim,act_max,act_min,features):
with tf.variable_scope(scope):
s = tf.feature_column.input_layer(features=features, feature_columns=s)
init_w = tf.random_normal_initializer(0., 0.003)
init_b = tf.constant_initializer(0.1)
regularizer = tf.contrib.layers.l2_regularizer(scale=0.01)
net = tf.layers.batch_normalization(s,fused=True)
net = tf.layers.dense(net, 400, activation=tf.nn.relu,kernel_initializer=init_w, bias_initializer=init_b, name='l1',trainable=trainable,kernel_regularizer=regularizer)
net = tf.layers.batch_normalization(net,fused=True)
net = tf.layers.dense(net, 300, activation=tf.nn.relu,kernel_initializer=init_w, bias_initializer=init_b, name='l2',trainable=trainable,kernel_regularizer=regularizer)
with tf.variable_scope('actor_action'):
actions = tf.layers.dense(net, a_dim, activation=tf.nn.sigmoid,name='actions', trainable=trainable)
scaled_a = tf.add(tf.multiply(actions,tf.subtract(act_max,act_min)), act_min, name='scaled_a')
return scaled_a
def build_critic_net(s, a, scope, trainable,s_dim,a_dim,features):
with tf.variable_scope(scope):
s = tf.feature_column.input_layer(features=features, feature_columns=s)
init_w = tf.random_normal_initializer(0., 0.003)
init_b = tf.constant_initializer(0.1)
regularizer = tf.contrib.layers.l2_regularizer(scale=0.01)
net = tf.layers.batch_normalization(s,fused=True)
net = tf.layers.dense(net,400,activation=tf.nn.relu,kernel_initializer=init_w,bias_initializer=init_b,name='l1',trainable=trainable,kernel_regularizer=regularizer)
net = tf.layers.batch_normalization(net+a,fused=True)
net = tf.layers.dense(net, 300, activation=tf.nn.relu,kernel_initializer=init_w, bias_initializer=init_b, name='l2',trainable=trainable,kernel_regularizer=regularizer)
with tf.variable_scope('q'):
q = tf.layers.dense(net, a_dim, kernel_initializer=init_w, bias_initializer=init_b, trainable=trainable,kernel_regularizer=regularizer)# Q(s,a)
return q
def model_fn(features, mode, params):
state = (There are several state columns here that dont matter)
reward = tf.feature_column.numeric_column('reward')
action = tf.feature_column.numeric_column('action')
# ---------------------- Build Actor Networks ---------------------------
with tf.variable_scope('Actor'):
act_a = build_actor_net(state, 'act_eval_net', True,params['a_dim'],params['act_max'],params['act_min'],features)
act_a_ = build_actor_net(state_, 'act_target_net', True,params['a_dim'],params['act_max'],params['act_min'],features)
# ---------------------- Build Critic Networks ---------------------------
with tf.variable_scope('Critic'):
crit_a = tf.feature_column.input_layer(features=features,feature_columns=[action]) #Interchange action and mult here
crit_q = build_critic_net(state, crit_a, 'crit_eval_net', True,params['s_dim'],params['a_dim'],features)
crit_actor_update = build_critic_net(state,act_a,'crit_update_actor_net',True,params['s_dim'],params['a_dim'],features)
crit_q_ = build_critic_net(state_, act_a_, 'crit_target_net', True,params['s_dim'],params['a_dim'],features)
# ---------------------- Set up target, loss, and gradient --------------------
with tf.variable_scope('target_q'):
r = tf.feature_column.input_layer(features=features, feature_columns=[reward])
crit_target_q = r #+ params['gamma'] * crit_q_ #(Session-max model)
with tf.variable_scope('crit_loss'):
crit_loss = tf.reduce_mean(tf.squared_difference(crit_target_q, crit_q))
with tf.variable_scope('update'):
act_e_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Actor/act_eval_net')
act_t_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Actor/act_target_net')
crit_e_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Critic/crit_eval_net')
crit_t_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Critic/crit_target_net')
crit_update_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='Critic/crit_update_actor_net')
crit_update_replace_op = [crit_update_params[i].assign(crit_e_params[i]) for i in range(len(crit_update_params))]
with tf.control_dependencies(crit_update_replace_op):
crit_target_replace_op = [crit_t_params[i].assign(tf.multiply(crit_e_params[i], params['tau']) +
tf.multiply(crit_t_params[i], 1. - params['tau'])) for i in range(len(crit_t_params))]
with tf.control_dependencies(crit_target_replace_op):
act_target_replace_op = [act_t_params[i].assign(tf.multiply(act_e_params[i], params['tau']) +
tf.multiply(act_t_params[i], 1. - params['tau'])) for i in range(len(act_t_params))]
with tf.variable_scope('C_train'):
reg = tf.losses.get_regularization_loss()
with tf.control_dependencies(act_target_replace_op):
crit_train_op = tf.train.AdamOptimizer(params['clr']).minimize(loss=crit_loss, global_step=tf.train.get_global_step())
with tf.variable_scope('a_grad'):
with tf.control_dependencies([crit_train_op]):###Not sure about this. May be necessary may not be
with tf.control_dependencies([act_a]):
a_grads = tf.gradients(crit_actor_update, act_a)[0] # tensor of gradients of each sample (None, a_dim)
with tf.variable_scope('A_train'):
act_policy_grads = tf.gradients(ys=act_a, xs=act_e_params, grad_ys=a_grads)
actor_gradients = list(map(lambda x: tf.div(x,params['bsize']),act_policy_grads))
with tf.control_dependencies([crit_train_op]):
act_train_op = tf.train.AdamOptimizer(-params['alr']).apply_gradients(zip(actor_gradients, act_e_params))
return tf.estimator.EstimatorSpec(mode=mode,loss=crit_loss,train_op=tf.group(crit_train_op,act_train_op))
In addition, there's a main function that sets up some parameters (assume any parameters called in the above code are defined) and then calls the Estimator:
DDPG = tf.estimator.Estimator(model_fn=model_fn, params=params, model_dir=model_dir)
DDPG.train(input_fn=lambda: my_input_fn(path,True,args.maxe,args.batch,args.buffer,feature_names))
I skipped the code for my_input_fn as well, but assume that it works just fine.
I'm fairly certain there's a problem with my gradients somewhere, either in the computation of the gradients themselves, the backpropagation of the gradients, or in the train op. The end result is that I end up with every suggested action being either the maximum allowed action (200) or the minimum allowed action (0) but never the correct action (100). If anyone could help me shed some light on my error I would greatly appreciate it.