I am using CNN to do short text classification. I know that if overfitting may lead all neurons become zero. But it is wired that, when overfitting, all training batch accuracies are 1, which does not make sense because not all true categories are 0. I think the accuracy should be low, but not 1.
Below is the part of the code that I use:
... Define some input placeholders here ...
pooled_outputs = []
for filter_size in filter_sizes:
filter_shape = [filter_size, embed_dimen, 1, num_filters]
W_filter = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1))
b_filter = tf.Variable(tf.constant(0.1, shape=[num_filters]))
x_embed_expanded = tf.expand_dims(x_embed, -1)
conv = tf.nn.conv2d(x_embed_expanded, W_filter, strides=[1, 1, 1, 1], padding="VALID")
h = tf.nn.relu(tf.nn.bias_add(conv, b_filter), name="relu")
pooled = tf.nn.max_pool(h, ksize=[1, self.params['max_domain_segments_len'] - filter_size + 1, 1, 1],
strides=[1, 1, 1, 1], padding='VALID')
pooled_outputs.append(pooled)
h_pool = tf.concat(pooled_outputs, axis=3)
num_filters_total = num_filters * len(filter_sizes)
output_vec = tf.reshape(h_pool, [-1, num_filters_total])
logits = tf.contrib.layers.fully_connected(output_vec, num_outputs=n_rnn_neurons, activation_fn=tf.nn.relu)
logits = tf.contrib.layers.fully_connected(logits, self.params['num_targets'], activation_fn=tf.nn.relu)
crossentropy = tf.losses.sparse_softmax_cross_entropy(labels=y, logits=logits)
loss_mean = tf.reduce_mean(crossentropy)
optimizer = tf.train.AdamOptimizer(learning_rate=lr_rate)
training_op = optimizer.minimize(loss_mean)
prediction = tf.argmax(logits, axis=-1)
is_correct = tf.nn.in_top_k(logits, y, 1) # logits are unscaled, but here we only care the argmax
n_correct = tf.reduce_sum(tf.cast(is_correct, tf.float32))
accuracy = tf.reduce_mean(tf.cast(is_correct, tf.float32))
init = tf.global_variables_initializer()
with tf.Session() as sess:
init.run()
......