Why does recognition rate drop after multiple online training epochs?

I am using tensorflow to do image recognition on the MNIST dataset. In each training epoch, I picked 10,000 random images and conducted online training with batch size of 1. The recognition rate increased for the first few epochs, however, after several epochs the recognition rate started to drop greatly. (In the first 20 epochs, the recognition rate goes up to ~94%. Afterwards, the recognition rate went from 90->50->40->30->20). What is the reason for this?

Also, with a batch size of 1, the performance is worse than when using a batch size of 100 (max recognition rate 94% vs. 96%). I looked through several references but there seems to be contradictory results on whether small or large batch sizes achieve better performance. What would be this case in this situation?

Edit: I also added a figure of the recognition rate of the training dataset and the test dataset.Recognition rate vs. epoch

I have attached a copy of the code below. Thanks for the help!

import tensorflow as tf
import numpy as np
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("/tmp/data/", one_hot = True)

#parameters
n_nodes_hl1 = 500
n_nodes_hl2 = 500
n_nodes_hl3 = 500
n_classes = 10
batch_size = 1
x = tf.placeholder('float', [None, 784])
y = tf.placeholder('float')

#model of neural network
def neural_network_model(data):
    hidden_1_layer = {'weights':tf.Variable(tf.random_normal([784, n_nodes_hl1])               , name='l1_w'),
                      'biases': tf.Variable(tf.random_normal([n_nodes_hl1])                    , name='l1_b')}

    hidden_2_layer = {'weights':tf.Variable(tf.random_normal([n_nodes_hl1, n_nodes_hl2])       , name='l2_w'),
                      'biases' :tf.Variable(tf.random_normal([n_nodes_hl2])                    , name='l2_b')}

    hidden_3_layer = {'weights':tf.Variable(tf.random_normal([n_nodes_hl2, n_nodes_hl3])       , name='l3_w'),
                      'biases' :tf.Variable(tf.random_normal([n_nodes_hl3])                    , name='l3_b')}

    output_layer   = {'weights':tf.Variable(tf.random_normal([n_nodes_hl3, n_classes])     , name='lo_w'),
                      'biases' :tf.Variable(tf.random_normal([n_classes])                   , name='lo_b')}

    l1 = tf.add(tf.matmul(data,hidden_1_layer['weights']), hidden_1_layer['biases'])
    l1 = tf.nn.relu(l1) 
    l2 = tf.add(tf.matmul(l1,hidden_2_layer['weights']), hidden_2_layer['biases'])
    l2 = tf.nn.relu(l2)     
    l3 = tf.add(tf.matmul(l2,hidden_3_layer['weights']), hidden_3_layer['biases'])
    l3 = tf.nn.relu(l3)
    output = tf.matmul(l3,output_layer['weights']) + output_layer['biases']    
return output

#train neural network
def train_neural_network(x):
    prediction = neural_network_model(x)
    cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=prediction, labels=y))
    optimizer = tf.train.AdamOptimizer().minimize(cost)
    hm_epoches = 100
    with tf.Session() as sess:
        sess.run(tf.global_variables_initializer())
        for epoch in range(hm_epoches):
            epoch_loss=0
            for batch in range (10000):
                epoch_x, epoch_y=mnist.train.next_batch(batch_size)                
                _,c =sess.run([optimizer, cost], feed_dict = {x:epoch_x, y:epoch_y})
                epoch_loss += c
            correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y,1))
            accuracy = tf.reduce_mean(tf.cast(correct, 'float'))
            print(epoch_loss)
            print('Accuracy_test:', accuracy.eval({x:mnist.test.images, y:mnist.test.labels}))
            print('Accuracy_train:', accuracy.eval({x:mnist.train.images, y:mnist.train.labels}))

train_neural_network(x)