Tensorboard weights histogram only last layer visible change

I added a TensorBoard visualization to my network and noticed that only the outlier changes a lot. Why do the weights of the network not change a lot? This is especially visible in the overlay histograms.

Histograms same but overlay view

My model

def neural_network_model(inputdata):
    """The blueprint of the network and the tensorboard information
        :param inputdata: the placeholder for the inputdata
        :returns: the output of the network?
            """
    W1 = tf.get_variable("W1", shape=[set.input, nodes_h1],
                         initializer=tf.contrib.layers.xavier_initializer())
    B1 = tf.get_variable("B1", shape=[nodes_h1],
                         initializer=tf.random_normal_initializer())
    layer1 = tf.matmul(inputdata, W1)
    layer1_bias = tf.add(layer1, B1)
    layer1_act = tf.nn.relu(layer1)

    W2 = tf.get_variable("W2", shape=[nodes_h1, nodes_h2],
                         initializer=tf.contrib.layers.xavier_initializer())
    B2 = tf.get_variable("B2", shape=[nodes_h2],
                         initializer=tf.random_normal_initializer())
    layer2 = tf.matmul(layer1_act, W2)
    layer2_bias = tf.add(layer2, B2)
    layer2_act = tf.nn.relu(layer2)

    W3 = tf.get_variable("W3", shape=[nodes_h2, nodes_h3],
                         initializer=tf.contrib.layers.xavier_initializer())
    B3 = tf.get_variable("B3", shape=[nodes_h3],
                         initializer=tf.random_normal_initializer())

    layer3 = tf.matmul(layer2_act, W3)
    layer3_bias = tf.add(layer3, B3)
    layer3_act = tf.nn.relu(layer3)
    WO = tf.get_variable("WO", shape=[nodes_h3, set.output],
                         initializer=tf.contrib.layers.xavier_initializer())
    layerO = tf.matmul(layer3_act, WO)

    with tf.name_scope('Layer1'):
        tf.summary.histogram("weights", W1)
        tf.summary.histogram("layer", layer1)
        tf.summary.histogram("bias", layer1_bias)
        tf.summary.histogram("activations", layer1_act)
    with tf.name_scope('Layer2'):
        tf.summary.histogram("weights", W2)
        tf.summary.histogram("layer", layer2)
        tf.summary.histogram("bias", layer2_bias)
        tf.summary.histogram("activations", layer2_act)
    with tf.name_scope('Layer3'):
        tf.summary.histogram("weights", W3)
        tf.summary.histogram("layer", layer3)
        tf.summary.histogram("bias", layer3_bias)
        tf.summary.histogram("activations", layer3_act)
    with tf.name_scope('Output'):
        tf.summary.histogram("weights", WO)
        tf.summary.histogram("layer", layerO)
    return layerO

My understanding of the training process is that the weight should get adjusted which barely happens in the images. The loss goes done however, I've trained the network for 10000 epochs so I expected a bit more change overall. Especially the lack of change in the weights I don't understand.