Problem in implementing a CNN with Tensorflow 2.0 for digit recognition

I'm trying to implement a CNN in order to recognize single numbers coming from ttf files(RGB, in 3 channels). It does not learn and always stuck at some point. What I m doing wrong? Is the structure right? All examples I found are implemented with Keras or with Tensorflow 1.x. I tried multiples options for batchs, filters, learning rates...

# image
image_channels = 3
image_width = 30
image_height = 30
image_flatten = image_width * image_height
# batch
batch_size = 200
# Training parameters.
learning_rate = 0.005
training_epoch = 45000
display_step = 10
# Network parameters.
number_of_filters = 64
neurons_layer_1 = 1024
neurons_layer_2 = 512
neurons_output = 10
dropout_rate = 0.4

weights = {
    'W1': tf.Variable(tf.random.truncated_normal([3, 3, 3, number_of_filters], dtype=tf.float32, stddev=0.1),
                      name='Weights_1'),
    'W2': tf.Variable(tf.random.truncated_normal([14 * 14 * number_of_filters, neurons_layer_1], stddev=0.1),
                      name='Weights_2'),
    'W3': tf.Variable(tf.random.truncated_normal([neurons_layer_1, neurons_layer_2], stddev=0.1), name='Weights_3'),
    'W4': tf.Variable(tf.random.truncated_normal([neurons_layer_2, neurons_output], stddev=0.1), name='Weights_3')
}
biases = {
    'b1': tf.Variable(tf.constant(0.1, shape=[number_of_filters]), name='bias_1'),
    'b2': tf.Variable(tf.constant(0.1, shape=[neurons_layer_1]), name='bias_2'),
    'b3': tf.Variable(tf.constant(0.1, shape=[neurons_layer_2]), name='bias_2'),
    'b4': tf.Variable(tf.constant(0.1, shape=[neurons_output]), name='bias_3')
}

# Create model.
def neural_net(inputdata, debug=False):
    if debug:
        print("--------------- starting step:")
    # ## EXTRACTING FEATURES
    result_conv2d_1 = tf.nn.conv2d(inputdata, weights['W1'], strides=[1, 2, 2, 1], padding='VALID') + biases['b1']
    result_relu_1 = tf.nn.relu(result_conv2d_1)
    result_maxpool_1 = tf.nn.max_pool(result_relu_1, ksize=[1, 1, 1, 1], strides=[1, 1, 1, 1], padding='VALID')
    if debug:
        print("result_conv2d_1-->",result_conv2d_1.shape)
        print("result_relu_1  -->",result_relu_1.shape)
        print("maxpool1.shape --> ", result_maxpool_1.shape)

    # flat
    flatten = tf.reshape(result_maxpool_1, shape=(tf.shape(result_maxpool_1)[0], -1))
    if debug:
        print("flatten.shape  --> ", flatten.shape)

    # ## CLASIFFICATION
    # layer 1 - dense / fully connected
    layer1 = tf.nn.relu(tf.add(tf.matmul(flatten, weights['W2']), biases['b2']))
    # dropout
    #layer_drop = tf.nn.dropout(layer1, rate=dropout_rate)
    # layer 2 - dense / fully connected
    layer2 = tf.nn.relu(tf.add(tf.matmul(layer1, weights['W3']), biases['b3']))
    # layer 2 - dense / fully connected
    layer3 = tf.nn.relu(tf.add(tf.matmul(layer2, weights['W4']), biases['b4']))
    result = tf.nn.softmax(layer3)

    return result

# Cross-Entropy loss function.
def cross_entropy(predicted, desired):
    desired2 = tf.one_hot(desired, depth=neurons_output)
    cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=desired2, logits=predicted, name="cross_entropy")
    mean = tf.reduce_mean(cross_entropy, name="cross_entropy_mean")
    return mean


# Accuracy metric.
def accuracy(y_pred, y_true):
    # Predicted class is the index of highest score in prediction vector (i.e. argmax).
    correct_prediction = tf.equal(tf.argmax(y_pred, 1), tf.cast(y_true, tf.int64))
    return tf.reduce_mean(tf.cast(correct_prediction, tf.float32), axis=-1)


optimizer = tf.optimizers.Adam(learning_rate)

# Optimization process.
def run_optimization(inputdata, expected, debug=False):
    # Wrap computation inside a GradientTape for automatic differentiation.
    with tf.GradientTape() as g:
        predicted = neural_net(inputdata, debug)
        loss = cross_entropy(predicted, expected)
        #print(f'\r          loss: {loss}', end="")
    # Variables to update, i.e. train able variables.
    trainable_variables = list(weights.values()) + list(biases.values())
    # Compute gradients.
    gradients = g.gradient(loss, trainable_variables)
    # Update W and b following gradients.
    optimizer.apply_gradients(zip(gradients, trainable_variables))    


datafile = open("W:/machinelearning/dataset_ttf_single/ttf_single_full.p", "rb")
with datafile:
    for step in range(training_steps):
        # Run the optimization to update W and b values.
        batch_images, batch_labels = load_batch_dataset(datafile, batch_size)
        run_optimization(batch_images, batch_labels, False)
        if step % display_step == 0:
            predicted = neural_net(batch_images)
            loss = cross_entropy(predicted, batch_labels)
            acc = accuracy(predicted, batch_labels)
            print("\nstep: %i, loss: %f, accuracy: %f" % (step, loss, acc))
datafile.close()

The results are usually like this:

step: 330, loss: 2.341150, accuracy: 0.120000

step: 340, loss: 2.396150, accuracy: 0.065000

step: 350, loss: 2.361150, accuracy: 0.100000

step: 360, loss: 2.376150, accuracy: 0.085000

step: 370, loss: 2.371150, accuracy: 0.090000

step: 380, loss: 2.331150, accuracy: 0.130000

step: 390, loss: 2.326150, accuracy: 0.135000

It stacks between 0.10 and 0.15 usually. I followed an example of CNN MNIST in Tensorflow 2.0, it works but I'm not able to do it with different images not coming from MNIST. I'm sure the optimizers and the CNN are not perfect but I'm trying to start from the beginning creating a little piece of CNN and then improve it, I want to learn how it works, but something easy like this and very similar to MNIST, I'm stuck on it.

Someone can give me some clue about what is going on? Thanks!