I am trying to build a Siamese-Model that abstractly looks like this:

When I try to train this model on the DeepFashion dataset, the loss increases over time until a constant value is reached. At this point the Model is returning Constant Embeddings.

To Validate if all Embeddings are the Same i calcuated the distances between all A, P, N pairs:

is_constant = lambda a, b: tf.math.reduce_sum((a - b)) == 0

for batch in (dataset["val"].take(1)): #siamese_model.validate_embedding
  anchors, positives, negatives = siamese_network(a)
  for a, p, n in (zip(anchors, positives, negative)):
    all_const = all([is_constant(a, p), is_constant(a, n), is_constant(p, n)])
  
    print(a)
    print(p)
    print(n)
    print("All Embds. Constant", all_const)
    print("*"*72)
  break

This leads to the same result (number of batch sizes times):

tf.Tensor(
[-0.03565513 -0.06550519  0.01807223 ... -0.0639673  -0.01235839
  0.04788744], shape=(2048,), dtype=float32)
tf.Tensor(
[-0.03565513 -0.06550519  0.01807223 ... -0.0639673  -0.01235839
  0.04788744], shape=(2048,), dtype=float32)
tf.Tensor(
[-0.03565513 -0.06550519  0.01807223 ... -0.0639673  -0.01235839
  0.04788744], shape=(2048,), dtype=float32)
All Embds. Constant True
************************************************************************

I cannot find a solution to this problem.

ResNet-Backbone

The creation of my ResNet backbone looks like this:

class ResnetIdentityBlock(tf.keras.Model):
    def __init__(self, kernel_size, filters) -> object:
        super(ResnetIdentityBlock, self).__init__(name='')
        filters1, filters2, filters3 = filters

        self.conv2a = tf.keras.layers.Conv2D(filters1, (1, 1), name="conv2a")
        self.bn2a = tf.keras.layers.BatchNormalization(name="bn2a")

        self.conv2b = tf.keras.layers.Conv2D(filters2, kernel_size, padding='same', name="conv2b")
        self.bn2b = tf.keras.layers.BatchNormalization(name="bn2b")

        self.conv2c = tf.keras.layers.Conv2D(filters3, (1, 1), name="conv2c")
        self.bn2c = tf.keras.layers.BatchNormalization(name="bn2c")

    def call(self, input_tensor, training=False):
        x = self.conv2a(input_tensor)
        x = self.bn2a(x, training=training)
        x = tf.nn.relu(x)

        x = self.conv2b(x)
        x = self.bn2b(x, training=training)
        x = tf.nn.relu(x)

        x = self.conv2c(x)
        x = self.bn2c(x, training=training)

        x += input_tensor
        return tf.nn.relu(x)

input_shape = (224, 224)
weights = "imagenet"  

R = resnet.ResNet50(
    weights=weights, input_shape=input_shape + (3,), include_top=False
)

embedding_model = Sequential([
    R,
    tf.keras.layers.Conv2D(256, (3, 3), activation='relu',
        kernel_initializer='he_uniform', padding='same',
        kernel_regularizer=l2(2e-4)),
    ResnetIdentityBlock(3, [64, 64, 256]),
    tf.keras.layers.AveragePooling2D(pool_size=(2, 2), strides=(1, 1), padding='same'),
    tf.keras.layers.Flatten(),
    tf.keras.layers.Dense(embedding_dim, activation=None, use_bias=False)
])

This Backbone is wrapped up as SiameseNetwork, which predicts Embeddings for all Inputs and calculates the distances between the Datapoints (Anchor&Positives and Anchor&Negatives).

anchor_input = layers.Input(name="anchor", shape=input_shape +(3,))
positive_input = layers.Input(name="positive", shape=input_shape +(3,))
negative_input = layers.Input(name="negative", shape=input_shape +(3,))

anchor_encoding   = back_bone(resnet.preprocess_input(anchor_input))
positive_encoding = back_bone(resnet.preprocess_input(positive_input))
negative_encoding = back_bone(resnet.preprocess_input(negative_input))

loss_output = TripletLoss(alpha=1.0)(anchor_encoding, positive_encoding, negative_encoding)

siamese_network = Model(
    inputs=[anchor_input, positive_input, negative_input], 
    outputs=loss_output
)

The Output Layer calculates the Triplet-Loss: My Triplet-Loss Layer:

@tf.function
def euclidean_distance(x, y):
    l2_distance = tf.math.square(tf.subtract(x, y))
    return tf.math.reduce_sum(l2_distance, axis=-1)
  
class TripletLoss(layers.Layer):
    def __init__(self, alpha, **kwargs):
        super(TripletLoss, self).__init__(**kwargs)
        self.alpha = alpha # im using Alpha = 1.0
        self.distance_layer = TripletDistance()

    def call(self, anchor, positive, negative):
        ap_distance = euclidean_distance(anchor, positive)
        an_distance = euclidean_distance(anchor, negative)

        loss = ap_distance - an_distance
        loss = tf.maximum(loss + self.alpha, 0.0)

        return loss

I'm using Adam-Optimizer with a Learning-Rate of 1e-4

Is there anything I am doing inherently wrong?