tensorflow accuracy, val_accuracy remains the same while training

I build a CNN based on the Chest X-Ray Images (Pneumonia) dataset and for some reason when I train the model I get the same accuracy and val_accuracy over epochs.

train_ds = ImageDataGenerator()
traindata = train_ds.flow_from_directory(directory="../input/chest-xray-pneumonia/chest_xray/train",target_size=(IMG_HEIGHT,IMG_WIDTH),shuffle=True)
// Found 5216 images belonging to 2 classes.

test_ds = ImageDataGenerator()
testdata = test_ds.flow_from_directory(directory="../input/chest-xray-pneumonia/chest_xray/test",target_size=(IMG_HEIGHT,IMG_WIDTH),shuffle=True)
//Found 624 images belonging to 2 classes.

model = keras.Sequential([
    keras.layers.Conv2D(input_shape=(224,224,3),filters=64,kernel_size=(3,3),padding="same", activation="relu"),
    keras.layers.Conv2D(filters=64,kernel_size=(3,3),padding="same", activation="relu"),
    keras.layers.MaxPool2D(pool_size=(2,2),strides=(2,2)),
    keras.layers.Conv2D(filters=128, kernel_size=(3,3), padding="same", activation="relu"),
    keras.layers.Conv2D(filters=128, kernel_size=(3,3), padding="same", activation="relu"),
    keras.layers.MaxPool2D(pool_size=(2,2),strides=(2,2)),
    keras.layers.Conv2D(filters=256, kernel_size=(3,3), padding="same", activation="relu"),
    keras.layers.Conv2D(filters=256, kernel_size=(3,3), padding="same", activation="relu"),
    keras.layers.Conv2D(filters=256, kernel_size=(3,3), padding="same", activation="relu"),
    keras.layers.MaxPool2D(pool_size=(2,2),strides=(2,2)),
    keras.layers.Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu"),
    keras.layers.Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu"),
    keras.layers.Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu"),
    keras.layers.MaxPool2D(pool_size=(2,2),strides=(2,2)),
    keras.layers.Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu"),
    keras.layers.Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu"),
    keras.layers.Conv2D(filters=512, kernel_size=(3,3), padding="same", activation="relu"),
    keras.layers.MaxPool2D(pool_size=(2,2),strides=(2,2)),
    keras.layers.Flatten(),
    keras.layers.Dense(units=4096,activation="relu"),
#     keras.layers.Dropout(.5),
    keras.layers.Dense(units=4096,activation="relu"),
#     keras.layers.Dropout(.5),
    keras.layers.Dense(units=2, activation="softmax"),
])

opt = keras.optimizers.Adam(lr=0.001)

model.compile(optimizer=opt,
            loss="categorical_crossentropy",
            metrics=['accuracy'])


logdir = "logs\\training\\" + datetime.now().strftime("%Y%m%d-%H%M%S")

checkpoint = keras.callbacks.ModelCheckpoint("vgg16_1.h5", verbose=1, monitor='val_accuracy', save_best_only=True, mode='auto')
early = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=3)
tensorboard_callback = keras.callbacks.TensorBoard(log_dir=logdir)


hist = model.fit(traindata, 
      steps_per_epoch=STEPS_PER_EPOCH,
      epochs=100,
      validation_data=testdata,
      validation_steps=VALIDATION_STEPS,
      callbacks=[early, tensorboard_callback])

Epoch 1/100
163/163 [==============================] - 172s 1s/step - loss: 62.6885 - accuracy: 0.7375 - val_loss: 0.6827 - val_accuracy: 0.6250
Epoch 2/100
163/163 [==============================] - 157s 961ms/step - loss: 0.5720 - accuracy: 0.7429 - val_loss: 0.7133 - val_accuracy: 0.6250
Epoch 3/100
163/163 [==============================] - 159s 975ms/step - loss: 0.5725 - accuracy: 0.7429 - val_loss: 0.6691 - val_accuracy: 0.6250
Epoch 4/100
163/163 [==============================] - 159s 973ms/step - loss: 0.5721 - accuracy: 0.7429 - val_loss: 0.7036 - val_accuracy: 0.6250
Epoch 5/100
163/163 [==============================] - 158s 971ms/step - loss: 0.5715 - accuracy: 0.7429 - val_loss: 0.7169 - val_accuracy: 0.6250
Epoch 6/100
163/163 [==============================] - 160s 983ms/step - loss: 0.5718 - accuracy: 0.7429 - val_loss: 0.6982 - val_accuracy: 0.6250

I've tried changing the activation function for the last layer, adding dropout layers and toyed around with the number of neurons but nothing seemed to work. does anyone have an ideas what causes this strange behaviour?