Long prediction time when using converter.optimization in a VGG16 model and Tensorflow lite

I wrote a model that is based on VGG16 and I just only added two additional convolution layers. The output is an array of size 16x16x1 which is just the result of simple binary classification. I used TensorFlow-lite and the code is based on the documentation available. The problem is that when I'm making a prediction using the model it takes a looong time (near 5 mins) to give me the result. I'm using Tensorflow 2.4 on GPU, Python 3.7, my graphics card is GTX 1660Ti (mobile version) and the CPU is intel i7 9750H.
The code is available below.

import tensorflow as tf
import os
import time
import numpy as np
import cv2
import keras
import pathlib

saved_model_dir= 'model/'
saved_modelh5 = 'model.h5'
dataset_path = 'bound box dataset/img'
out_path = 'converted_model.tflite'
num_calibration_steps = 10


#-----------------------------------------------------------
images = []
for file in os.listdir(dataset_path):
    img = cv2.imread( os.path.join(dataset_path,file) )
    images.append(img)
images = np.array( images )

imgs_tensor = tf.cast( images, dtype = tf.float32)/255.0
ds = tf.data.Dataset.from_tensor_slices((imgs_tensor)).batch(1)
print('data loaded')

#-----------------------------------------------------------
def representative_dataset_gen():
  for input_value in ds.take(num_calibration_steps):
    yield [input_value]




#converter = tf.lite.TFLiteConverter.from_saved_model(saved_model_dir)
converter = tf.lite.TFLiteConverter.from_keras_model(keras.models.load_model(saved_modelh5))  
converter.optimizations = [tf.lite.Optimize.DEFAULT ]
#converter.representative_dataset = tf.lite.RepresentativeDataset( representative_dataset_gen )
#converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS_INT8]
tflite_model = converter.convert()


#------------------------------------------------------------
#with open(out_path, "wb")as f:
#    f.write(tflite_model)
print('converted')
tflite_model_file = pathlib.Path(out_path)
tflite_model_file.write_bytes(tflite_model)
print('Saved')



img = cv2.imread('bound box dataset/img/1.png')
input_data = img.reshape(1,512,512,3).astype(np.float32)/255.0

interpreter = tf.lite.Interpreter( model_content = tflite_model)
interpreter.allocate_tensors()

# Get input and output tensors.
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()

# Test model on random input data.
t = time.time()
input_shape = input_details[0]['shape']
#input_data = np.array(np.random.random_sample(input_shape), dtype=np.float32)
interpreter.set_tensor(input_details[0]['index'], input_data)

interpreter.invoke()

# The function `get_tensor()` returns a copy of the tensor data.
# Use `tensor()` in order to get a pointer to the tensor.
output_data = interpreter.get_tensor(output_details[0]['index'])

t = time.time() - t
print('predict time:',t)