I am trying to run video captioning on tensorflow. However, when I attempt to run train.py I am getting this warning and the problem is that the saved models encoder_model.h5 and decoder_model_weights.h5 are remain the same size no matter how many epoch you set (even epoch = 500) and the system performance is too bad at predicting. I think there is an error in the way to save the model. Can any one help me please.
training function:
def train_model(self):
"""
an encoder decoder sequence to sequence model
reference : https://arxiv.org/abs/1505.00487
"""
encoder_inputs = Input(shape=(config.time_steps_encoder, config.num_encoder_tokens), name="encoder_inputs")
encoder = LSTM(config.latent_dim, return_state=True, return_sequences=True, name='encoder_lstm')
_, state_h, state_c = encoder(encoder_inputs)
encoder_states = [state_h, state_c]
decoder_inputs = Input(shape=(config.time_steps_decoder, config.num_decoder_tokens), name="decoder_inputs")
decoder_lstm = LSTM(config.latent_dim, return_sequences=True, return_state=True, name='decoder_lstm')
decoder_outputs, _, _ = decoder_lstm(decoder_inputs, initial_state=encoder_states)
decoder_dense = Dense(config.num_decoder_tokens, activation='relu', name='decoder_relu')
decoder_outputs = decoder_dense(decoder_outputs)
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
# model.summary()
training_list, validation_list = self.preprocessing()
train = self.load_dataset(training_list)
valid = self.load_dataset(validation_list)
early_stopping = EarlyStopping(monitor='val_loss', patience=4, verbose=1, mode='min')
# Run training
opt = keras.optimizers.Adam(lr=0.0003)
reduce_lr = keras.callbacks.ReduceLROnPlateau(monitor="val_loss",
factor=0.1, patience=5, verbose=0,
mode="auto")
model.compile(metrics=['accuracy'], optimizer=opt, loss='categorical_crossentropy')
validation_steps = len(validation_list)//self.batch_size
steps_per_epoch = len(training_list)//self.batch_size
model.fit(train, validation_data=valid, validation_steps=validation_steps,
epochs=self.epochs, steps_per_epoch=steps_per_epoch,
callbacks=[reduce_lr]) # , early_stopping
if not os.path.exists(self.save_model_path):
os.makedirs(self.save_model_path)
self.encoder_model = Model(encoder_inputs, encoder_states)
decoder_state_input_h = Input(shape=(self.latent_dim,))
decoder_state_input_c = Input(shape=(self.latent_dim,))
decoder_states_inputs = [decoder_state_input_h, decoder_state_input_c]
decoder_outputs, state_h, state_c = decoder_lstm(
decoder_inputs, initial_state=decoder_states_inputs)
decoder_states = [state_h, state_c]
decoder_outputs = decoder_dense(decoder_outputs)
self.decoder_model = Model(
[decoder_inputs] + decoder_states_inputs,
[decoder_outputs] + decoder_states)
# self.encoder_model.summary()
# self.decoder_model.summary()
# saving the models
self.encoder_model.save( os.path.join(self.save_model_path, 'encoder_model.h5'))
self.decoder_model.save_weights(os.path.join(self.save_model_path, 'decoder_model_weights.h5'))
with open(os.path.join(self.save_model_path, 'tokenizer' + str(self.num_decoder_tokens)), 'wb') as file:
joblib.dump(self.tokenizer, file)
output of running trian.py for 5 epoch:
C:\Users\MSI\anaconda3\envs\my_envir_gpu\lib\site-packages\keras\optimizers\optimizer_v2\adam.py:110: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.
super(Adam, self).__init__(name, **kwargs)
Epoch 1/5
2022-09-07 19:36:09.455987: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations: AVX AVX2
To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
2022-09-07 19:36:10.160168: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1532] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 3497 MB memory: -> device: 0, name: NVIDIA GeForce RTX 3060 Laptop GPU, pci bus id: 0000:01:00.0, compute capability: 8.6
2022-09-07 19:36:21.205093: I tensorflow/stream_executor/cuda/cuda_dnn.cc:384] Loaded cuDNN version 8401
2022-09-07 19:36:22.038099: I tensorflow/stream_executor/cuda/cuda_blas.cc:1786] TensorFloat-32 will be used for the matrix multiplication. This will only be logged once.
245/245 [==============================] - 32s 112ms/step - loss: 5.7208 - accuracy: 0.2957 - val_loss: 5.3780 - val_accuracy: 0.3274 - lr: 3.0000e-04
Epoch 2/5
245/245 [==============================] - 27s 111ms/step - loss: 5.2917 - accuracy: 0.3304 - val_loss: 5.3651 - val_accuracy: 0.3324 - lr: 3.0000e-04
Epoch 3/5
245/245 [==============================] - 27s 112ms/step - loss: 5.2468 - accuracy: 0.3328 - val_loss: 5.2229 - val_accuracy: 0.3407 - lr: 3.0000e-04
Epoch 4/5
245/245 [==============================] - 28s 113ms/step - loss: 5.1038 - accuracy: 0.3347 - val_loss: 4.9872 - val_accuracy: 0.3355 - lr: 3.0000e-04
Epoch 5/5
245/245 [==============================] - ETA: 0s - loss: 4.9417 - accuracy: 0.3367 WARNING:tensorflow:Your input ran out of data; interrupting training. Make sure that your dataset or generator can generate at least `steps_per_epoch * epochs` batches (in this case, 43 batches). You may need to use the repeat() function when building your dataset.
245/245 [==============================] - 28s 115ms/step - loss: 4.9417 - accuracy: 0.3367 - val_loss: 4.8479 - val_accuracy: 0.3398 - lr: 3.0000e-04
WARNING:tensorflow:Compiled the loaded model, but the compiled metrics have yet to be built. `model.compile_metrics` will be empty until you train or evaluate the model.
C:\GGGGGGGG\Projects\Video_captioning_shyne _updated\train.py:94: VisibleDeprecationWarning: Creating an ndarray from ragged nested sequences (which is a list-or-tuple of lists-or-tuples-or ndarrays with different lengths or shapes) is deprecated. If you meant to do this, you must specify 'dtype=object' when creating the ndarray.
train_sequences = np.array(train_sequences)