How to apply the ConvLSTM layers

user_14128879 0
Viewed 1105

I was doing a classification machine learning with an input of (700,50,34) (batch, step,features)

def convLSTM_model(X_train, y_train, X_test, y_test, num_classes,loss, batch_size=68, units=128, learning_rate=0.005,
                           epochs=20, dropout=0.2, recurrent_dropout=0.2):
    class myCallback(tf.keras.callbacks.Callback):

        def on_epoch_end(self, epoch, logs={}):
            if (logs.get('acc') > 0.9):
                print("\nReached 90% accuracy so cancelling training!")
                self.model.stop_training = True

    callbacks = myCallback()

    model = tf.keras.models.Sequential()
    model.add(Masking(mask_value=0.0, input_shape=(None,X_train.shape[0],X_train.shape[1], X_train.shape[2])))
    
    model.add(ConvLSTM2D(filters=40, kernel_size=(3, 3), padding="same", return_sequences=True))
    model.add(BatchNormalization())

    model.add(Bidirectional(LSTM(units, dropout=dropout, recurrent_dropout=recurrent_dropout, return_sequences=True)))
    model.add(Bidirectional(LSTM(units, dropout=dropout, recurrent_dropout=recurrent_dropout, return_sequences=True)))
    model.add(Bidirectional(LSTM(units, dropout=dropout, recurrent_dropout=recurrent_dropout)))
    model.add(Dense(30, activation='relu'))
    model.add(Dense(10, activation='relu'))
    model.add(Dense(num_classes, activation='softmax'))

    adamopt = tf.keras.optimizers.Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-8)

    model.compile(loss=loss,
                  optimizer=adamopt,
                  metrics=['accuracy'])

    history = model.fit(X_train, y_train,
                        batch_size=batch_size,
                        epochs=epochs,
                        validation_data=(X_test, y_test),
                        verbose=1,
                        callbacks=[callbacks])

    score, acc = model.evaluate(X_test, y_test,
                                batch_size=batch_size)

    yhat = model.predict(X_test)

    return history, that

Apparently, changing the input_shape and simply adding

model.add(ConvLSTM2D(filters=40, kernel_size=(3, 3), padding="same", return_sequences=True))
model.add(BatchNormalization())

does not work.

ValueError: Dimension 1 in both shapes must be equal, but are 708 and 501264. Shapes are [?,708,50,40] and [?,501264,2500,40]. for 'conv_lst_m2d/while/Select' (op: 'Select') with input shapes: [?,501264,2500,40], [?,708,50,40], [?,708,50,40].

How should I approach? Is there any suggestion on the number of filter?

tensorflow deep-learning machine-learning python
2 Answers

Try this one, I believe the CNN layer should be implemented before the masking layer.

    model = tf.keras.models.Sequential()
    
    model.add(Conv1D(filters=32, kernel_size=8, strides=1, activation="relu", padding="same",input_shape=(X_train.shape[1], X_train.shape[2])))
    model.add(MaxPooling1D(pool_size = 2))
    model.add(Conv1D(filters=16, kernel_size=8, strides=1, activation="relu", padding="same"))
    model.add(MaxPooling1D(pool_size = 2))
    
    
    model.add(Masking(mask_value=0.0))
    model.add(LSTM(units, dropout=dropout, recurrent_dropout=recurrent_dropout, return_sequences=True))
    model.add(Bidirectional(LSTM(units, dropout=dropout, recurrent_dropout=recurrent_dropout, return_sequences=True)))
    model.add(Bidirectional(LSTM(units, dropout=dropout, recurrent_dropout=recurrent_dropout, return_sequences=True)))
    model.add(Bidirectional(LSTM(units, dropout=dropout, recurrent_dropout=recurrent_dropout)))
    model.add(Dense(30, activation='relu'))
    model.add(Dense(10, activation='relu'))
    
    model.add(Dense(num_classes, activation='softmax'))
user_147113930

I now tried to use your code. You will get better anwers at stackoverflow when you supply a minimum working example. I build some test data with the same shape that you used.

When usind an LSTMConv2D I could reproduce your problem. But since your Data just doesn´t fit to the Conv2D Layer and substituted this with a normal LSTM Layer. If instead you want to convolute the timestep I would suggest using an CONV1D Layer first and LSTM afterwards.

Minimum working example:

import tensorflow as tf
import numpy as np
from tensorflow.python.keras.layers import Masking, ConvLSTM2D, LSTM, Bidirectional, Dense
from tensorflow.python.keras.layers.normalization_v2 import BatchNormalization

batch_size=68
units=128
learning_rate=0.005
epochs=20
dropout=0.2
recurrent_dropout=0.2
X_train = np.random.rand(700, 50,34)
y_train = np.random.choice([0, 1], 700)
X_test = np.random.rand(100, 50, 34)
y_test = np.random.choice([0, 1], 100)
loss = tf.losses.binary_crossentropy

model = tf.keras.models.Sequential()
model.add(Masking(mask_value=0.0, input_shape=(X_train.shape[1], X_train.shape[2])))
# uncomment the line beneath for convolution
# model.add(Conv1D(filters=32, kernel_size=8, strides=1, activation="relu", padding="same"))
model.add(LSTM(units, dropout=dropout, recurrent_dropout=recurrent_dropout, return_sequences=True))
model.add(BatchNormalization())

model.add(Bidirectional(LSTM(units, dropout=dropout, recurrent_dropout=recurrent_dropout, return_sequences=True)))
model.add(Bidirectional(LSTM(units, dropout=dropout, recurrent_dropout=recurrent_dropout, return_sequences=True)))
model.add(Bidirectional(LSTM(units, dropout=dropout, recurrent_dropout=recurrent_dropout)))
model.add(Dense(30, activation='relu'))
model.add(Dense(10, activation='relu'))
model.add(Dense(1, activation='sigmoid'))

adamopt = tf.keras.optimizers.Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-8)

model.compile(loss=loss,
              optimizer=adamopt,
              metrics=['accuracy'])

history = model.fit(X_train, y_train,
                    batch_size=batch_size,
                    epochs=epochs,
                    validation_data=(X_test, y_test),
                    verbose=1)

score, acc = model.evaluate(X_test, y_test,
                            batch_size=batch_size)

yhat = model.predict(X_test)
user_83840470
Related