How to fix grid search issues in LSTM

user_18526455 0
Viewed 401

I want to do grid search for my model, and here my model shown below.

def model_lstm(time_steps=24, n_features=40,
                optimizer = tf.keras.optimizers.Adam,
                learning_rate = 0.001,
                dropout = 0.5,
                n_units_LSTM = 256,
                n_units_1 = 200):
    activation2 = 'relu'
    model = Sequential()
    model.add(LSTM(units=n_units_LSTM, input_shape=(time_steps, n_features)))
    model.add(Dropout(dropout))
    model.add(Dense(units=n_units_1, activation=activation2))
    model.add(Dense(units=n_units_1, activation=activation2))
    model.add(Dense(units=n_units_1, activation=activation2))
    model.add(Dense(units=n_units_1, activation=activation2))
    model.add(Dense(units=n_units_1, activation=activation2))
    model.add(Dense(1, activation='sigmoid'))
    optimizer = optimizer(learning_rate=learning_rate)
    model.compile(loss='binary_crossentropy', 
                  optimizer=optimizer,
                  metrics = ['accuracy'])
    print(model.summary())
    return model

there are 4 parameters that i want to grid, learning rate, n_unit_LSTM, n_units_1, and dropout. i want to get the same value of each dense layer. so i add variable named n_units_1.

def grid_search(data, metode):
        keras.backend.clear_session()
        x_train, y_train = sequence_data(data)
        params = {
                'learning_rate' : [0.01, 0.001, 0.0001],
                'n_units_LSTM' : [64,128,256],
                'n_units_1' : [50, 100, 150, 200, 250, 300],
                'dropout' : [0.1, 0.2, 0.25, 0.5]
                }
        scorers = {
                'accuracy_score' : make_scorer(accuracy_score)
                }
        model = KerasClassifier(build_fn=model_lstm, verbose=0)
        cv = cross_validate(5)
        start = time.time()
        grid = GridSearchCV(estimator = model,
                        param_grid = params,
                        n_jobs = -1,
                        verbose = 1,
                        cv = cv, 
                        scoring = scorers,
                        refit = 'accuracy_score')
        tf.random.set_seed(123)
        grid.fit(x_train, y_train)
        end = time.time()
        runtime = end-start
        result = grid.best_params_
        results = grid.cv_results_
        print('-----------------------------------------')
        print(f'Best Parameter : {result}')
        print(f'Runtime : {runtime}')
        print('-----------------------------------------')
        return grid

and when i run my grid, i got an error.

A task has failed to un-serialize. Please ensure that the arguments of the function are all picklable.
grid-search lstm deep-learning python
1 Answers

Step 1: Lets get the model to predict an age based on 9 abalone features. I changed the model to mean_squared_error for the age prediction Step 2: Add a gridsearch (pending)

import pandas as pd
from pandas import Series
from pandas import concat
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
from tensorflow.keras.models import Sequential
from tensorflow.keras import Model
from tensorflow.keras.layers import Input, LSTM, Embedding,Flatten,Dropout, Dense, Concatenate, TimeDistributed, Bidirectional,Attention
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import make_scorer
from sklearn.metrics import accuracy_score
from tensorflow.keras.wrappers.scikit_learn import KerasClassifier



def create_dataset(dataset, look_back=8):
    dataX, dataY = [], []
    for i in range(len(dataset)-look_back-1):
        a = dataset[i:(i+look_back)]
        dataX.append(a)
        dataY.append(dataset[i + look_back])
    return np.array(dataX), np.array(dataY)

def difference(dataset, interval=1):
    diff = list()
    for i in range(interval, len(dataset)):
        value = dataset[i] - dataset[i - interval]
        diff.append(value)
    return Series(diff)

def timeseries_to_supervised(data, lag=1):
    df = pd.DataFrame(data)
    columns = [df.shift(i) for i in range(1, lag+1)]
    columns.append(df)
    df = concat(columns, axis=1)
    return df

def scale(train, test):
    # fit scaler
    scaler = MinMaxScaler(feature_range=(-1, 1))
    scaler = scaler.fit(train)
    # transform train
    train = train.reshape(train.shape[0], train.shape[1])
    train_scaled = scaler.transform(train)
    # transform test
    test = test.reshape(test.shape[0], test.shape[1])
    test_scaled = scaler.transform(test)
    return scaler, train_scaled, test_scaled

df=pd.read_csv('abalone.data',names=['Sex'
,'Length'
,'Diameter'
,'Height'
,'Whole weight'
,'Shucked weight'
,'Viscera weight'
,'Shell weight'
,'Rings'])

df.reset_index(inplace=True)

df['Age']=df['Rings'].apply(lambda x: x/1.5)


encoder=LabelEncoder()
df['Sex']=encoder.fit_transform(df['Sex'])

raw_values = df.values
#diff_values = difference(raw_values, 1)

features=10
#supervised = timeseries_to_supervised(diff_values, features)
supervised = timeseries_to_supervised(raw_values, features)
supervised_values = supervised.values[features:,:]

train_size = int(len(df) * 0.70)
test_size = len(df) - train_size

# split data into train and test-sets
train, test = supervised_values[0:-train_size, :], supervised_values[-train_size:, :]

def model_lstm(X,time_steps=24, n_features=9,
                optimizer = tf.keras.optimizers.Adam,
                learning_rate = 0.001,
                dropout = 0.5,
                n_units_LSTM = 256,
                n_units_1 = 200,
                batch_size=1
              ):
    activation2 = 'relu'
    
    model = Sequential()
    model.add(LSTM(units=n_units_LSTM, batch_input_shape=(batch_size, X.shape[1], X.shape[2]), stateful=True))
    model.add(Dropout(dropout))
    model.add(Dense(units=n_units_1, activation=activation2))
    model.add(Dense(units=n_units_1, activation=activation2))
    model.add(Dense(units=n_units_1, activation=activation2))
    model.add(Dense(units=n_units_1, activation=activation2))
    model.add(Dense(units=n_units_1, activation=activation2))
    model.add(Dense(1))
    optimizer = optimizer(learning_rate=learning_rate)
    model.compile(loss='mean_squared_error', 
                  optimizer=optimizer)
    print(model.summary())
    return model


scaler, train_scaled, test_scaled = scale(train, test)

X_train, y_train = train_scaled[:, 0:-1], train_scaled[:, -1]
X_train = X_train.reshape(X_train.shape[0], 1, X_train.shape[1])

X_test, y_test = test_scaled[:, 0:-1], test_scaled[:, -1]
X = X_test.reshape(X_test.shape[0], 1, X_test.shape[1])


#print(y)
model=model_lstm(X_train)    
history=model.fit(X_train,y_train, epochs=100)

model.summary()

plt.plot(history.history['loss'])
plt.title('loss accuracy')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()
user_40011770
Related