I am using tf.keras with tensorflow 2.x and try to tune hyperparameter of a simple MLP model with RandomizedSearchCV in scikit-learn.
Pakages and python version: python 3.7.5 , scikit-learn 0.22.2 , and tensorflow 2.1.0
I try to search for:
- the best depth(numbers) of hidden layes(i.e.
n_hidden) - the best number of neurous in each layer, where each hidden layer has the same number of neurous(i.e.
n_neurous) - the best learning rate(i.e.
lr)
from sklearn.model_selection import RandomizedSearchCV
from scipy.stats import reciprocal
from tensorflow import keras
import tensorflow as tf
import numpy as np
def build_keras(n_hidden = 3, n_neurous = 30, lr = 1e-3, input_shape = [8]):
# use for building KerasRegression
model = keras.models.Sequential()
model.add(keras.layers.InputLayer(input_shape=input_shape))
# with n_hidden hidden Dense layers
for i in range(n_hidden):
model.add(keras.layers.Dense(n_neurous, activation='relu'))
# Output layer
model.add(keras.layers.Dense(1))
model.compile(loss='mse', optimizer=keras.optimizers.SGD(lr=lr))
return model
keras_reg = keras.wrappers.scikit_learn.KerasRegressor(build_keras)
search_params = {
'n_hidden':[1,3,5,7],
'n_neurous':np.arange(1,100),
'lr':reciprocal(3e-4, 3e-2)
}
rnd_search = RandomizedSearchCV(keras_reg, search_params, cv=3, n_iter=10)
rnd_search.fit(X_train_trans, y_train, epochs=100, validation_data=(X_val_trans, y_val),
callbacks=[keras.callbacks.EarlyStopping(patience=5)])
rnd_search.best_params_ #works fine
rnd_search.best_score_ #works fine
rnd_search_cv.best_estimator_ #Error: No attribute