How to realize the repeatability of neural network model?

user_17418925 0
Viewed 51

What if the following code is not repeatable?  Adding random seeds before iteration doesn't work either.  How do you achieve repeatability?

library(keras)
library(caret)

set.seed(123)

n = 400
s = seq(.1, n / 10, .1)
x1 = s * sin(s / 50) - rnorm(n) * 5 
x2 = s * sin(s) + rnorm(n) * 10
x3 = s * sin(s / 100) + 2 + rnorm(n) * 10
y1 = x1 + x2 + x3 + 2 + rnorm(n) * 2
y2 = x1 + x2 / 2 - x3 - 4 - rnorm(n)

df = data.frame(x1, x2, x3, y1, y2)

plot(s, df$y1, ylim = c(min(df), max(df)), type = "l", col = "blue")
lines(s, df$y2, type = "l", col = "red")
lines(s, df$x1, type = "l", col = "green")
lines(s, df$x2, type = "l", col = "yellow")
lines(s, df$x3, type = "l", col = "gray")

indexes = createDataPartition(df$x1, p = .85, list = F)
train = df[indexes,]
test = df[-indexes,]

xtrain = as.matrix(data.frame(train$x1, train$x2, train$x3))
ytrain = as.matrix(data.frame(train$y1, train$y2))
xtest = as.matrix(data.frame(test$x1, test$x2, test$x3))
ytest = as.matrix(data.frame(test$y1, test$y2))

in_dim = dim(xtrain)[2]
out_dim = dim(ytrain)[2] 

set.seed(165461)
model = keras_model_sequential() %>%
  layer_dense(units = 100, activation="relu", input_shape=in_dim) %>%
  layer_dense(units = 32, activation = "relu") %>%
  layer_dense(units = out_dim, activation = "linear")

model %>% compile(
  loss = "mse",
  optimizer = "adam")

model %>% summary()

set.seed(165461)
model %>% fit(xtrain, ytrain, epochs = 100, verbose = 1)
scores = model %>% evaluate(xtrain, ytrain, verbose = 0)
print(scores)

ypred = model %>% predict(xtest)

cat("y1 RMSE:", RMSE(ytest[, 1], ypred[, 1]))
cat("y2 RMSE:", RMSE(ytest[, 2], ypred[, 2]))

Running the above code repeatedly, the loss obtained in each iteration is different, and the final predicted value is also different. This is a terrible phenomenon, and I wish I could get the same result every time I run it.

keras neural-network r
1 Answers

You can try to set.seed before importing the packages and set a tensorflow random seed like this:

set.seed(42)

library(keras)
#> Warning: package 'keras' was built under R version 4.1.2
library(tensorflow)
#> Warning: package 'tensorflow' was built under R version 4.1.2
library(caret)
#> Warning: package 'caret' was built under R version 4.1.2
#> Loading required package: ggplot2
#> Warning: package 'ggplot2' was built under R version 4.1.2
#> Loading required package: lattice
#> 
#> Attaching package: 'caret'
#> The following object is masked from 'package:tensorflow':
#> 
#>     train

set_random_seed(42)
#> Loaded Tensorflow version 2.8.0

n = 400
s = seq(.1, n / 10, .1)
x1 = s * sin(s / 50) - rnorm(n) * 5 
x2 = s * sin(s) + rnorm(n) * 10
x3 = s * sin(s / 100) + 2 + rnorm(n) * 10
y1 = x1 + x2 + x3 + 2 + rnorm(n) * 2
y2 = x1 + x2 / 2 - x3 - 4 - rnorm(n)

df = data.frame(x1, x2, x3, y1, y2)

plot(s, df$y1, ylim = c(min(df), max(df)), type = "l", col = "blue")
lines(s, df$y2, type = "l", col = "red")
lines(s, df$x1, type = "l", col = "green")
lines(s, df$x2, type = "l", col = "yellow")
lines(s, df$x3, type = "l", col = "gray")


indexes = createDataPartition(df$x1, p = .85, list = F)
train = df[indexes,]
test = df[-indexes,]

xtrain = as.matrix(data.frame(train$x1, train$x2, train$x3))
ytrain = as.matrix(data.frame(train$y1, train$y2))
xtest = as.matrix(data.frame(test$x1, test$x2, test$x3))
ytest = as.matrix(data.frame(test$y1, test$y2))

in_dim = dim(xtrain)[2]
out_dim = dim(ytrain)[2] 

model = keras_model_sequential() %>%
  layer_dense(units = 100, activation="relu", input_shape=in_dim) %>%
  layer_dense(units = 32, activation = "relu") %>%
  layer_dense(units = out_dim, activation = "linear")

model %>% compile(
  loss = "mse",
  optimizer = "adam")

model %>% summary()
#> Model: "sequential"
#> ________________________________________________________________________________
#>  Layer (type)                       Output Shape                    Param #     
#> ================================================================================
#>  dense_2 (Dense)                    (None, 100)                     400         
#>  dense_1 (Dense)                    (None, 32)                      3232        
#>  dense (Dense)                      (None, 2)                       66          
#> ================================================================================
#> Total params: 3,698
#> Trainable params: 3,698
#> Non-trainable params: 0
#> ________________________________________________________________________________

model %>% fit(xtrain, ytrain, epochs = 100, verbose = 1)
scores = model %>% evaluate(xtrain, ytrain, verbose = 0)
print(scores)
#>     loss 
#> 1.920717

ypred = model %>% predict(xtest)

cat("y1 RMSE:", RMSE(ytest[, 1], ypred[, 1]))
#> y1 RMSE: 2.294828
cat("y2 RMSE:", RMSE(ytest[, 2], ypred[, 2]))
#> y2 RMSE: 1.200249

Created on 2022-07-11 by the reprex package (v2.0.1)

Run another time:

set.seed(42)

library(keras)
#> Warning: package 'keras' was built under R version 4.1.2
library(tensorflow)
#> Warning: package 'tensorflow' was built under R version 4.1.2
library(caret)
#> Warning: package 'caret' was built under R version 4.1.2
#> Loading required package: ggplot2
#> Warning: package 'ggplot2' was built under R version 4.1.2
#> Loading required package: lattice
#> 
#> Attaching package: 'caret'
#> The following object is masked from 'package:tensorflow':
#> 
#>     train

set_random_seed(42)
#> Loaded Tensorflow version 2.8.0

n = 400
s = seq(.1, n / 10, .1)
x1 = s * sin(s / 50) - rnorm(n) * 5 
x2 = s * sin(s) + rnorm(n) * 10
x3 = s * sin(s / 100) + 2 + rnorm(n) * 10
y1 = x1 + x2 + x3 + 2 + rnorm(n) * 2
y2 = x1 + x2 / 2 - x3 - 4 - rnorm(n)

df = data.frame(x1, x2, x3, y1, y2)

plot(s, df$y1, ylim = c(min(df), max(df)), type = "l", col = "blue")
lines(s, df$y2, type = "l", col = "red")
lines(s, df$x1, type = "l", col = "green")
lines(s, df$x2, type = "l", col = "yellow")
lines(s, df$x3, type = "l", col = "gray")


indexes = createDataPartition(df$x1, p = .85, list = F)
train = df[indexes,]
test = df[-indexes,]

xtrain = as.matrix(data.frame(train$x1, train$x2, train$x3))
ytrain = as.matrix(data.frame(train$y1, train$y2))
xtest = as.matrix(data.frame(test$x1, test$x2, test$x3))
ytest = as.matrix(data.frame(test$y1, test$y2))

in_dim = dim(xtrain)[2]
out_dim = dim(ytrain)[2] 

model = keras_model_sequential() %>%
  layer_dense(units = 100, activation="relu", input_shape=in_dim) %>%
  layer_dense(units = 32, activation = "relu") %>%
  layer_dense(units = out_dim, activation = "linear")

model %>% compile(
  loss = "mse",
  optimizer = "adam")

model %>% summary()
#> Model: "sequential"
#> ________________________________________________________________________________
#>  Layer (type)                       Output Shape                    Param #     
#> ================================================================================
#>  dense_2 (Dense)                    (None, 100)                     400         
#>  dense_1 (Dense)                    (None, 32)                      3232        
#>  dense (Dense)                      (None, 2)                       66          
#> ================================================================================
#> Total params: 3,698
#> Trainable params: 3,698
#> Non-trainable params: 0
#> ________________________________________________________________________________

model %>% fit(xtrain, ytrain, epochs = 100, verbose = 1)
scores = model %>% evaluate(xtrain, ytrain, verbose = 0)
print(scores)
#>     loss 
#> 1.920717

ypred = model %>% predict(xtest)

cat("y1 RMSE:", RMSE(ytest[, 1], ypred[, 1]))
#> y1 RMSE: 2.294828
cat("y2 RMSE:", RMSE(ytest[, 2], ypred[, 2]))
#> y2 RMSE: 1.200249

Created on 2022-07-11 by the reprex package (v2.0.1)

As you can see, the results are exactly the same!

set_random_seed according to documentation:

Sets all random seeds needed to make TensorFlow code reproducible.

user_142827140
Related