Python keras sequential model predicts the same value (y_train average) for all inputs

Viewed 68

I'm trying to build a sequential neural network with keras. I generate a dataset with inserting randoms in a known function and train my model with this dataset, long enough to get a steady loss. Then I ask the model to predict the x_train values, but instead of predicting something close to y_train, it returns the same value regardless of the input x. This value also happens to be the average of y_train values. I don't understand what I'm doing wrong and why this is happening.

I'm using the following function for training the model:

def train_model(x_train,y_train,batch_size,input_size,layer_sizes,activations,optimizer,epochs,loss='MeanSquaredError'):
  assert len(layer_sizes) == len(activations)
  n_layers=len(layer_sizes)
  model = Sequential()
  model.add(LayerNormalization(input_dim=input_size))
  model.add(Dense(layer_sizes[0],kernel_regularizer='l2',kernel_initializer='ones',activation=activations[0],input_dim=input_size,name='layer1'))
  
  for i in range(1,n_layers):
    model.add(Dense(layer_sizes[i],kernel_initializer='ones',activation=activations[i],name=f'layer{i+1}'))
  
  model.compile(
    optimizer = optimizer,
    loss = loss, #MeanSquaredLogarithmicError
  )

  print(model.summary())

  history = model.fit(x_train,y_train,batch_size=batch_size,epochs=epochs)
  loss_history = history.history['loss']

  plt.scatter(x=np.arange(1,epochs+1),y=loss_history)
  plt.show()

  return model
  

I then created an arbitrary function (just for test purposes) as:

def func(x1,x2,x3,x4):
  y=(x1**3+(x2*x3+2))/(x4+x2*x1)
  return y

and made a random dataset with this function:

def random_points_in_range(n,ranges):
  points = np.empty((n,len(ranges)))
  for i,element in enumerate(ranges):
    start=min(element[1],element[0])
    interval=abs(element[1]-element[0])
    rand_check = np.random.rand(n)
    randoms = ( rand_check*interval ) + start
    points[:,i] = randoms.T
  return points

def generate_random_dataset(n=200,ranges=[(0,10),(0,10),(0,10),(0,10)]):
  x_dataset = random_points_in_range(n,ranges)
  y_dataset = np.empty(n)
  for i in range(n):
    x1,x2,x3,x4 = x_dataset[i]
    y_dataset[i] = func(x1,x2,x3,x4)
  return x_dataset,y_dataset

I then train a model with these functions:

x_train,y_train = generate_random_dataset()
layer_sizes = [6,8,10,10,1]
activations = [LeakyReLU(),'relu','swish','relu','linear']
opt = Adam(learning_rate=0.001)
epochs = 3000
model=train_model(x_train,y_train,5,4,layer_sizes,activations,opt,epochs,loss='MeanSquaredError')

if you want to run the code these are things you need to import:

import numpy as np
from matplotlib import pyplot as plt
from sklearn.model_selection import train_test_split
import random
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers import LayerNormalization
from tensorflow.keras.optimizers import Adam
from tensorflow.keras import regularizers
0 Answers
Related