If I generate a tf.dataset from the whole data and then split it to training, test, and val sets, the training works fine and loss goes down on both training and validation sets.
But, if I first split my data to training, test, and val and then generate a tf.dataset for each of them, the loss goes down on the training set but not on the validation set.
I do not know why there is a difference between these two scenarios. I was wondering if somebody could explain it to me. Thanks
def gen_dataset(molset, n_feat):
def generator():
for mol in molset:
X = mol.feat()
y = mol.value()
yield X, y
dataset = tf.data.Dataset.from_generator(generator,
output_types=((tf.float32, tf.float32),tf.float32),
output_shapes=((tf.TensorShape([None,n_feat]),
tf.TensorShape([None, n_feat])),
tf.TensorShape([])))
return dataset
#In scenario 1, tf.data.Dataset.from_generator is called on data and then the dataset is split to training, test, and val
dataset = gen_dataset(molset=data, n_feat)
training = dataset.take(training_size)
test = dataset.skip(test_size).take(test_size)
val = dataset.skip(valid_size).take(valid_size)
#In scenario 2, data is first split to test_set, train_set, val_set and then tf.data.Dataset.from_generator is called on each of them
training = gen_dataset(molset=train_set, n_feat)
test = gen_dataset(molset=test_set, n_feat)
val = gen_dataset(molset=val_set, n_feat)