Removing NaN from data frame increases error TensorFlow

Question

I have my code pasted below but the story was I have a panda data frame of historical option prices and one column is the result values I want to predict. This is a similar problem to this post.

Ive tested the code on a really small dataset with flawless data entries and it worked just fine. but for some reason when I added a line of code that dropped nan values the error increased by a factor of 10^16

*edit: I did find NaN values in the tiny.txt

Data cleansing:

dataSet = pd.read_csv("tiny.txt")
#dataSet = pd.read_csv("qqqMerged.csv")
#dataSet=dataSet.dropna()

for i in list(dataSet):   #some entries were empty strings or spaces so I got rid of them
    dataSet[i].replace(' ', np.nan, inplace=True)
    dataSet[i].replace('', np.nan, inplace=True)
    #dataSet[i].replace('  ', np.nan, inplace=True)

dataSet = dataSet.dropna()
#all of the popes columns are data I dont need to be used to train
dataSet.pop(' [QUOTE_READTIME]')
(...abridged the code to make post shorter) 
dataSet = dataSet.astype(float)

#instead of bid/ask I used the average between them to be the "label" I want to predict

avgPut = (dataSet[' [P_BID]']+dataSet[' [P_ASK]'])/2
avgCall = (dataSet[' [C_BID]']+dataSet[' [C_ASK]'])/2
dataSet[' [avgCall]'] = avgCall
dataSet[' [avgPut]'] = avgPut
dataSet.pop(' [P_BID]')
dataSet.pop(' [C_BID]')
dataSet.pop(' [P_ASK]')
dataSet.pop(' [C_ASK]')

callDataSet = dataSet.copy()
.... (I just removed a bunch of lines that got rid of cols related to put contracts)
callDataSet.pop(' [avgPut]')
callTrainFeature = callTrain.copy()
callTestFeature = callTest.copy()
callTrainLabel = callTrainFeature.pop(' [avgCall]')
callTestLabel = callTestFeature.pop(' [avgCall]')

Tensorflow code

def generateModel(numReluLayers128, numReluLayers64, numReluLayers32):
    model = tf.keras.models.Sequential()
    for i in range(numReluLayers128):
        model.add(tf.keras.layers.Dense(128, activation = tf.nn.relu))
    for i in range(numReluLayers64):
        model.add(tf.keras.layers.Dense(64, activation = tf.nn.relu))
    for i in range(numReluLayers32):
        model.add(tf.keras.layers.Dense(32, activation = tf.nn.relu))
    model.add(tf.keras.layers.Dense(units=1))
    #model.compile(optimizer='adam', loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), metrics=['accuracy'])
    model.compile(loss='mean_absolute_error', optimizer=tf.keras.optimizers.Adam(0.001))
    
    return model

def trainModelAndUse(model, trainX, trainY, testX, testY, epoch, batchSize):
    trainX = tf.keras.utils.normalize(trainX, axis=1)#normalises features
    history = model.fit(trainX, trainY, validation_split = .15, verbose=True, epochs=epoch)
    predictions = model.predict(testX).flatten()
    size = len(predictions)
    model.save("optionV2NeuralNet")
    rawError = predictions-testY
    plt.hist(rawError, bins=25)
    plt.xlabel("raw error")
    _ = plt.ylabel('Count')
    plt.show()
    print("average rawError "+str(np.average(rawError)/size))
    print("median rawError: "+str(np.median(rawError)))
    print("stdv rawError "+str(np.var(rawError)))

basicNN = generateModel(0,3,0)
print("was there nan? " + str(callTrainFeature.isnull().values.any()))
#basicNN = altGenModel(callTrainFeature, callTrainLabel)
trainModelAndUse(basicNN, callTrainFeature, callTrainLabel, callTestFeature, callTestLabel, 1, 10)

When I was running the code on a really small dataset with couple hundred data entries the largest raw error was by about ~40. Even on that same small dataset when I dropped NaN values the error skyrocketed to 10^16. I think it's where I tried to clean the data but I'm not sure where im getting it wrong

Example output:

average rawError 7767314.741863536
median rawError: 6447409663.3475
stdv rawError 1234237423961719.8

Anyone interested on downloading the tiny.txt: https://drive.google.com/file/d/1sqv_quRGGO8pA4rix9LtWYtMgRsQJjiL/view?usp=sharing

Removing NaN from data frame increases error TensorFlow

0 Answers