I am trying to create a custom loss function based on the minimizing the return difference between actual and predicted. I want to base my loss function on model return not on accuracy of any other metrics. I have the following code with data and the custom function I created. I, however, having an error that I don't know how to overcome it.
ValueError Traceback (most recent call last)
<ipython-input-45-774132da5942> in <module>
36
37 model.fit(train_x, train_y, epochs=100, batch_size=32, verbose=2,validation_split=0.2,class_weight=class_weight,use_multiprocessing=True,
---> 38 workers=6)
1 frames
/usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/func_graph.py in autograph_handler(*args, **kwargs)
1145 except Exception as e: # pylint:disable=broad-except
1146 if hasattr(e, "ag_error_metadata"):
-> 1147 raise e.ag_error_metadata.to_exception(e)
1148 else:
1149 raise
ValueError: in user code:
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 1021, in train_function *
return step_function(self, iterator)
File "<ipython-input-43-5b70d7b8daf6>", line 6, in ret *
Algo=ret*y_pred
ValueError: Dimensions must be equal, but are 998 and 2 for '{{node mul_1}} = Mul[T=DT_FLOAT](mul_1/x, sequential/dense/Softmax)' with input shapes: [998], [?,2].
This is my code. Any help would be appreciated.
!pip install yfinance
!pip install git+https://github.com/quantopian/pyfolio
import yfinance as yf
import pandas as pd
import numpy as np
from tensorflow.keras.utils import to_categorical
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from tensorflow import keras
from tensorflow.keras import layers
import pyfolio as pf
from pyfolio.utils import extract_rets_pos_txn_from_zipline
from pyfolio.plotting import (plot_perf_stats,
show_perf_stats,
plot_rolling_beta,
plot_rolling_returns,
plot_rolling_sharpe,
plot_drawdown_periods,
plot_drawdown_underwater)
from sklearn.model_selection import train_test_split
data = yf.download('GOOG', period='5y')
ret=data.Close.diff()
data['trgt']=np.where(ret>0,1,0)
data
# Data normalization function
# normalize data
def norm(x, window, method):
temp = pd.DataFrame()
for col in x.columns:
if ("trgt" in col) or ():
temp[col] = x[col]
x = x.drop([col], axis=1)
if method == 'scale':
r = x.rolling(window=window)
max_r = r.max()
min_r = r.min()
s = (x - min_r) / (max_r - min_r)
elif method == 'zscore':
r = x.rolling(window=window)
m = r.mean()
st = r.std(ddof=0)
s = (x - m) / st
s = pd.concat([s, temp], axis=1)
s.dropna(inplace=True)
#s = s.reset_index(drop=True)
return s
#To create the data sequence
timesteps=15
def sequences_creation(data_x, data_y, timesteps):
# Create sequences of T timesteps
data_x_seq, data_y_seq, data_symbol_seq= [], [], []
for i in range(data_x.shape[0] - (timesteps-1)):
data_x_seq.append(data_x.iloc[i:i+timesteps].values)
data_y_seq.append(data_y.loc[i + (timesteps-1), 'trgt'])
data_symbol_seq.append(data_y.loc[i + (timesteps-1), 'Date'])
data_x_seq, data_y_seq, data_symbol_seq = np.array(data_x_seq), np.array(data_y_seq).reshape(-1,1), np.array(data_symbol_seq).reshape(-1,1)
print(f'Data dimensions: {data_x_seq.shape}, {data_y_seq.shape}')
return data_x_seq, data_y_seq, pd.DataFrame(data_symbol_seq)
import keras.backend as K
def ret(y_true, y_pred):
ret=train_close.diff()
ret=np.array(ret.reset_index(drop=True))
Algo=ret*y_pred
Logic=ret*y_true
eval = K.abs(Algo-Logic)
eval = K.mean(eval, axis=-1)
return eval
#Create the data
data['trgt']=data.trgt.shift(-2)
data.dropna(inplace=True)
X=norm(data,10,'zscore')
X.dropna(inplace=True)
X=X[~X.isin([np.nan, np.inf, -np.inf]).any(1)]
data_x=X.copy()
data_x.drop(['trgt'], axis=1, inplace=True)
data_y = X['trgt'].copy()
# Data split note that I put shuffle false bc it is timeseries data
train_x, test_x, train_y, test_y = train_test_split(data_x, data_y,
test_size=0.2,shuffle=False)
train_close =(train_x.Close)
train_x.drop(['Close'], 1, inplace=True)
test_x.drop(['Close'], 1, inplace=True)
from sklearn.utils import class_weight
# Scaling by total/2 helps keep the loss to a similar magnitude.
# The sum of the weights of all examples stays the same.
neg=data_y.value_counts()[0]
pos=data_y.value_counts()[1]
total=neg+pos
weight_for_0 = (1 / neg) * (total / 2.0)
weight_for_1 = (1 / pos) * (total / 2.0)
class_weight = {0: weight_for_0, 1: weight_for_1}
test=test_x.reset_index(inplace=False)
test=test[timesteps-1:]
test_date=test.Date
test_date=pd.DataFrame(test_date)
test_date.set_index('Date',inplace=True)
train_y = pd.DataFrame(train_y)
train_y['Date'] = train_x.index
test_y = pd.DataFrame(test_y)
test_y['Date'] = test_x.index
test_y.reset_index(inplace=True, drop=True)
train_y.reset_index(inplace=True, drop=True)
test_y.reset_index(inplace=True, drop=True)
train_x, train_y, _ = sequences_creation(train_x, train_y, timesteps=timesteps)
test_x, test_y, test_symbol = sequences_creation(test_x, test_y, timesteps=timesteps)
train_y = to_categorical(train_y, num_classes=2)
test_y = to_categorical(test_y, num_classes=2)
model = Sequential()
model.add(LSTM(32,return_sequences=True, input_shape=(timesteps, train_x.shape[2])))
model.add(LSTM(62,return_sequences=True))
model.add(LSTM(120))
model.add(Dense(train_y.shape[1], activation='softmax'))
model.compile(loss='categorical_crossentropy', optimizer='Adam',metrics = [ret])
model.fit(train_x, train_y, epochs=100, batch_size=32, verbose=2,validation_split=0.2,class_weight=class_weight,use_multiprocessing=True,
workers=6)