The problem I have is, as soon as I want to train the model based on the data split (model.fit(...)) I get an error
InvalidArgumentError: indices [0] = 261429 is not in [0, 235061)
[[node recommender_net_3 / embedding_15 / embedding_lookup (defined at <ipython-input-46-e2a6cff5eb06>: 29)]] [Op: __ inference_train_function_9058]
I'm using the RetailRocket dataset. You can find it here https://www.kaggle.com/retailrocket/ecommerce-dataset.
You can see my implementation below
import pandas as pd
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
from tensorflow import keras
from tensorflow.keras import layers
from pathlib import Path
from google.colab import drive
drive.mount('/content/drive')
import os
for dirname, _, filenames in os.walk('/content/drive/My Drive/Dataset'):
for filename in filenames:
print(os.path.join(dirname, filename))
path = '/content/drive/My Drive/Dataset/'
# items = pd.concat([pd.read_csv(path+'item_properties_part2.csv'),
# pd.read_csv(path+'item_properties_part1.csv')])
# items.shape
events = pd.read_csv(path+'events.csv')
events.shape
df_event = pd.read_csv(path+ "events.csv")
print("file events.csv")
df_event.head()
df_event['code'] = df_event['event'].map({"view":1, "addtocart":2, "addtocart":3})
df_event.head()
# Visitor
visitor_ids = df_event["visitorid"].unique().tolist()
visitor2visitor_encoded = {x: i for i, x in enumerate(visitor_ids)}
visitorencoded2visitor = {i: x for i, x in enumerate(visitor_ids)}
# Items
items_ids = df_event["itemid"].unique().tolist()
item2item_encoded = {x: i for i, x in enumerate(items_ids)}
item_encoded2item = {i: x for i, x in enumerate(items_ids)}
df_event["visitor"] = df_event["visitorid"].map(visitor2visitor_encoded)
df_event["item"] = df_event["itemid"].map(item2item_encoded)
num_visitors = len(visitor2visitor_encoded)
num_items = len(item_encoded2item)
event = df_event["event"].value_counts()
#min_rating = min(df["rating"])
#max_rating = max(df["rating"])
print("Number of visitors: {}, Number of items: {}".format(num_visitors, num_items))
print("Number of views: {}, Number of addtocart: {}, Number of transactions: {}".format(event[0], event[1], event[2]))
## The Error
x = df[["visitorid", "itemid"]].values
# Normalize the targets between 0 and 1. Makes it easy to train.
#y = (df[col] - df[col].mean())/df[col].std()
df['z_score'] = (df['code'] - df['code'].mean())/df['code'].std()
y = df['z_score'].values
# Assuming training on 90% of the data and validating on 10%.
train_indices = int(0.9 * df.shape[0])
x_train, x_val, y_train, y_val = (
x[:train_indices],
x[train_indices:],
y[:train_indices],
y[train_indices:],
)
##
EMBEDDING_SIZE = 50
class RecommenderNet(keras.Model):
def __init__(self, num_visitors, num_items, embedding_size, **kwargs):
super(RecommenderNet, self).__init__(**kwargs)
self.num_visitors = num_visitors
self.num_items = num_items
self.embedding_size = embedding_size
self.visitor_embedding = layers.Embedding(
num_visitors,
embedding_size,
embeddings_initializer="he_normal",
embeddings_regularizer=keras.regularizers.l2(1e-6),
)
self.visitor_bias = layers.Embedding(num_visitors, 1)
self.item_embedding = layers.Embedding(
num_items,
embedding_size,
embeddings_initializer="he_normal",
embeddings_regularizer=keras.regularizers.l2(1e-6),
)
self.item_bias = layers.Embedding(num_items, 1)
def call(self, inputs):
visitor_vector = self.visitor_embedding(inputs[:, 0])
visitor_bias = self.visitor_bias(inputs[:, 0])
item_vector = self.item_embedding(inputs[:, 1])
item_bias = self.item_bias(inputs[:, 1])
dot_visitor_item = tf.tensordot(visitor_vector, item_vector, 2)
# Add all the components (including bias)
x = dot_visitor_item + visitor_bias + item_bias
# The sigmoid activation forces the rating to between 0 and 1
return tf.nn.sigmoid(x)
model = RecommenderNet(num_visitors, num_items, EMBEDDING_SIZE)
model.compile(
loss=tf.keras.losses.BinaryCrossentropy(), optimizer=keras.optimizers.Adam(lr=0.001)
)
## The InvalidArgumentError
history = model.fit(
x=x_train,
y=y_train,
batch_size=64,
epochs=5,
verbose=1,
validation_data=(x_val, y_val),)
But I found out when I change
# old code
x_train, x_val, y_train, y_val = (
x[:train_indices],
x[train_indices:],
y[:train_indices],
y[train_indices:],
)
to
# new code
X_temp, X_test, y_temp, y_test = train_test_split(df_event[['visitorid', 'itemid']],
df_event['code'],
test_size=0.2,
random_state=1)
X_train, X_val, y_train, y_val = train_test_split(X_temp, y_temp, test_size=0.2, random_state=1)
X_train.shape, X_val.shape, X_test.shape
With the new code, I don't get any errors and the implementation is error-free. Can someone tell me how to correctly write the "old" code so that it runs through? And what's the difference between the two? Thank you in advance!