I have some large CSV files that are all numerical data, 3 columns each but each of varying lengths. There's a master CSV that has the ID and path of each CSV file. I am trying to make a generator to batch through these files without crashing my computer. I was trying to modify the generator seen here however this has not gone well. My code looks like this:
class DataGenerator(keras.utils.Sequence):
'Generates data for Keras'
def __init__(self, list_IDs, labels, batch_size=32, dim=(3,),
n_classes=9, shuffle=True):
'Initialization'
self.dim = dim
self.batch_size = batch_size
self.labels = labels
self.list_IDs = list_IDs
self.n_classes = n_classes
self.shuffle = shuffle
self.on_epoch_end()
def __len__(self):
'Denotes the number of batches per epoch'
return int(np.floor(len(self.list_IDs) / self.batch_size))
def __getitem__(self, index):
'Generate one batch of data'
# Generate indexes of the batch
indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
print(indexes)
print(self.list_IDs)
# Find list of IDs
list_IDs_temp = [self.list_IDs.loc[[k]].index[0] for k in indexes]
print(list_IDs_temp)
# Generate data
X, y = self.__data_generation(list_IDs_temp)
print('x',X.shape,'y',y)
return X, y
def on_epoch_end(self):
'Updates indexes after each epoch'
self.indexes = np.arange(len(self.list_IDs))
if self.shuffle == True:
np.random.shuffle(self.indexes)
def __data_generation(self, list_IDs_temp):
'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)
# Initialization
X = np.empty((self.batch_size, *self.dim,))
print(X.shape)
y = np.empty((self.batch_size))
# Generate data
for i, ID in enumerate(list_IDs_temp):
# Store sample
print('generate data')
#print(type(ID['file_path']))
print('print i', i, 'print id', ID)
#print(self.list_IDs.loc[[ID]]['features_path'][0])
t = pd.read_csv(self.list_IDs.loc[ID,'file_path'])
print(t.head())
t = t.to_numpy()
print(t.shape)
#print(X[i])
#print('X', X)
#print(X.shape)
#X[i] = t
X = t
# Store class
y = self.labels.loc[[ID]]
y = y.to_numpy()
print(y.shape)
return X, keras.utils.to_categorical(y, num_classes=self.n_classes)
followed by:
training_generator = DataGenerator(to_train, labels.loc[[0,1,2,3]], **params)
model = keras.Sequential([
layers.Dense(32, activation='relu',input_shape=[3]),
layers.Dense(9, activation='softmax')]
)
model.compile(
optimizer='adam',
loss='categorical_cross_entropy',
run_eagerly=True,
metrics=['accuracy']
)
model.fit(training_generator,
epochs=10,
use_multiprocessing=True,
workers=6)
The only result I get is an error: Unexpected result of train_function (Empty logs)
I know there has to be a reasonable way to import a CSV, train the model, then move onto the next CSV but I am struggling hard. Any help is appreciated.