I am trying to implement an autoencoder using a datagenerator, the autoencoder works fine if I implement it directly with X_train, but when I try to use the data generator always returns that error.
#!/usr/bin/env python
# coding: utf-8
import zipfile
import pandas as pd
import tensorflow as tf
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import Dense, Conv2D, Flatten, Activation, MaxPooling2D, Reshape, Input, Dropout
from tensorflow.keras.layers import BatchNormalization, Lambda, Conv2DTranspose, Add
from tensorflow.keras import regularizers
from tensorflow.keras import initializers
from tensorflow.keras import constraints
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.utils import Sequence
import random
import numpy as np
from sklearn.preprocessing import MinMaxScaler
from matplotlib import pyplot as plt
# descargamos el conjunto de datos
train_zip_path = tf.keras.utils.get_file(
fname='train_femto.zip',
origin='https://hdvirtual.us.es/discovirt/index.php/s/z75ekAqY3tt3aKy/download',
cache_subdir='datasets', extract=False,
)
WINDOW_SIZE = 500
HEADERS = ['Hour', 'Minute', 'Second', 'Microsecond', 'H-acc', 'V-acc']
FEATURES = ['H-acc', 'V-acc']
TEST_RULS = {
'Bearing1_3': 5730,
'Bearing1_4': 339,
'Bearing1_5': 1610,
'Bearing1_6': 1460,
'Bearing1_7': 7570,
'Bearing2_3': 7530,
'Bearing2_4': 1390,
'Bearing2_5': 3090,
'Bearing2_6': 1290,
'Bearing2_7': 580,
'Bearing3_3': 820
}
def read_femto_file(z, file_path, bearing):
with z.open(file_path) as f:
X_aux = pd.read_csv(f, names=HEADERS, delimiter=",")
X_aux['Bearing'] = bearing.split('/')[-1][-3:]
del X_aux['Hour']
del X_aux['Minute']
del X_aux['Second']
del X_aux['Microsecond']
return X_aux
def read_femto_dataset(file_path, RULS=None):
datasets = []
with zipfile.ZipFile(file_path) as z:
files = z.namelist()
dirs = sorted(set(['/'.join(f.split('/')[:-1]) for f in files if len(f.split('/')) > 2]))
ds = []
for bearing in dirs:
bearing_name = bearing.split('/')[-1]
print("Reading", bearing_name)
bearing_files = sorted([f for f in files if bearing in f and 'acc' in f])
for i, bearing_file in enumerate(bearing_files):
ds.append(read_femto_file(z, bearing_file, bearing))
X = pd.concat(ds, axis=0)
X = X.reset_index(drop=True)
# compute RUL
X['RUL'] = (X.index / 256).astype('int')[::-1].values
if RULS is not None:
X['RUL'] += RULS[bearing.split('/')[-1]]
X.RUL = X.RUL.astype('int32')
datasets.append(X)
X = pd.concat(datasets, axis=0)
X['H-acc'] = X['H-acc'].astype('float32')
X['V-acc'] = X['V-acc'].astype('float32') #errata, H-acc en vez de V-acc
X['RUL'] = X['RUL'].astype('int32')
return X
# leemos el conjunto de entrenamiento
X_train = read_femto_dataset(train_zip_path)
# truncamos el RUL y lo normalzamos entre 100 y 0
X_train['RUL'] = X_train.RUL.clip(0, 10000) / 100
# dividimos entre entrenamiento y validación
X_val = X_train[X_train.Bearing.isin(['1_1', '2_1', '3_1'])]
X_train = X_train[X_train.Bearing.isin(['1_2', '2_2', '3_2'])]
# normalizamos los sensores
min_max = {}
for feature in FEATURES:
# calculamos los parámetros de la normalización con el train
min_max[feature] = {}
min_max[feature]['min'] = X_train[feature].min()
min_max[feature]['max'] = X_train[feature].max()
# normalizamos ambos datasets
X_train[feature] = ((X_train[feature] - min_max[feature]['min']) /
(min_max[feature]['max'] - min_max[feature]['min']))
X_val[feature] = ((X_val[feature] - min_max[feature]['min']) /
(min_max[feature]['max'] - min_max[feature]['min']))
print("Feature %s: train(%f, %f), val(%f, %f)" % (feature, X_train[feature].min(),
X_train[feature].max(), X_val[feature].min(),
X_val[feature].max()))
# generador de datos
class DataGenerator(Sequence):
def __init__(self, X, attributes, window_size=10, batch_size=32,
epoch_len_reducer=100, add_extra_channel=False,
return_label=True, y_key='Y', unit_key='id'):
self.batch_size = batch_size
self.return_label = return_label
self.window_size = window_size
self.attributes = attributes
self.epoch_len_reducer = epoch_len_reducer
self._X = {}
self._Y = {}
self._ids = X[unit_key].unique()
self.add_extra_channel = add_extra_channel
for _id in self._ids:
self._X[_id] = X.loc[(X[unit_key]==_id), self.attributes].values
self._Y[_id] = X.loc[(X[unit_key]==_id), y_key].values
self.__len = int((X.groupby(unit_key).size() - self.window_size).sum() /
self.batch_size)
del X
def __len__(self):
return int(self.__len / self.epoch_len_reducer)
def __getitem__(self, index):
X = self._X
_X = []
_y = []
for _ in range(self.batch_size):
sid = random.choice(self._ids)
unit = self._X[sid]
nrows = unit.shape[0]
cut = random.randint(0, nrows - self.window_size)
s = unit[cut: cut + self.window_size].T
y =self._Y[sid][cut + self.window_size-1]
_X.append(s)
_y.append(y)
_X = np.array(_X)
if self.add_extra_channel:
_X = _X.reshape(_X.shape + (1,))
if self.return_label:
return _X, np.array(_y).reshape((self.batch_size, 1))
else:
return _X, _X
def on_epoch_end(self):
pass
# generators
train_gen = DataGenerator(X_train, attributes=FEATURES, window_size=WINDOW_SIZE, batch_size=256,
add_extra_channel=True, return_label=True, y_key='RUL', unit_key='Bearing')
val_gen = DataGenerator(X_val, attributes=FEATURES, window_size=WINDOW_SIZE, batch_size=256,
add_extra_channel=True, return_label=True, y_key='RUL',
unit_key='Bearing', epoch_len_reducer=1000)
#Autoencoder
class AutoEncoder(Model):
def __init__(self):
super(AutoEncoder, self).__init__()
self.encoder = tf.keras.Sequential([
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(32, activation='relu'),
tf.keras.layers.Dense(16, activation='relu'),
tf.keras.layers.Dense(8, activation='relu')])
self.decoder = tf.keras.Sequential([
tf.keras.layers.Dense(16, activation='relu'),
tf.keras.layers.Dense(32, activation='relu'),
tf.keras.layers.Dense(64, activation='relu'),
tf.keras.layers.Dense(2, activation='sigmoid')
])
def call(self, x):
encoded = self.encoder(x)
decoded = self.decoder(encoded)
return decoded
definition of the autoencoder
model = AutoEncoder()
early_stopping = tf.keras.callbacks.EarlyStopping(monitor = 'val_loss', patience = 2, mode='min')
model.compile(optimizer = 'adam', loss='mae')
history = model.fit_generator(train_gen,
validation_data = val_gen,
epochs = 10)
After this is where I get the error,I have tried using different parameters but the result is always the same:
Epoch 1/10
/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:3: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.
This is separate from the ipykernel package so we can avoid doing imports until
---------------------------------------------------------------------------
InvalidArgumentError Traceback (most recent call last)
<ipython-input-12-ee4502a47627> in <module>()
1 history = model.fit_generator(train_gen,
2 validation_data = val_gen,
----> 3 epochs = 10)
2 frames
/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/execute.py in quick_execute(op_name, num_outputs, inputs, attrs, ctx, name)
53 ctx.ensure_initialized()
54 tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,
---> 55 inputs, attrs, num_outputs)
56 except core._NotOkStatusException as e:
57 if name is not None:
InvalidArgumentError: Graph execution error:
Detected at node 'mean_absolute_error/sub' defined at (most recent call last):
File "/usr/lib/python3.7/runpy.py", line 193, in _run_module_as_main
"__main__", mod_spec)
File "/usr/lib/python3.7/runpy.py", line 85, in _run_code
exec(code, run_globals)
File "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py", line 16, in <module>
app.launch_new_instance()
File "/usr/local/lib/python3.7/dist-packages/traitlets/config/application.py", line 846, in launch_instance
app.start()
File "/usr/local/lib/python3.7/dist-packages/ipykernel/kernelapp.py", line 499, in start
self.io_loop.start()
File "/usr/local/lib/python3.7/dist-packages/tornado/platform/asyncio.py", line 132, in start
self.asyncio_loop.run_forever()
File "/usr/lib/python3.7/asyncio/base_events.py", line 541, in run_forever
self._run_once()
File "/usr/lib/python3.7/asyncio/base_events.py", line 1786, in _run_once
handle._run()
File "/usr/lib/python3.7/asyncio/events.py", line 88, in _run
self._context.run(self._callback, *self._args)
File "/usr/local/lib/python3.7/dist-packages/tornado/platform/asyncio.py", line 122, in _handle_events
handler_func(fileobj, events)
File "/usr/local/lib/python3.7/dist-packages/tornado/stack_context.py", line 300, in null_wrapper
return fn(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/zmq/eventloop/zmqstream.py", line 452, in _handle_events
self._handle_recv()
File "/usr/local/lib/python3.7/dist-packages/zmq/eventloop/zmqstream.py", line 481, in _handle_recv
self._run_callback(callback, msg)
File "/usr/local/lib/python3.7/dist-packages/zmq/eventloop/zmqstream.py", line 431, in _run_callback
callback(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/tornado/stack_context.py", line 300, in null_wrapper
return fn(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/ipykernel/kernelbase.py", line 283, in dispatcher
return self.dispatch_shell(stream, msg)
File "/usr/local/lib/python3.7/dist-packages/ipykernel/kernelbase.py", line 233, in dispatch_shell
handler(stream, idents, msg)
File "/usr/local/lib/python3.7/dist-packages/ipykernel/kernelbase.py", line 399, in execute_request
user_expressions, allow_stdin)
File "/usr/local/lib/python3.7/dist-packages/ipykernel/ipkernel.py", line 208, in do_execute
res = shell.run_cell(code, store_history=store_history, silent=silent)
File "/usr/local/lib/python3.7/dist-packages/ipykernel/zmqshell.py", line 537, in run_cell
return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py", line 2718, in run_cell
interactivity=interactivity, compiler=compiler, result=result)
File "/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py", line 2822, in run_ast_nodes
if self.run_code(code, result):
File "/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py", line 2882, in run_code
exec(code_obj, self.user_global_ns, self.user_ns)
File "<ipython-input-7-342600d44df8>", line 3, in <module>
epochs = 10)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 2223, in fit_generator
initial_epoch=initial_epoch)
File "/usr/local/lib/python3.7/dist-packages/keras/utils/traceback_utils.py", line 64, in error_handler
return fn(*args, **kwargs)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 1384, in fit
tmp_logs = self.train_function(iterator)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 1021, in train_function
return step_function(self, iterator)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 1010, in step_function
outputs = model.distribute_strategy.run(run_step, args=(data,))
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 1000, in run_step
outputs = model.train_step(data)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 860, in train_step
loss = self.compute_loss(x, y, y_pred, sample_weight)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/training.py", line 919, in compute_loss
y, y_pred, sample_weight, regularization_losses=self.losses)
File "/usr/local/lib/python3.7/dist-packages/keras/engine/compile_utils.py", line 201, in __call__
loss_value = loss_obj(y_t, y_p, sample_weight=sw)
File "/usr/local/lib/python3.7/dist-packages/keras/losses.py", line 141, in __call__
losses = call_fn(y_true, y_pred)
File "/usr/local/lib/python3.7/dist-packages/keras/losses.py", line 245, in call
return ag_fn(y_true, y_pred, **self._fn_kwargs)
File "/usr/local/lib/python3.7/dist-packages/keras/losses.py", line 1457, in mean_absolute_error
return backend.mean(tf.abs(y_pred - y_true), axis=-1)
Node: 'mean_absolute_error/sub'
required broadcastable shapes
[[{{node mean_absolute_error/sub}}]] [Op:__inference_train_function_1827]