I want to obtain the y_pred value of my sequential model using y_pred = model.predict(X_val) after doing train_test_split. My code raised ValueError: Input 0 of layer sequential is incompatible with the layer: expected axis -1 of input shape to have value *** but received input with shape (None, %%%%)
import pandas as pd
import numpy as np
import tensorflow as tf
from tensorflow import keras
from keras import layers
from sklearn.preprocessing import LabelEncoder, MinMaxScaler
from sklearn.feature_selection import SelectKBest, f_classif
from tensorflow.keras import backend as K
from sklearn.model_selection import train_test_split
from keras.callbacks import EarlyStopping
from keras.models import Sequential
from keras.layers.core import Dense, Dropout, Activation, Flatten
from sklearn.metrics import classification_report, confusion_matrix
shuffle_buffer = 500
batch_size = 2
Label encoding:
encoder = LabelEncoder()
df["subtype"] = encoder.fit_transform(df[["subtype"]])
Define X and y:
X = df.iloc[:,7:-2]
y = df[["subtype"]]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.25, random_state=1) # 0.25 x 0.8 = 0.2
Sequential model:
# Define Sequential model
def get_model():
model = keras.Sequential()
model.add(layers.Dense(10000))
model.add(keras.layers.Dropout(0.2))
model.add(layers.Dense(8000, activation='relu'))
model.add(keras.layers.Dropout(0.2))
model.add(layers.Dense(1, activation='softmax'))
model.compile(optimizer='adam',
loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
metrics=['accuracy'])
return model
model = get_model()
# Early stopping
es_callback = keras.callbacks.EarlyStopping(monitor='val_loss', patience=3)
# Feature selection
selector = SelectKBest(f_classif, k=10000) # Retrieve 10000 best features
Training:
selected_features_subtype = selector.fit_transform(X_train, y_train.values.ravel())
model.fit(selected_features_subtype, y_train, validation_split=0.5, batch_size=batch_size, callbacks=[es_callback])
y_pred = model.predict(X_val)
Traceback:
---------------------------------------------------------------------------
ValueError Traceback (most recent call last)
/tmp/ipykernel_16/582177622.py in <module>
1 selected_features_subtype = selector.fit_transform(X_train, y_train.values.ravel())
2 model.fit(selected_features_subtype, y_train, validation_split=0.5, batch_size=batch_size, callbacks=[es_callback])
----> 3 y_pred = model.predict(X_val)
/opt/conda/lib/python3.7/site-packages/keras/engine/training.py in predict(self, x, batch_size, verbose, steps, callbacks, max_queue_size, workers, use_multiprocessing)
1749 for step in data_handler.steps():
1750 callbacks.on_predict_batch_begin(step)
-> 1751 tmp_batch_outputs = self.predict_function(iterator)
1752 if data_handler.should_sync:
1753 context.async_wait()
/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in __call__(self, *args, **kwds)
883
884 with OptionalXlaContext(self._jit_compile):
--> 885 result = self._call(*args, **kwds)
886
887 new_tracing_count = self.experimental_get_tracing_count()
/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in _call(self, *args, **kwds)
922 # In this case we have not created variables on the first call. So we can
923 # run the first trace but we should fail if variables are created.
--> 924 results = self._stateful_fn(*args, **kwds)
925 if self._created_variables and not ALLOW_DYNAMIC_VARIABLE_CREATION:
926 raise ValueError("Creating variables on a non-first call to a function"
/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in __call__(self, *args, **kwargs)
3036 with self._lock:
3037 (graph_function,
-> 3038 filtered_flat_args) = self._maybe_define_function(args, kwargs)
3039 return graph_function._call_flat(
3040 filtered_flat_args, captured_inputs=graph_function.captured_inputs) # pylint: disable=protected-access
/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _maybe_define_function(self, args, kwargs)
3458 call_context_key in self._function_cache.missed):
3459 return self._define_function_with_shape_relaxation(
-> 3460 args, kwargs, flat_args, filtered_flat_args, cache_key_context)
3461
3462 self._function_cache.missed.add(call_context_key)
/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _define_function_with_shape_relaxation(self, args, kwargs, flat_args, filtered_flat_args, cache_key_context)
3380
3381 graph_function = self._create_graph_function(
-> 3382 args, kwargs, override_flat_arg_shapes=relaxed_arg_shapes)
3383 self._function_cache.arg_relaxed[rank_only_cache_key] = graph_function
3384
/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/function.py in _create_graph_function(self, args, kwargs, override_flat_arg_shapes)
3306 arg_names=arg_names,
3307 override_flat_arg_shapes=override_flat_arg_shapes,
-> 3308 capture_by_value=self._capture_by_value),
3309 self._function_attributes,
3310 function_spec=self.function_spec,
/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py in func_graph_from_py_func(name, python_func, args, kwargs, signature, func_graph, autograph, autograph_options, add_control_dependencies, arg_names, op_return_value, collections, capture_by_value, override_flat_arg_shapes, acd_record_initial_resource_uses)
1005 _, original_func = tf_decorator.unwrap(python_func)
1006
-> 1007 func_outputs = python_func(*func_args, **func_kwargs)
1008
1009 # invariant: `func_outputs` contains only Tensors, CompositeTensors,
/opt/conda/lib/python3.7/site-packages/tensorflow/python/eager/def_function.py in wrapped_fn(*args, **kwds)
666 # the function a weak reference to itself to avoid a reference cycle.
667 with OptionalXlaContext(compile_with_xla):
--> 668 out = weak_wrapped_fn().__wrapped__(*args, **kwds)
669 return out
670
/opt/conda/lib/python3.7/site-packages/tensorflow/python/framework/func_graph.py in wrapper(*args, **kwargs)
992 except Exception as e: # pylint:disable=broad-except
993 if hasattr(e, "ag_error_metadata"):
--> 994 raise e.ag_error_metadata.to_exception(e)
995 else:
996 raise
ValueError: in user code:
/opt/conda/lib/python3.7/site-packages/keras/engine/training.py:1586 predict_function *
return step_function(self, iterator)
/opt/conda/lib/python3.7/site-packages/keras/engine/training.py:1576 step_function **
outputs = model.distribute_strategy.run(run_step, args=(data,))
/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/distribute_lib.py:1286 run
return self._extended.call_for_each_replica(fn, args=args, kwargs=kwargs)
/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/distribute_lib.py:2849 call_for_each_replica
return self._call_for_each_replica(fn, args, kwargs)
/opt/conda/lib/python3.7/site-packages/tensorflow/python/distribute/distribute_lib.py:3632 _call_for_each_replica
return fn(*args, **kwargs)
/opt/conda/lib/python3.7/site-packages/keras/engine/training.py:1569 run_step **
outputs = model.predict_step(data)
/opt/conda/lib/python3.7/site-packages/keras/engine/training.py:1537 predict_step
return self(x, training=False)
/opt/conda/lib/python3.7/site-packages/keras/engine/base_layer.py:1020 __call__
input_spec.assert_input_compatibility(self.input_spec, inputs, self.name)
/opt/conda/lib/python3.7/site-packages/keras/engine/input_spec.py:254 assert_input_compatibility
' but received input with shape ' + display_shape(x.shape))
ValueError: Input 0 of layer sequential is incompatible with the layer: expected axis -1 of input shape to have value 10000 but received input with shape (None, 142513)
Sample data (as dict):
{'cg00000289': {'TCGA-5P-A9JW-01A': 0.619047033443068,
'TCGA-5P-A9JY-01A': 0.662345356057447,
'TCGA-5P-A9JZ-01A': 0.699464419990523,
'TCGA-5P-A9K0-01A': 0.581701228463189,
'TCGA-5P-A9K2-01A': 0.673198201701496,
'TCGA-5P-A9K3-01A': 0.626858745753386,
'TCGA-5P-A9K4-01A': 0.68070511240185,
'TCGA-5P-A9K6-01A': 0.736978843263676,
'TCGA-5P-A9K8-01A': 0.520786654021559,
'TCGA-5P-A9K9-01A': 0.630167782543268,
'TCGA-5P-A9KA-01A': 0.626926559177059,
'TCGA-5P-A9KC-01A': 0.630977089669812,
'TCGA-5P-A9KE-01A': 0.67563913188817,
'TCGA-5P-A9KF-01A': 0.645760654461395,
'TCGA-5P-A9KH-01A': 0.743286554209644,
'TCGA-6D-AA2E-01A': 0.664370540359405,
'TCGA-A3-3357-01A': 0.604326417072586,
'TCGA-A3-3358-01A': 0.458291671200214,
'TCGA-A3-3367-01A': 0.643363591881443,
'TCGA-A3-3370-01A': 0.75808536817831},
'cg00000292': {'TCGA-5P-A9JW-01A': 0.833751448211989,
'TCGA-5P-A9JY-01A': 0.761479643484317,
'TCGA-5P-A9JZ-01A': 0.555219387831784,
'TCGA-5P-A9K0-01A': 0.911434986725676,
'TCGA-5P-A9K2-01A': 0.592839496707225,
'TCGA-5P-A9K3-01A': 0.728724740061591,
'TCGA-5P-A9K4-01A': 0.871647081931081,
'TCGA-5P-A9K6-01A': 0.687137365325043,
'TCGA-5P-A9K8-01A': 0.377068349756215,
'TCGA-5P-A9K9-01A': 0.885089826740071,
'TCGA-5P-A9KA-01A': 0.678749255227915,
'TCGA-5P-A9KC-01A': 0.82812139328519,
'TCGA-5P-A9KE-01A': 0.864590733749797,
'TCGA-5P-A9KF-01A': 0.858070865799283,
'TCGA-5P-A9KH-01A': 0.914435928566657,
'TCGA-6D-AA2E-01A': 0.502154665104261,
'TCGA-A3-3357-01A': 0.662047197870235,
'TCGA-A3-3358-01A': 0.837059251611538,
'TCGA-A3-3367-01A': 0.519939858249351,
'TCGA-A3-3370-01A': 0.515743234863198},
'cg00000321': {'TCGA-5P-A9JW-01A': 0.489674207394165,
'TCGA-5P-A9JY-01A': 0.558997284357574,
'TCGA-5P-A9JZ-01A': 0.169654991100549,
'TCGA-5P-A9K0-01A': 0.524017780921585,
'TCGA-5P-A9K2-01A': 0.613973121455874,
'TCGA-5P-A9K3-01A': 0.695670625292722,
'TCGA-5P-A9K4-01A': 0.54705053331032,
'TCGA-5P-A9K6-01A': 0.430048300391044,
'TCGA-5P-A9K8-01A': 0.198107812192402,
'TCGA-5P-A9K9-01A': 0.688004412660976,
'TCGA-5P-A9KA-01A': 0.463728628068311,
'TCGA-5P-A9KC-01A': 0.195610858899826,
'TCGA-5P-A9KE-01A': 0.550900635535007,
'TCGA-5P-A9KF-01A': 0.183061803083559,
'TCGA-5P-A9KH-01A': 0.0687041391027568,
'TCGA-6D-AA2E-01A': 0.570614767869701,
'TCGA-A3-3357-01A': 0.680216582417148,
'TCGA-A3-3358-01A': 0.288408070866453,
'TCGA-A3-3367-01A': 0.68957300320214,
'TCGA-A3-3370-01A': 0.432450706527388},
'cg00000363': {'TCGA-5P-A9JW-01A': 0.276273359465442,
'TCGA-5P-A9JY-01A': 0.25703867804956,
'TCGA-5P-A9JZ-01A': 0.0981746197111535,
'TCGA-5P-A9K0-01A': 0.569434380074143,
'TCGA-5P-A9K2-01A': 0.229164840583894,
'TCGA-5P-A9K3-01A': 0.819181728250669,
'TCGA-5P-A9K4-01A': 0.385400157144987,
'TCGA-5P-A9K6-01A': 0.121114850921845,
'TCGA-5P-A9K8-01A': 0.0913002964111161,
'TCGA-5P-A9K9-01A': 0.157709640291589,
'TCGA-5P-A9KA-01A': 0.182786461024344,
'TCGA-5P-A9KC-01A': 0.413992919815649,
'TCGA-5P-A9KE-01A': 0.461011690059099,
'TCGA-5P-A9KF-01A': 0.206300754004666,
'TCGA-5P-A9KH-01A': 0.105913974360403,
'TCGA-6D-AA2E-01A': 0.414195666341229,
'TCGA-A3-3357-01A': 0.236700230821645,
'TCGA-A3-3358-01A': 0.26841943012007,
'TCGA-A3-3367-01A': 0.324092383012077,
'TCGA-A3-3370-01A': 0.20675761437022},
'cg00000622': {'TCGA-5P-A9JW-01A': 0.0127307580597565,
'TCGA-5P-A9JY-01A': 0.0120665526567922,
'TCGA-5P-A9JZ-01A': 0.0129414663486043,
'TCGA-5P-A9K0-01A': 0.0151053469374601,
'TCGA-5P-A9K2-01A': 0.0147191458229104,
'TCGA-5P-A9K3-01A': 0.0128586680833482,
'TCGA-5P-A9K4-01A': 0.0131839246367822,
'TCGA-5P-A9K6-01A': 0.0145337257462313,
'TCGA-5P-A9K8-01A': 0.0131548295351121,
'TCGA-5P-A9K9-01A': 0.0164371867221858,
'TCGA-5P-A9KA-01A': 0.0153209540816947,
'TCGA-5P-A9KC-01A': 0.0146341900882511,
'TCGA-5P-A9KE-01A': 0.0138002584809048,
'TCGA-5P-A9KF-01A': 0.012958575912875,
'TCGA-5P-A9KH-01A': 0.0142346121115625,
'TCGA-6D-AA2E-01A': 0.0139666701044385,
'TCGA-A3-3357-01A': 0.0082183731354049,
'TCGA-A3-3358-01A': 0.0143527756424356,
'TCGA-A3-3367-01A': 0.0100636145864037,
'TCGA-A3-3370-01A': 0.0108528842825329},
'type': {'TCGA-5P-A9JW-01A': 'tumor',
'TCGA-5P-A9JY-01A': 'tumor',
'TCGA-5P-A9JZ-01A': 'tumor',
'TCGA-5P-A9K0-01A': 'tumor',
'TCGA-5P-A9K2-01A': 'tumor',
'TCGA-5P-A9K3-01A': 'tumor',
'TCGA-5P-A9K4-01A': 'tumor',
'TCGA-5P-A9K6-01A': 'tumor',
'TCGA-5P-A9K8-01A': 'tumor',
'TCGA-5P-A9K9-01A': 'tumor',
'TCGA-5P-A9KA-01A': 'tumor',
'TCGA-5P-A9KC-01A': 'tumor',
'TCGA-5P-A9KE-01A': 'tumor',
'TCGA-5P-A9KF-01A': 'tumor',
'TCGA-5P-A9KH-01A': 'tumor',
'TCGA-6D-AA2E-01A': 'tumor',
'TCGA-A3-3357-01A': 'tumor',
'TCGA-A3-3358-01A': 'tumor',
'TCGA-A3-3367-01A': 'tumor',
'TCGA-A3-3370-01A': 'tumor'},
'subtype': {'TCGA-5P-A9JW-01A': 'KIRP',
'TCGA-5P-A9JY-01A': 'KIRP',
'TCGA-5P-A9JZ-01A': 'KIRP',
'TCGA-5P-A9K0-01A': 'KIRP',
'TCGA-5P-A9K2-01A': 'KIRP',
'TCGA-5P-A9K3-01A': 'KIRP',
'TCGA-5P-A9K4-01A': 'KIRP',
'TCGA-5P-A9K6-01A': 'KIRP',
'TCGA-5P-A9K8-01A': 'KIRP',
'TCGA-5P-A9K9-01A': 'KIRP',
'TCGA-5P-A9KA-01A': 'KIRP',
'TCGA-5P-A9KC-01A': 'KIRP',
'TCGA-5P-A9KE-01A': 'KIRP',
'TCGA-5P-A9KF-01A': 'KIRP',
'TCGA-5P-A9KH-01A': 'KIRP',
'TCGA-6D-AA2E-01A': 'KIRC',
'TCGA-A3-3357-01A': 'KIRC',
'TCGA-A3-3358-01A': 'KIRC',
'TCGA-A3-3367-01A': 'KIRC',
'TCGA-A3-3370-01A': 'KIRC'}}