I am trying to run a Resnet model through Skorch for classification which I found in a research paper. I am still learning the ways of Torch and Skorch, and I'm unable to find what to fix to get this to work.
ResNet class:
class ResNet(nn.Module):
def __init__(
self,
*,
d_numerical: int,
categories: ty.Optional[ty.List[int]],
d_embedding: int,
d: int,
d_hidden_factor: float,
n_layers: int,
activation: str,
normalization: str,
hidden_dropout: float,
residual_dropout: float,
d_out: int,
regression: bool,
categorical_indicator
) -> None:
super().__init__()
#categories = None #TODO
def make_normalization():
return {'batchnorm': nn.BatchNorm1d, 'layernorm': nn.LayerNorm}[
normalization[0]
](d)
self.categorical_indicator = categorical_indicator #Added
self.regression = regression
self.main_activation = deep.get_activation_fn(activation)
self.last_activation = deep.get_nonglu_activation_fn(activation)
self.residual_dropout = residual_dropout
self.hidden_dropout = hidden_dropout
d_in = d_numerical
d_hidden = int(d * d_hidden_factor)
if categories is not None:
d_in += len(categories) * d_embedding
category_offsets = torch.tensor([0] + categories[:-1]).cumsum(0)
self.register_buffer('category_offsets', category_offsets)
self.category_embeddings = nn.Embedding(int(sum(categories)), d_embedding)
nn.init.kaiming_uniform_(self.category_embeddings.weight, a=math.sqrt(5))
print(f'{self.category_embeddings.weight.shape}')
self.first_layer = nn.Linear(d_in, d) # 1, 256
self.layers = nn.ModuleList(
[
nn.ModuleDict(
{
'norm': make_normalization(),
'linear0': nn.Linear(
d, d_hidden * (2 if activation.endswith('glu') else 1)
),
'linear1': nn.Linear(d_hidden, d),
}
)
for _ in range(n_layers)
]
)
self.last_normalization = make_normalization()
self.head = nn.Linear(d, d_out) # 256, 1
def forward(self, x) -> Tensor:
if not self.categorical_indicator is None:
x_num = x[:, ~self.categorical_indicator].float()
x_cat = x[:, self.categorical_indicator].long() #TODO
else:
x_num = x
x_cat = None
x = []
if x_num is not None:
x.append(x_num)
if x_cat is not None:
x.append(
self.category_embeddings(x_cat + self.category_offsets[None]).view(
x_cat.size(0), -1
)
)
x = torch.cat(x, dim=-1)
x = self.first_layer(x)
for layer in self.layers:
layer = ty.cast(ty.Dict[str, nn.Module], layer)
z = x
z = layer['norm'](z)
z = layer['linear0'](z)
z = self.main_activation(z)
if self.hidden_dropout:
z = F.dropout(z, self.hidden_dropout, self.training)
z = layer['linear1'](z)
if self.residual_dropout:
z = F.dropout(z, self.residual_dropout, self.training)
x = x + z
x = self.last_normalization(x)
x = self.last_activation(x)
x = self.head(x)
if not self.regression:
x = x.squeeze(-1)
return x
class InputShapeSetterResnet(skorch.callbacks.Callback):
def __init__(self, regression=False, batch_size=None,
categorical_indicator=None):
self.categorical_indicator = categorical_indicator
self.regression = regression
self.batch_size = batch_size
def on_train_begin(self, net, X, y):
print("categorical_indicator", self.categorical_indicator)
if self.categorical_indicator is None:
d_numerical = X.shape[1]
categories = None
else:
d_numerical = X.shape[1] - sum(self.categorical_indicator)
# categories = list((X[:, self.categorical_indicator].max(0) + 1).astype(int))
categories = [sum(self.categorical_indicator)]
net.set_params(module__d_numerical=d_numerical,
module__categories=categories, #FIXME #lib.get_categories(X_cat),
module__d_out=2 if self.regression == False else 1) #FIXME#D.info['n_classes'] if D.is_multiclass else 1,
print("Numerical features: {}".format(d_numerical))
print("Categories {}".format(categories))
Skorch Wrapper:
def create_resnet_skorch(id, wandb_run=None, use_checkpoints=True,
categorical_indicator=None, **kwargs):
print(kwargs)
if "verbose" not in kwargs:
verbose = 0
else:
verbose = kwargs.pop("verbose")
if "lr_scheduler" not in kwargs:
lr_scheduler = False
else:
lr_scheduler = kwargs.pop("lr_scheduler")
if "es_patience" not in kwargs.keys():
es_patience = 40
else:
es_patience = kwargs.pop('es_patience')
if "lr_patience" not in kwargs.keys():
lr_patience = 30
else:
lr_patience = kwargs.pop('lr_patience')
optimizer = kwargs.pop('optimizer')
if optimizer == "adam":
optimizer = Adam
elif optimizer == "adamw":
optimizer = AdamW
elif optimizer == "sgd":
optimizer = SGD
device = kwargs.pop('device')
if device == "cuda": # ! only for CPU training, is cuda by default
device = "cpu"
batch_size = kwargs.pop('batch_size')
callbacks = [InputShapeSetterResnet(categorical_indicator=categorical_indicator),
EarlyStopping(monitor="valid_loss",
patience=es_patience)]
callbacks.append(EpochScoring(scoring='accuracy', name='train_accuracy', on_train=True))
if lr_scheduler:
callbacks.append(LRScheduler(policy=ReduceLROnPlateau, patience=lr_patience, min_lr=2e-5,
factor=0.2)) # FIXME make customizable
if use_checkpoints:
callbacks.append(Checkpoint(dirname="skorch_cp", f_params=r"params_{}.pt".format(id), f_optimizer=None,
f_criterion=None))
if not wandb_run is None:
callbacks.append(WandbLogger(wandb_run, save_model=False))
callbacks.append(LearningRateLogger())
if not categorical_indicator is None:
categorical_indicator = torch.BoolTensor(categorical_indicator)
mlp_skorch = NeuralNetClassifier(
ResNet,
# Shuffle training data on each epoch
criterion=torch.nn.CrossEntropyLoss,
optimizer=optimizer,
batch_size=max(batch_size, 1), # if batch size is float, it will be reset during fit
iterator_train__shuffle=True,
module__d_numerical=1, # will be change when fitted
module__categories=None, # will be change when fitted
module__d_out=1, # idem
module__regression=False,
module__categorical_indicator=categorical_indicator,
verbose=verbose,
callbacks=callbacks,
**kwargs
)
return mlp_skorch
Skorch Model:
<class 'skorch.classifier.NeuralNetClassifier'>[uninitialized](
module=<class 'tabular.bin.resnet.ResNet'>,
module__activation=reglu,
module__categorical_indicator=tensor([False, True, False, False, False, False, False, False]),
module__categories=None,
module__d=256,
module__d_embedding=128,
module__d_hidden_factor=2,
module__d_numerical=1,
module__d_out=1,
module__hidden_dropout=0.2,
module__n_layers=8,
module__normalization=['batchnorm'],
module__regression=False,
module__residual_dropout=0.2,
)
I have 8 columns in X for training, 1 of which is a categorical column which is to be embedding through an embedding layer in the NN. From what I've found so far, that is the root of this error since it's coming across this categorical class in execution. But in the forward method, it's supposed to have an embedding layer for the same. Any idea what changes I might need to make for the same?
Error stack:
Traceback (most recent call last):
File "/test.py", line 639, in <module>
model.fit(X_train, y_train)
File "/anaconda3/lib/python3.9/site-packages/skorch/classifier.py", line 142, in fit
return super(NeuralNetClassifier, self).fit(X, y, **fit_params)
File "/anaconda3/lib/python3.9/site-packages/skorch/net.py", line 917, in fit
self.partial_fit(X, y, **fit_params)
File "/anaconda3/lib/python3.9/site-packages/skorch/net.py", line 876, in partial_fit
self.fit_loop(X, y, **fit_params)
File "/anaconda3/lib/python3.9/site-packages/skorch/net.py", line 789, in fit_loop
self.run_single_epoch(dataset_train, training=True, prefix="train",
File "/anaconda3/lib/python3.9/site-packages/skorch/net.py", line 822, in run_single_epoch
for data in self.get_iterator(dataset, training=training):
File "/anaconda3/lib/python3.9/site-packages/torch/utils/data/dataloader.py", line 521, in __next__
data = self._next_data()
File "/anaconda3/lib/python3.9/site-packages/torch/utils/data/dataloader.py", line 561, in _next_data
data = self._dataset_fetcher.fetch(index) # may raise StopIteration
File "/anaconda3/lib/python3.9/site-packages/torch/utils/data/_utils/fetch.py", line 52, in fetch
return self.collate_fn(data)
File "/anaconda3/lib/python3.9/site-packages/torch/utils/data/_utils/collate.py", line 84, in default_collate
return [default_collate(samples) for samples in transposed]
File "/anaconda3/lib/python3.9/site-packages/torch/utils/data/_utils/collate.py", line 84, in <listcomp>
return [default_collate(samples) for samples in transposed]
File "/anaconda3/lib/python3.9/site-packages/torch/utils/data/_utils/collate.py", line 74, in default_collate
return {key: default_collate([d[key] for d in batch]) for key in elem}
File "/anaconda3/lib/python3.9/site-packages/torch/utils/data/_utils/collate.py", line 74, in <dictcomp>
return {key: default_collate([d[key] for d in batch]) for key in elem}
File "/anaconda3/lib/python3.9/site-packages/torch/utils/data/_utils/collate.py", line 86, in default_collate
raise TypeError(default_collate_err_msg_format.format(elem_type))
TypeError: default_collate: batch must contain tensors, numpy arrays, numbers, dicts or lists; found <class 'pandas.core.arrays.categorical.Categorical'>