I know there is an issue with the inputs to the network. But cannot figure out where exactly. As soon as I start training the code I get this error:
Epoch 0
0%| | 0/198 [00:00<?, ?it/s]
Traceback (most recent call last):
File "train.py", line 86, in <module>
dt.engine.train(model1, optims, [crit_segm, crit_depth], trainloader, loss_coeffs)
File "/project/xfu/aamir/multi-task-refinenet/DenseTorch/densetorch/engine/trainval.py", line 76, in train
outputs = model(input)
File "/project/xfu/aamir/anaconda3/envs/MTLRefineNet/lib/python3.6/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/project/xfu/aamir/anaconda3/envs/MTLRefineNet/lib/python3.6/site-packages/torch/nn/parallel/data_parallel.py", line 166, in forward
return self.module(*inputs[0], **kwargs[0])
File "/project/xfu/aamir/anaconda3/envs/MTLRefineNet/lib/python3.6/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/project/xfu/aamir/anaconda3/envs/MTLRefineNet/lib/python3.6/site-packages/torch/nn/modules/container.py", line 141, in forward
input = module(input)
File "/project/xfu/aamir/anaconda3/envs/MTLRefineNet/lib/python3.6/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/project/xfu/aamir/multi-task-refinenet/DenseTorch/densetorch/nn/decoders.py", line 363, in forward
x = self.layer1(x)
File "/project/xfu/aamir/anaconda3/envs/MTLRefineNet/lib/python3.6/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/project/xfu/aamir/anaconda3/envs/MTLRefineNet/lib/python3.6/site-packages/torch/nn/modules/container.py", line 141, in forward
input = module(input)
File "/project/xfu/aamir/anaconda3/envs/MTLRefineNet/lib/python3.6/site-packages/torch/nn/modules/module.py", line 1102, in _call_impl
return forward_call(*input, **kwargs)
File "/project/xfu/aamir/anaconda3/envs/MTLRefineNet/lib/python3.6/site-packages/torch/nn/modules/conv.py", line 446, in forward
return self._conv_forward(input, self.weight, self.bias)
File "/project/xfu/aamir/anaconda3/envs/MTLRefineNet/lib/python3.6/site-packages/torch/nn/modules/conv.py", line 443, in _conv_forward
self.padding, self.dilation, self.groups)
TypeError: conv2d() received an invalid combination of arguments - got (list, Parameter, NoneType, tuple, tuple, tuple, int), but expected one of:
* (Tensor input, Tensor weight, Tensor bias, tuple of ints stride, tuple of ints padding, tuple of ints dilation, int groups)
didn't match because some of the arguments have invalid types: (list, Parameter, NoneType, tuple, tuple, tuple, int)
* (Tensor input, Tensor weight, Tensor bias, tuple of ints stride, str padding, tuple of ints dilation, int groups)
didn't match because some of the arguments have invalid types: (list, Parameter, NoneType, tuple, tuple, tuple, int)
Here is the Decoder Module based on MobileNet
class Net(nn.Module):
"""Net Definition"""
mobilenet_config = [[1, 16, 1, 1], # expansion rate, output channels, number of repeats, stride
[6, 24, 2, 2],
[6, 32, 3, 2],
[6, 64, 4, 2],
[6, 96, 3, 1],
[6, 160, 3, 2],
[6, 320, 1, 1],
]
in_planes = 32 # number of input channels
num_layers = len(mobilenet_config)
def __init__(self, num_classes, num_tasks=2):
super(Net, self).__init__()
self.num_tasks = num_tasks
assert self.num_tasks in [2, 3], "Number of tasks supported is either 2 or 3, got {}".format(self.num_tasks)
self.layer1 = convbnrelu(3, self.in_planes, kernel_size=3, stride=2)
c_layer = 2
for t,c,n,s in (self.mobilenet_config):
layers = []
for idx in range(n):
layers.append(InvertedResidualBlock(self.in_planes, c, expansion_factor=t, stride=s if idx == 0 else 1))
self.in_planes = c
setattr(self, 'layer{}'.format(c_layer), nn.Sequential(*layers))
c_layer += 1
## Light-Weight RefineNet ##
self.conv8 = conv1x1(320, 256, bias=False)
self.conv7 = conv1x1(160, 256, bias=False)
self.conv6 = conv1x1(96, 256, bias=False)
self.conv5 = conv1x1(64, 256, bias=False)
self.conv4 = conv1x1(32, 256, bias=False)
self.conv3 = conv1x1(24, 256, bias=False)
self.crp4 = self._make_crp(256, 256, 4, groups=False)
self.crp3 = self._make_crp(256, 256, 4, groups=False)
self.crp2 = self._make_crp(256, 256, 4, groups=False)
self.crp1 = self._make_crp(256, 256, 4, groups=True)
self.conv_adapt4 = conv1x1(256, 256, bias=False)
self.conv_adapt3 = conv1x1(256, 256, bias=False)
self.conv_adapt2 = conv1x1(256, 256, bias=False)
self.pre_depth = conv1x1(256, 256, groups=256, bias=False)
self.depth = conv3x3(256, 1, bias=True)
self.pre_segm = conv1x1(256, 256, groups=256, bias=False)
self.segm = conv3x3(256, num_classes, bias=True)
self.relu = nn.ReLU6(inplace=True)
if self.num_tasks == 3:
self.pre_normal = conv1x1(256, 256, groups=256, bias=False)
self.normal = conv3x3(256, 3, bias=True)
self._initialize_weights()
def forward(self, x):
x = self.layer1(x)
x = self.layer2(x) # x / 2
l3 = self.layer3(x) # 24, x / 4
l4 = self.layer4(l3) # 32, x / 8
l5 = self.layer5(l4) # 64, x / 16
l6 = self.layer6(l5) # 96, x / 16
l7 = self.layer7(l6) # 160, x / 32
l8 = self.layer8(l7) # 320, x / 32
l8 = self.conv8(l8)
l7 = self.conv7(l7)
l7 = self.relu(l8 + l7)
l7 = self.crp4(l7)
l7 = self.conv_adapt4(l7)
l7 = nn.Upsample(size=l6.size()[2:], mode='bilinear', align_corners=False)(l7)
l6 = self.conv6(l6)
l5 = self.conv5(l5)
l5 = self.relu(l5 + l6 + l7)
l5 = self.crp3(l5)
l5 = self.conv_adapt3(l5)
l5 = nn.Upsample(size=l4.size()[2:], mode='bilinear', align_corners=False)(l5)
l4 = self.conv4(l4)
l4 = self.relu(l5 + l4)
l4 = self.crp2(l4)
l4 = self.conv_adapt2(l4)
l4 = nn.Upsample(size=l3.size()[2:], mode='bilinear', align_corners=False)(l4)
l3 = self.conv3(l3)
l3 = self.relu(l3 + l4)
l3 = self.crp1(l3)
out_segm = self.pre_segm(l3)
out_segm = self.relu(out_segm)
out_segm = self.segm(out_segm)
out_d = self.pre_depth(l3)
out_d = self.relu(out_d)
out_d = self.depth(out_d)
if self.num_tasks == 3:
out_n = self.pre_normal(l3)
out_n = self.relu(out_n)
out_n = self.normal(out_n)
return out_segm, out_d, out_n
else:
return out_segm, out_d
def _initialize_weights(self):
for m in self.modules():
if isinstance(m, nn.Conv2d):
m.weight.data.normal_(0, 0.01)
if m.bias is not None:
m.bias.data.zero_()
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
And here is the training code
# model setup
enc = dt.nn.mobilenetv2(pretrained=pretrained, return_idx=return_idx)
#dec = dt.nn.MTLWRefineNet(enc._out_c, collapse_ind, num_classes)
dec = dt.nn.Net(num_classes, num_tasks)
model1 = nn.DataParallel(nn.Sequential(enc, dec).cuda())
print("Model has {} parameters".format(dt.misc.compute_params(model1)))
start_epoch, _, state_dict = saver.maybe_load(
ckpt_path=ckpt_path, keys_to_load=["epoch", "best_val", "state_dict"],
)
dt.misc.load_state_dict(model1, state_dict)
if start_epoch is None:
start_epoch = 0
# optim setup
optims = [
dt.misc.create_optim(
optim_enc, enc.parameters(), lr=lr_enc, momentum=mom_enc, weight_decay=wd_enc
),
dt.misc.create_optim(
optim_dec, dec.parameters(), lr=lr_dec, momentum=mom_dec, weight_decay=wd_dec
),
]
# schedulers
opt_scheds = []
for opt in optims:
opt_scheds.append(
torch.optim.lr_scheduler.MultiStepLR(
opt, np.arange(start_epoch + 1, n_epochs, 100), gamma=0.1
)
)
for i in range(start_epoch, n_epochs):
for sched in opt_scheds:
sched.step(i)
model1.train()
print("Epoch {:d}".format(i))
dt.engine.train(model1, optims, [crit_segm, crit_depth], trainloader, loss_coeffs)
if i % val_every == 0:
metrics = [
dt.engine.MeanIoU(num_classes[0]),
dt.engine.RMSE(ignore_val=ignore_depth),
]
model1.eval()
with torch.no_grad():
vals = dt.engine.validate(model1, metrics, valloader)
saver.maybe_save(
new_val=vals, dict_to_save={"state_dict": model1.state_dict(), "epoch": i}
)
Need help with resolving the error. I have'nt shared the code related to other modules such as InvertedResidualBlock and CRP to keep the code to minimum reproducible example