I have modified VGG16 in pytorch to insert things like BN and dropout within the feature extractor. By chance I now noticed something strange when I changed the definition of the forward method from:
def forward(self, x):
x = self.model(x)
return x
to:
def forward(self, x):
x = self.model.features(x)
x = self.model.avgpool(x)
x = self.model.classifier(x)
return x
In the second method I am now getting an error that the sizes of the matrices don't match
(mat1 dim 1 must match mat2 dim 0)
Below is the entire code For the editted version of VGG that I've been using.
class Vgg(nn.Module):
def __init__(self, n_classes, bias= None, dropout = 0.3):
super().__init__()
self.model = models.vgg16()
#self.bn64 = nn.BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
#self.bn128 = nn.BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
#self.bn256 = nn.BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
#self.bn512 = nn.BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
# change to allow 4 channels input
self.model.features[0] = nn.Conv2d(4, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3))
# remove/edit some of the first layers to make it more similar to Resnet
del self.model.features[2]
del self.model.features[2]
del self.model.features[-1]
self.model.features[2] = nn.MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)
# add dropout
for m in self.model.modules():
if isinstance(m, nn.Dropout):
m.p = dropout
else:
pass
self.dropout = nn.Dropout(p=dropout)
self.r = nn.ReLU(inplace=True)
modules = nn.Sequential(*[self.model.features[0],
#self.bn64,
self.model.features[1:3],
#self.bn64,
self.model.features[3:5],
#self.dropout,
self.model.features[5],
#self.bn128,
self.model.features[6:8],
#self.bn128,
self.model.features[8:10],
#self.dropout,
self.model.features[10],
#self.bn256,
self.model.features[11:13],
#self.bn256,
self.model.features[13:15],
#self.bn256,
self.model.features[15:17],
#self.dropout,
self.model.features[17],
#self.bn512,
self.model.features[18:20],
#self.bn512,
self.model.features[20:22],
#self.bn512,
self.model.features[22:24],
#self.dropout,
self.model.features[24],
#self.bn512,
self.model.features[25:27],
#self.bn512,
self.model.features[27:29],
#self.bn512,
#self.dropout
nn.MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
])
self.model.features = modules
# change the pooling layer
self.model.avgpool = nn.AdaptiveAvgPool2d(output_size=(1, 1))
# set output to correct num classes
self.model.classifier = nn.Linear(in_features=512, out_features=n_classes, bias=True)
# use predefined bias
if bias is not None:
assert isinstance(bias, torch.Tensor), 'bias must be tensor'
self.model.classifier.bias = nn.Parameter(bias)
def forward(self, x):
x = self.model.features(x)
x = self.model.avgpool(x)
x = self.model.classifier(x)
return x
I know it is very ugly and hacky looking. I have tried to re-write it but the re-written version does not work either for some reason, and I am assuming that this current issue is related to that as well. I think that the input is not being fed through the forward method the way I think it is. My assumption is that calling x = self.model(x) does not run 'x' through all of the editted layers I have made, otherwise I would get the same behaviour with the two version of the forward method above. But my question is then, what is happening when I call self.model(x) in forward? Is it running the input through the original vgg16 from pytorch? Because when I print self.model in my console it shows the changes I made to the architecture of self.model.features as well as self.model.avgpool and self.model.classifier.
Edit:
Below is the entire trace of the error. Some extra information. t is a class I made to deal with the training steps (so looping through training and validation modes, ect)
---------------------------------------------------------------------------
RuntimeError Traceback (most recent call last)
<ipython-input-7-0b3983ae9702> in <module>
77 print_cl_met = True
78 )
---> 79 model = t.run()
80 t.save_to_json()
81 print(np.max(np.array(t.f1_tracker)))
/home/stevea/treesat/TreeSat/TreeSat/trainers/basetrainer.py in run(self)
342 for phase in ['training', 'testing']:
343 self.phase = phase
--> 344 self.model_mode()
345
346 if self.phase == 'testing':
/home/stevea/treesat/TreeSat/TreeSat/trainers/basetrainer.py in model_mode(self)
154 if self.phase == 'training':
155 print('*********TRAINING PHASE*********')
--> 156 self.trainModel()
157 else:
158 print('*********VALIDATION PHASE*********')
/home/stevea/treesat/TreeSat/TreeSat/trainers/basetrainer.py in trainModel(self)
234 # loop through all batches to perform an epoch
235 for loaded in self.loaders[self.phase]:
--> 236 epoch_loss = self.train_step(loaded, epoch_loss)
237
238 mean_loss = np.mean(np.array(epoch_loss))
/home/stevea/treesat/TreeSat/TreeSat/trainers/basetrainer.py in train_step(self, loaded, epoch_loss)
262
263 # process batch through network
--> 264 self.out = self.model(self.img_batch.float())
265
266 # get loss value
/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
887 result = self._slow_forward(*input, **kwargs)
888 else:
--> 889 result = self.forward(*input, **kwargs)
890 for hook in itertools.chain(
891 _global_forward_hooks.values(),
/home/stevea/treesat/TreeSat/TreeSat/models/vgg.py in forward(self, x)
143 x = self.model.features(x)
144 x = self.model.avgpool(x)
--> 145 x = self.model.classifier(x)
146 return x
147
/usr/local/lib/python3.6/dist-packages/torch/nn/modules/module.py in _call_impl(self, *input, **kwargs)
887 result = self._slow_forward(*input, **kwargs)
888 else:
--> 889 result = self.forward(*input, **kwargs)
890 for hook in itertools.chain(
891 _global_forward_hooks.values(),
/usr/local/lib/python3.6/dist-packages/torch/nn/modules/linear.py in forward(self, input)
92
93 def forward(self, input: Tensor) -> Tensor:
---> 94 return F.linear(input, self.weight, self.bias)
95
96 def extra_repr(self) -> str:
/usr/local/lib/python3.6/dist-packages/torch/nn/functional.py in linear(input, weight, bias)
1751 if has_torch_function_variadic(input, weight):
1752 return handle_torch_function(linear, (input, weight), input, weight, bias=bias)
-> 1753 return torch._C._nn.linear(input, weight, bias)
1754
1755
RuntimeError: mat1 dim 1 must match mat2 dim 0
I also tried to print off the shape of x at each step of the forward method:
def forward(self, x):
x = self.model.features(x)
print(x.shape)
x = self.model.avgpool(x)
print(x.shape)
x = self.model.classifier(x)
print(x.shape)
return x
And it shows me that the shapes seem to be fine as the classifier should be taking in 512 features:
torch.Size([64, 512, 4, 4])
torch.Size([64, 512, 1, 1])