I'm building two parallel convolutional neural networks (CNN) in parallel with a Transformer encoder network to classify image data. I'm working on a dataset of faces to classify emotions from one of 7 classes. But when I ran the model, I got this error:
AttributeError: 'NoneType' object has no attribute 'size'. The model is as follows:
class parallel_all_you_want(nn.Module):
# Define all layers present in the network
def __init__(self,num_emotions):
super().__init__()
################ TRANSFORMER BLOCK #############################
# maxpool the input feature map/tensor to the transformer
# a rectangular kernel worked better here for the rectangular input spectrogram feature map/tensor
self.transformer_maxpool = nn.MaxPool2d(kernel_size=[1,4], stride=[1,4])
transformer_layer = nn.TransformerEncoderLayer(
d_model=40, # input feature (frequency) dim after maxpooling 40*282 -> 40*70 (MFC*time)
nhead=4, # 4 self-attention layers in each multi-head self-attention layer in each encoder block
dim_feedforward=512, # 2 linear layers in each encoder block's feedforward network: dim 40-->512--->40
dropout=0.4,
activation='relu' # ReLU: avoid saturation/tame gradient/reduce compute time
)
self.transformer_encoder = nn.TransformerEncoder(transformer_layer, num_layers=4)
############### 1ST PARALLEL 2D CONVOLUTION BLOCK ############
# 3 sequential conv2D layers: (1,40,282) --> (16, 20, 141) -> (32, 5, 35) -> (64, 1, 8)
self.conv2Dblock1 = nn.Sequential(
# 1st 2D convolution layer
nn.Conv2d(
in_channels=1, # input volume depth == input channel dim == 1
out_channels=16, # expand output feature map volume's depth to 16
kernel_size=3, # typical 3*3 stride 1 kernel
stride=1,
padding=1
),
nn.BatchNorm2d(16), # batch normalize the output feature map before activation
nn.ReLU(), # feature map --> activation map
nn.MaxPool2d(kernel_size=2, stride=2), #typical maxpool kernel size
nn.Dropout(p=0.3), #randomly zero 30% of 1st layer's output feature map in training
# 2nd 2D convolution layer identical to last except output dim, maxpool kernel
nn.Conv2d(
in_channels=16,
out_channels=32, # expand output feature map volume's depth to 32
kernel_size=3,
stride=1,
padding=1
),
nn.BatchNorm2d(32),
nn.ReLU(),
nn.MaxPool2d(kernel_size=4, stride=4), # increase maxpool kernel for subsequent filters
nn.Dropout(p=0.3),
# 3rd 2D convolution layer identical to last except output dim
nn.Conv2d(
in_channels=32,
out_channels=64, # expand output feature map volume's depth to 64
kernel_size=3,
stride=1,
padding=1
),
nn.BatchNorm2d(64),
nn.ReLU(),
nn.MaxPool2d(kernel_size=4, stride=4),
nn.Dropout(p=0.3),
)
############### 2ND PARALLEL 2D CONVOLUTION BLOCK ############
# 3 sequential conv2D layers: (1,40,282) --> (16, 20, 141) -> (32, 5, 35) -> (64, 1, 8)
self.conv2Dblock2 = nn.Sequential(
# 1st 2D convolution layer
nn.Conv2d(
in_channels=1, # input volume depth == input channel dim == 1
out_channels=16, # expand output feature map volume's depth to 16
kernel_size=3, # typical 3*3 stride 1 kernel
stride=1,
padding=1
),
nn.BatchNorm2d(16), # batch normalize the output feature map before activation
nn.ReLU(), # feature map --> activation map
nn.MaxPool2d(kernel_size=2, stride=2), #typical maxpool kernel size
nn.Dropout(p=0.3), #randomly zero 30% of 1st layer's output feature map in training
# 2nd 2D convolution layer identical to last except output dim, maxpool kernel
nn.Conv2d(
in_channels=16,
out_channels=32, # expand output feature map volume's depth to 32
kernel_size=3,
stride=1,
padding=1
),
nn.BatchNorm2d(32),
nn.ReLU(),
nn.MaxPool2d(kernel_size=4, stride=4), # increase maxpool kernel for subsequent filters
nn.Dropout(p=0.3),
# 3rd 2D convolution layer identical to last except output dim
nn.Conv2d(
in_channels=32,
out_channels=64, # expand output feature map volume's depth to 64
kernel_size=3,
stride=1,
padding=1
),
nn.BatchNorm2d(64),
nn.ReLU(),
nn.MaxPool2d(kernel_size=4, stride=4),
nn.Dropout(p=0.3),
)
self.fc1_linear = nn.Linear(512*2+40,num_emotions)
self.softmax_out = nn.Softmax(dim=1) # dim==1 is the freq embedding
# define one complete parallel fwd pass of input feature tensor thru 2*conv+1*transformer blocks
def forward(self,x):
print(x.shape)
conv2d_embedding1 = self.conv2Dblock1(x) # x == N/batch * channel * freq * time
conv2d_embedding1 = torch.flatten(conv2d_embedding1, start_dim=1)
conv2d_embedding2 = self.conv2Dblock2(x) # x == N/batch * channel * freq * time
conv2d_embedding2 = torch.flatten(conv2d_embedding2, start_dim=1)
x_maxpool = self.transformer_maxpool(x)
x_maxpool_reduced = torch.squeeze(x_maxpool,1)
x = x_maxpool_reduced.permute(2,0,1)
transformer_output = self.transformer_encoder(x)
transformer_embedding = torch.mean(transformer_output, dim=0) # dim 40x70 --> 40
complete_embedding = torch.cat([conv2d_embedding1, conv2d_embedding2,transformer_embedding], dim=1)
output_logits = self.fc1_linear(complete_embedding)
output_softmax = self.softmax_out(output_logits)
return output_logits, output_softmax
This is the implementation of Keras' model.summary method:
from torchsummary import summary
# need device to instantiate model
device = 'cuda'
# instantiate model for 8 emotions and move to GPU
model = parallel_all_you_want(len(emotion_labels)).to(device)
# include input feature map dims in cemotions_dictall to summary()
summary(model, input_size=(1,40,282))