I've been doing a project regarding making my own WaveNet implementation as Deepmind delivered early in the 2016's in Python.
Preprocessing includes mu law encoding, and one hot encoding. The model itself functions well, my problem lies in the loss function torch.nn.functional.cross_entropy used during training, found here: https://pytorch.org/docs/stable/nn.functional.html
Particularly, the relation between my output and my target tensors, namely
input_tensor.shape = tensor([1, 256, 225332]) # [batch_size, sample_size, audio_length]
output.shape = tensor([1, 256, 225332])
According to F.cross_entropy, I must have output = (N, C) and target = input_tensor = (N). My supervisor told me to do the following:
output = output.T.reshape(-1, 256) = tensor([225332, 256])
target = input_tensor.T.long() = tensor([225332, 256, 1]) # This needs to be 1-dimensional, help?
For anyone interested in the explicit code, below: NOTE - the receptive field is not padded, so just for debugging purposes I have subtracted it, while I do know this is not natural.
>>> output.T.reshape(-1, 256).shape
torch.Size([225332, 256])
>>> input_tensor[:, :, model.input_size - model.output_size:].T.shape
torch.Size([225332, 256, 1])
>>> loss = F.cross_entropy(output.T.reshape(-1, 256), input_tensor[:, :, model.input_size - model.output_size:].T.long().to(device))
Traceback (most recent call last):
File "C:\Program Files\JetBrains\PyCharm Community Edition 2020.3.3\plugins\python-ce\helpers\pydev\_pydevd_bundle\pydevd_exec2.py", line 3, in Exec
exec(exp, global_vars, local_vars)
File "<input>", line 1, in <module>
File "C:\Users\JaQtae\anaconda3\envs\CortiGit\lib\site-packages\torch\nn\functional.py", line 2693, in cross_entropy
return nll_loss(log_softmax(input, 1), target, weight, None, ignore_index, None, reduction)
File "C:\Users\JaQtae\anaconda3\envs\CortiGit\lib\site-packages\torch\nn\functional.py", line 2388, in nll_loss
ret = torch._C._nn.nll_loss(input, target, weight, _Reduction.get_enum(reduction), ignore_index)
RuntimeError: 1D target tensor expected, multi-target not supported
Somewhat of a novice-in-training with ML and AI, particularly the PyTorch library.
Would appreciate any advice regarding how I should tackle this issue.
The training:
model = Wavenet(layers=3,blocks=2,output_size=32).to(device)
model.apply(initialize) # Initialize causalconv1d() with xavier_uniform_ weights and bias of 0.
model.train()
optimizer = optim.Adam(model.parameters(), lr=0.0003)
for i, batch in tqdm(enumerate(train_loader)):
mu_enc_my_x = encode_mu_law(x=batch, mu=256)
input_tensor = one_hot_encoding(mu_enc_my_x)
input_tensor = input_tensor.to(device)
output = model(input_tensor)
# TODO: Inspect input/output formats, maybe something wrong....
loss = F.cross_entropy(output.T.reshape(-1, 256), input_tensor[:,:,model.input_size - model.output_size:].long().to(device)) # subtract receptive field instead of pad it, workaround for quick debugging of loss-issue.
print("\nLoss:", loss.item())
optimizer.zero_grad()
loss.backward()
optimizer.step()
if i % 1000 == 0:
print("\nSaving model")
torch.save(model.state_dict(), "wavenet.pt")
The purpose is to get my loss function to work properly, so that I can generate sound files. The current ones with my bad loss function obviously return pure noise.
My full model if any help.
"""
Wavenet model
Sources:
https://github.com/kan-bayashi/PytorchWaveNetVocoder/blob/master/wavenet_vocoder/nets/wavenet.py
https://github.com/r9y9/wavenet_vocoder/blob/master/wavenet_vocoder/wavenet.py
https://github.com/Dankrushen/Wavenet-PyTorch/blob/master/wavenet/models.py
https://github.com/vincentherrmann/pytorch-wavenet
"""
from torch import nn
import torch
#TODO: Add local and global conditioning
def initialize(m):
"""
Initialize CNN with Xavier_uniform weight and 0 bias.
"""
if isinstance(m, torch.nn.Conv1d):
nn.init.xavier_uniform_(m.weight)
nn.init.constant_(m.bias, 0.0)
class CausalConv1d(torch.nn.Module):
"""
Causal Convolution for WaveNet
Causality can be introduced with padding as (kernel_size - 1) * dilation (see Keras documentation)
or it can be introduced as follows according to Golbin.
https://github.com/golbin/WaveNet/blob/05545339096c3a1d9909d96fb19da4fbae28d8c6/wavenet/networks.py#L38
Else, look at the following article, several ways to implement it using PyTorch:
https://github.com/pytorch/pytorch/issues/1333
- Jakob
"""
def __init__(self, in_channels, out_channels, kernel_size, dilation = 1, bias = True):
super(CausalConv1d, self).__init__()
# padding=1 for same size(length) between input and output for causal convolution
self.dilation = dilation
self.kernel_size = kernel_size
self.in_channels = in_channels
self.out_channels = out_channels
self.padding = padding = (kernel_size-1) * dilation # kernelsize = 2, -1 * dilation = 1, = 1. - Jakob.
self.conv = torch.nn.Conv1d(in_channels, out_channels,
kernel_size, padding=padding, dilation=dilation,
bias=bias) # Fixed for WaveNet but not sure
def forward(self, x):
output = self.conv(x)
if self.padding != 0:
output = output[:, :, :-self.padding]
return output
class Wavenet(nn.Module):
def __init__(self,
layers=3,
blocks=2,
dilation_channels=32,
residual_block_channels=512,
skip_connection_channels=512,
output_channels=256,
output_size=32,
kernel_size=3
):
super(Wavenet, self).__init__()
self.layers = layers
self.blocks = blocks
self.dilation_channels = dilation_channels
self.residual_block_channels = residual_block_channels
self.skip_connection_channels = skip_connection_channels
self.output_channels = output_channels
self.kernel_size = kernel_size
self.output_size = output_size
# initialize dilation variables
receptive_field = 1
init_dilation = 1
# List of layers and connections
self.dilations = []
self.residual_convs = nn.ModuleList()
self.filter_conv_layers = nn.ModuleList()
self.gate_conv_layers = nn.ModuleList()
self.skip_convs = nn.ModuleList()
# First convolutional layer
self.first_conv = CausalConv1d(in_channels=self.output_channels,
out_channels=residual_block_channels,
kernel_size = 2)
# Building the Modulelists for the residual blocks
for b in range(blocks):
additional_scope = kernel_size - 1
new_dilation = 1
for i in range(layers):
# dilations of this layer
self.dilations.append((new_dilation, init_dilation))
# dilated convolutions
self.filter_conv_layers.append(nn.Conv1d(in_channels=residual_block_channels, out_channels=dilation_channels, kernel_size=kernel_size, dilation=new_dilation))
self.gate_conv_layers.append(nn.Conv1d(in_channels=residual_block_channels, out_channels=dilation_channels, kernel_size=kernel_size, dilation=new_dilation))
# 1x1 convolution for residual connection
self.residual_convs.append(nn.Conv1d(in_channels=dilation_channels, out_channels=residual_block_channels, kernel_size=1))
# 1x1 convolution for skip connection
self.skip_convs.append(nn.Conv1d(in_channels=dilation_channels,
out_channels=skip_connection_channels,
kernel_size=1))
# Update receptive field and dilation
receptive_field += additional_scope
additional_scope *= 2
init_dilation = new_dilation
new_dilation *= 2
# Last two convolutional layers
self.last_conv_1 = nn.Conv1d(in_channels=skip_connection_channels,
out_channels=skip_connection_channels,
kernel_size=1)
self.last_conv_2 = nn.Conv1d(in_channels=skip_connection_channels,
out_channels=output_channels,
kernel_size=1)
#Calculate model receptive field and the required input size for the given output size
self.receptive_field = receptive_field
self.input_size = receptive_field + output_size - 1
def forward(self, input):
# Feed first convolutional layer with input
x = self.first_conv(input)
# Initialize skip connection
skip = 0
# Residual block
for i in range(self.blocks * self.layers):
(dilation, init_dilation) = self.dilations[i]
# Residual connection bypassing dilated convolution block
residual = x
# input to dilated convolution block
filter = self.filter_conv_layers[i](x)
filter = torch.tanh(filter)
gate = self.gate_conv_layers[i](x)
gate = torch.sigmoid(gate)
x = filter * gate
# Feed into 1x1 convolution for skip connection
s = self.skip_convs[i](x)
#Adding skip & Match size with decreasing dimensionality of x
if skip is not 0:
skip = skip[:, :, -s.size(2):]
skip = s + skip # Sum all skip connections
# Feed into 1x1 convolution for residual connection
x = self.residual_convs[i](x)
#Adding Residual & Match size with decreasing dimensionality of x
x = x + residual[:, :, dilation * (self.kernel_size - 1):]
# print(x.shape)
x = torch.relu(skip)
#Last conv layers
x = torch.relu(self.last_conv_1(x))
x = self.last_conv_2(x)
soft = torch.nn.Softmax(dim=1)
x = soft(x)
return x
EDIT: added code snippet of train for clarity, and full model