I am using 187 data as train set, which has 68 features and would like to extract 10 features then use PCA to plot in 2D
my original data is right skewed but the latent space becomes normal
even though the loss decreases well, the model doesn't seem to be learning
[model] variational autoencoder
layer 68 - 30 - 10 - 30 - 68, using leaky_relu as activation function and tanh in the final layer
added l1 regularization in loss function, and dropout in the encoder
class VAE(nn.Module):
def __init__(self):
super(VAE, self).__init__()
self.fc1 = nn.Linear(68, 30)
self.fc21 = nn.Linear(30, 10)
self.fc22 = nn.Linear(30, 10)
self.fc3 = nn.Linear(10, 30)
self.fc4 = nn.Linear(30, 68)
self.dropout = nn.Dropout(0.5)
def encode(self, x):
h1 = F.leaky_relu(self.fc1(x))
h1 = self.dropout(h1)
return self.fc21(h1), self.fc22(h1)
def reparameterize(self, mu, logvar):
std = logvar.mul(0.5).exp_()
if torch.cuda.is_available():
eps = torch.cuda.FloatTensor(std.size()).normal_()
else:
eps = torch.FloatTensor(std.size()).normal_()
eps = Variable(eps)
return eps.mul(std).add_(mu)
def decode(self, z):
h3 = F.leaky_relu(self.fc3(z))
return torch.tanh(self.fc4(h3)) # sigmoid -> relu
def forward(self, x):
mu, logvar = self.encode(x.view(-1, 68))
z = self.reparameterize(mu, logvar)
return self.decode(z), z, mu, logvar
model = VAE().to(device)
optimizer = optim.Adam(model.parameters(), lr=1e-3) # this will help for L2 regularization
def loss_function(recon_x, x, mu, logvar):
loss = nn.MSELoss(reduction = 'sum')
BCE = loss(recon_x, x)
KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
regularization_loss = 0 # l1 regularization
for param in model.parameters():
regularization_loss += torch.sum(torch.abs(param))
return BCE + KLD + regularization_loss
I have no clue why this is not working
this is the one I get -- very random
