RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [1, 64, 7, 7]]

Question

Trying to figure out errors with multiple grads. Each time i run the code, the 'expected version' changes. Also, the error triggers after 53 iterations of the program running, leaving me at a loss of understanding of why it will work for the first iterations but keep stalling on the 52-55th.

I am using retain_graph=True so it must not be coming from that but I just cant seem to figure out the issue. Thanks in advance.

for epoch in range(0, epoch_num):
    net.train()

    for i, data in enumerate(music_dataloader):
        torch.autograd.set_detect_anomaly(True)
        ite = ite + 1
        ite_num4val = ite_num4val + 1
        real_a, real_b, real_mixed = data['bar_a'], data['bar_b'], data['bar_mixed']
        real_a = torch.FloatTensor(real_a)
        real_b = torch.FloatTensor(real_b)
        real_mixed = torch.FloatTensor(real_mixed)

        if torch.cuda.is_available():
            real_a = real_a.cuda()
            real_b = real_b.cuda()
            real_mixed = real_mixed.cuda()

        # zero the parameter gradients
        optimizer_GA2B.zero_grad()
        optimizer_GB2A.zero_grad()
        optimizer_DA.zero_grad()
        optimizer_DB.zero_grad()
        optimizer_DA_all.zero_grad()
        optimizer_DB_all.zero_grad()

        c_loss, DA_real, DB_real, DA_fake, DB_fake, DA_fake_sample, DB_fake_sample, DA_real_all, DB_real_all, DA_fake_all, DB_fake_all = net(
            real_a, real_b, real_mixed)

        # Generator loss
        g_A2B_loss = l1loss(DB_fake, torch.ones_like(DB_fake)) + c_loss
        g_B2A_loss = l1loss(DA_fake, torch.ones_like(DA_fake)) + c_loss
        g_loss = g_A2B_loss + g_B2A_loss - c_loss

        # Discriminator loss
        d_A_loss_real = l2loss(DA_real, torch.ones_like(DA_real))
        d_A_loss_fake = l2loss(
            DA_fake_sample, torch.zeros_like(DA_fake_sample))
        d_A_loss = (d_A_loss_real + d_A_loss_fake) / 2
        d_B_loss_real = l2loss(DB_real, torch.ones_like(DB_real))
        d_B_loss_fake = l2loss(
            DB_fake_sample, torch.zeros_like(DB_fake_sample))
        d_B_loss = (d_B_loss_real + d_B_loss_fake) / 2
        d_loss = d_A_loss + d_B_loss

        d_A_all_loss_real = l2loss(DA_real_all, torch.ones_like(DA_real_all))
        d_A_all_loss_fake = l2loss(DA_fake_all, torch.zeros_like(DA_fake_all))
        d_A_all_loss = (d_A_all_loss_real + d_A_all_loss_fake) / 2
        d_B_all_loss_real = l2loss(DB_real_all, torch.ones_like(DB_real_all))
        d_B_all_loss_fake = l2loss(DB_fake_all, torch.zeros_like(DB_fake_all))
        d_B_all_loss = (d_B_all_loss_real + d_B_all_loss_fake) / 2
        d_all_loss = d_A_all_loss + d_B_all_loss
        D_loss = d_loss + gamma * d_all_loss
        
        d_A_loss.backward(retain_graph=True)
        d_B_loss.backward(retain_graph=True)

        g_A2B_loss.backward(retain_graph=True)
        g_B2A_loss.backward(retain_graph=True)


        d_A_all_loss.backward(retain_graph=True)
        d_B_all_loss.backward()

        optimizer_GA2B.step()
        optimizer_GB2A.step()
        optimizer_DA.step()
        optimizer_DB.step()
        optimizer_DA_all.step()
        optimizer_DB_all.step()

        g_running_loss += g_loss.data.item()
        d_running_loss += D_loss.data.item()

        del DA_real, DB_real, DA_fake, DB_fake, DA_fake_sample, DB_fake_sample, DA_real_all, DB_real_all, DA_fake_all, DB_fake_all
        del g_A2B_loss, g_B2A_loss, g_loss, d_A_loss_real, d_A_loss_fake, d_A_loss, d_B_loss_real, d_B_loss_fake, d_B_loss, d_loss, d_A_all_loss_real, d_A_all_loss_fake, d_A_all_loss, d_B_all_loss_real, d_B_all_loss_fake, d_B_all_loss, d_all_loss, D_loss

        print("[epoch: %3d/%3d, batch: %5d/%5d, ite: %d] Generator : %3f, Discriminator : %3f " % (
            epoch +
            1, epoch_num, (i + 1) *
            batch_size_train, train_num, ite, g_running_loss / ite_num4val,
            d_running_loss / ite_num4val))

        if ite % save_frq == 0:
            torch.save(net, model_dir + model_name + "_itr_%d_G_%3f_D_%3f.pth" % (
                ite, g_running_loss / ite_num4val, d_running_loss / ite_num4val))
            g_running_loss = 0.0
            d_running_loss = 0.0
            net.train()
            ite_num4val = 0

 in 
    132         D_loss = d_loss + gamma * d_all_loss
    133 
--> 134         d_A_loss.backward(retain_graph=True)
    135         d_B_loss.backward(retain_graph=True)
    136 

/software/python3/3.9.5/lib/python3.9/site-packages/torch/tensor.py in backward(self, gradient, retain_graph, create_graph, inputs)
    243                 create_graph=create_graph,
    244                 inputs=inputs)
--> 245         torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)
    246 
    247     def register_hook(self, hook):

/software/python3/3.9.5/lib/python3.9/site-packages/torch/autograd/__init__.py in backward(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)
    143         retain_graph = create_graph
    144 
--> 145     Variable._execution_engine.run_backward(
    146         tensors, grad_tensors_, retain_graph, create_graph, inputs,
    147         allow_unreachable=True, accumulate_grad=True)  # allow_unreachable flag

RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [1, 64, 7, 7]] is at version 53; expected version 25 instead. Hint: the backtrace further above shows the operation that failed to compute its gradient. The variable in question was changed in there or anywhere later. Good luck!

RuntimeError: one of the variables needed for gradient computation has been modified by an inplace operation: [torch.cuda.FloatTensor [1, 64, 7, 7]]

0 Answers