How to calculate F1 Score for Multi-label Classification

Viewed 98

I am trying to calculate F1 score (and accuracy) for my multi-label classification problem. Could you please provide feedback on my method, if I'm calculating it correctly. Note that I'm calculating IOU (intersection over union) when model predicts an object as 1, and mark it as TP only if IOU is greater than or equal to 0.5.

GT labels: 14 x 10 x 128
Output: 14 x 10 x 128

where 14 is the batch_size, 10 is the sequence_length, and 128 is the object vector (i.e., 1 if the object at an index belongs to the sequence and 0 otherwise).

def calculate_performance_metrics(total_padded_elements, gt_labels, predicted_labels):
        # check if TP pred objects overlap with TP gt objects
        TP_INDICES = (torch.logical_and(predicted_labels == 1, gt_labels == 1)).nonzero() # we only want the batch and object indices, i.e. the 0 and 2 indices
        TP = calculate_tp_with_iou() # details of this don't matter for now
        FP = torch.sum(torch.logical_and(predicted_labels, 1 - gt_labels)).item()
        TN = torch.sum(torch.logical_and(1 - predicted_labels, 1 - gt_labels)).item()
        FN = torch.sum(torch.logical_and(1 - predicted_labels, gt_labels)).item()
        return float(TP), float(FP), float(TN - total_padded_elements), float(FN)


for epoch in range(10):
    TP = FP = TN = FN = EPOCH_PRECISION = EPOCH_RECALL = EPOCH_F1 = 0.
    for inputs, gt_labels, masks in tr_dl:
        outputs = model(inputs) # out shape: (14, 10, 128)

        # mask shape: (14, 10). So need to expand it to the shape of output
        masks = masks[:, :, None].expand_as(outputs)
        
        pred_labels = (torch.sigmoid(outputs) >= 0.5).float().type(torch.int64) # consider all predictions above 0.5 as 1, rest 0
        pred_labels = pred_labels * masks
        gt_labels = (gt_labels * masks).type(torch.int64)
        total_padded_elements = masks.numel() - masks.sum() # need this to get accurate true negatives

        batch_tp, batch_fp, batch_tn, batch_fn = calculate_performance_metrics(gt_labels, pred_labels, total_padded_elements)
        EPOCH_TP += batch_tp
        EPOCH_FP += batch_fp
        EPOCH_TN += batch_tn
        EPOCH_FN += batch_fn

        EPOCH_ACCURACY = (EPOCH_TP + EPOCH_TN) / (EPOCH_TP + EPOCH_TN + EPOCH_FP + EPOCH_FN)
        
        if EPOCH_TP + EPOCH_FP > 0:
            EPOCH_PRECISION = EPOCH_TP / (EPOCH_TP + EPOCH_FP)

        if EPOCH_TP + EPOCH_FN > 0:
            EPOCH_RECALL = EPOCH_TP / (EPOCH_TP + EPOCH_FN)

        EPOCH_F1 = (2 * EPOCH_PRECISION * EPOCH_RECALL) / (EPOCH_PRECISION + EPOCH_RECALL)

0 Answers
Related