• Class-Balanced Loss Based on Effective Number of Samples


    参考:https://github.com/vandit15/Class-balanced-loss-pytorch

    其中的class_balanced_loss.py:

    import numpy as np
    import torch
    import torch.nn.functional as F
    
    
    
    def focal_loss(labels, logits, alpha, gamma):
        """Compute the focal loss between `logits` and the ground truth `labels`.
        Focal loss = -alpha_t * (1-pt)^gamma * log(pt)
        where pt is the probability of being classified to the true class.
        pt = p (if true class), otherwise pt = 1 - p. p = sigmoid(logit).
        Args:
          labels: A float tensor of size [batch, num_classes].
          logits: A float tensor of size [batch, num_classes].
          alpha: A float tensor of size [batch_size]
            specifying per-example weight for balanced cross entropy.
          gamma: A float scalar modulating loss from hard and easy examples.
        Returns:
          focal_loss: A float32 scalar representing normalized total loss.
        """    
        BCLoss = F.binary_cross_entropy_with_logits(input = logits, target = labels,reduction = "none")
    
        if gamma == 0.0:
            modulator = 1.0
        else:
            modulator = torch.exp(-gamma * labels * logits - gamma * torch.log(1 + 
                torch.exp(-1.0 * logits)))
    
        loss = modulator * BCLoss
    
        weighted_loss = alpha * loss
        focal_loss = torch.sum(weighted_loss)
    
        focal_loss /= torch.sum(labels)
        return focal_loss
    
    
    
    def CB_loss(labels, logits, samples_per_cls, no_of_classes, loss_type, beta, gamma):
        """Compute the Class Balanced Loss between `logits` and the ground truth `labels`.
        Class Balanced Loss: ((1-beta)/(1-beta^n))*Loss(labels, logits)
        where Loss is one of the standard losses used for Neural Networks.
        Args:
          labels: A int tensor of size [batch].
          logits: A float tensor of size [batch, no_of_classes].
          samples_per_cls: A python list of size [no_of_classes].
          no_of_classes: total number of classes. int
          loss_type: string. One of "sigmoid", "focal", "softmax".
          beta: float. Hyperparameter for Class balanced loss.
          gamma: float. Hyperparameter for Focal loss.
        Returns:
          cb_loss: A float tensor representing class balanced loss
        """
        effective_num = 1.0 - np.power(beta, samples_per_cls)
        weights = (1.0 - beta) / np.array(effective_num)
        weights = weights / np.sum(weights) * no_of_classes
    
        labels_one_hot = F.one_hot(labels, no_of_classes).float()
    
        weights = torch.tensor(weights).float()
        weights = weights.unsqueeze(0)
        weights = weights.repeat(labels_one_hot.shape[0],1) * labels_one_hot
        weights = weights.sum(1)
        weights = weights.unsqueeze(1)
        weights = weights.repeat(1,no_of_classes)
    
        if loss_type == "focal":
            cb_loss = focal_loss(labels_one_hot, logits, weights, gamma)
        elif loss_type == "sigmoid":
            cb_loss = F.binary_cross_entropy_with_logits(input = logits,target = labels_one_hot, weights = weights)
        elif loss_type == "softmax":
            pred = logits.softmax(dim = 1)
            cb_loss = F.binary_cross_entropy(input = pred, target = labels_one_hot, weight = weights)
        return cb_loss
    
    
    
    if __name__ == '__main__':
        no_of_classes = 5
        logits = torch.rand(10,no_of_classes).float()
        labels = torch.randint(0,no_of_classes, size = (10,))
        beta = 0.9999
        gamma = 2.0
        samples_per_cls = [2,3,1,2,2]
        loss_type = "focal"
        cb_loss = CB_loss(labels, logits, samples_per_cls, no_of_classes,loss_type, beta, gamma)
        print(cb_loss)
    View Code

    添加注释和输出的版本:

    #coding:utf-8
    import numpy as np
    import torch
    import torch.nn.functional as F
    
    
    
    def focal_loss(labels, logits, alpha, gamma):
        """Compute the focal loss between `logits` and the ground truth `labels`.
        Focal loss = -alpha_t * (1-pt)^gamma * log(pt)
        where pt is the probability of being classified to the true class.
        pt = p (if true class), otherwise pt = 1 - p. p = sigmoid(logit).
        Args:
          labels: A float tensor of size [batch, num_classes].
          logits: A float tensor of size [batch, num_classes].
          alpha: A float tensor of size [batch_size]
            specifying per-example weight for balanced cross entropy.
          gamma: A float scalar modulating loss from hard and easy examples.
        Returns:
          focal_loss: A float32 scalar representing normalized total loss.
        """    
        BCLoss = F.binary_cross_entropy_with_logits(input = logits, target = labels,reduction = "none")
    
        if gamma == 0.0:
            modulator = 1.0
        else:
            modulator = torch.exp(-gamma * labels * logits - gamma * torch.log(1 + 
                torch.exp(-1.0 * logits)))
    
        loss = modulator * BCLoss
    
        weighted_loss = alpha * loss
        
        # 然后求损失的均值mean()
        focal_loss = torch.sum(weighted_loss)
        focal_loss /= torch.sum(labels)
        return focal_loss
    
    
    
    def CB_loss(labels, logits, samples_per_cls, no_of_classes, loss_type, beta, gamma):
        """Compute the Class Balanced Loss between `logits` and the ground truth `labels`.
        Class Balanced Loss: ((1-beta)/(1-beta^n))*Loss(labels, logits)
        where Loss is one of the standard losses used for Neural Networks.
        Args:
          labels: A int tensor of size [batch].
          logits: A float tensor of size [batch, no_of_classes].
          samples_per_cls: A python list of size [no_of_classes].
          no_of_classes: total number of classes. int
          loss_type: string. One of "sigmoid", "focal", "softmax".
          beta: float. Hyperparameter for Class balanced loss.
          gamma: float. Hyperparameter for Focal loss.
        Returns:
          cb_loss: A float tensor representing class balanced loss
        """
    
        # 下面的操作用来计算((1-beta)/(1-beta^n)),即使用在损失函数中的weight
        effective_num = 1.0 - np.power(beta, samples_per_cls)
        print('effective_num shape: ', effective_num.shape)
        print(effective_num)
        weights = (1.0 - beta) / np.array(effective_num)
        print('weights shape : ', weights.shape)
        print(weights)
        weights = weights / np.sum(weights) * no_of_classes #归一化
        print('weights shape : ', weights.shape)
        print(weights)
    
        labels_one_hot = F.one_hot(labels, no_of_classes).float()
        print('labels_one_hot shape: ', labels_one_hot.shape)
        print(labels_one_hot)
    
        print('-'*50)
        weights = torch.tensor(weights).float()
        weights = weights.unsqueeze(0)
        print('unsqueeze weights shape : ', weights.shape)
        print(weights)
        #labels_one_hot.shape[0]得到样本数量,weight.repeat(,1)函数中的1表示weight对应位置的大小不变,所以是对行repeat labels_one_hot.shape[0]变
        print(weights.repeat(labels_one_hot.shape[0],1)) #([1,5])变成([10,5])
        weights = weights.repeat(labels_one_hot.shape[0],1) * labels_one_hot 
        print('repeat weights shape : ', weights.shape)
        print(weights)
        weights = weights.sum(1) #按dim=1相加,只留下dim=0,即得到每个样本的weight
        print('sum weights shape : ', weights.shape)
        print(weights)
        weights = weights.unsqueeze(1)
        print('unsqueeze weights shape : ', weights.shape)
        print(weights)
        weights = weights.repeat(1,no_of_classes) #这个就是按列相乘,([10,1])变成([10,5])
        print('repeat weights shape : ', weights.shape)
        print(weights)
        print('-'*50)
    
        if loss_type == "focal":
            cb_loss = focal_loss(labels_one_hot, logits, weights, gamma)
        elif loss_type == "sigmoid":
            cb_loss = F.binary_cross_entropy_with_logits(input = logits,target = labels_one_hot, weights = weights)
        elif loss_type == "softmax":
            pred = logits.softmax(dim = 1)
            cb_loss = F.binary_cross_entropy(input = pred, target = labels_one_hot, weight = weights)
        return cb_loss
    
    
    
    if __name__ == '__main__':
        no_of_classes = 5 #10个样本,5个类别
        logits = torch.rand(10,no_of_classes).float() # 预测10个样本分别是5个类别的概率
        print('logits shape : ', logits.shape)
        print(logits)
        labels = torch.randint(0,no_of_classes, size = (10,)) #10个样本的实际类别
        print('labels shape : ', labels.shape)
        print(labels)
    
        beta = 0.9999 #参数设置
        gamma = 2.0 #参数设置
        samples_per_cls = [2,3,1,2,2] # 每个类别的样本数
        loss_type = "focal"
        cb_loss = CB_loss(labels, logits, samples_per_cls, no_of_classes,loss_type, beta, gamma)
        print(cb_loss)
    View Code

    返回:

    (deeplearning) bogon:work_gender_age wanghui$ python test_delete.py 
    logits shape :  torch.Size([10, 5])
    tensor([[0.1505, 0.9621, 0.8622, 0.0237, 0.0270],
            [0.6218, 0.2745, 0.3015, 0.1501, 0.1728],
            [0.3590, 0.1760, 0.0807, 0.7440, 0.6973],
            [0.9401, 0.7118, 0.1725, 0.1843, 0.3226],
            [0.4655, 0.8319, 0.4336, 0.8718, 0.5842],
            [0.9423, 0.3339, 0.1081, 0.4718, 0.4329],
            [0.5122, 0.7010, 0.1736, 0.5903, 0.0712],
            [0.6442, 0.1365, 0.1391, 0.8278, 0.5986],
            [0.1245, 0.5662, 0.9571, 0.8515, 0.9883],
            [0.4654, 0.8924, 0.0224, 0.9056, 0.4517]])
    labels shape :  torch.Size([10])
    tensor([0, 3, 4, 2, 2, 1, 4, 0, 1, 2])
    effective_num shape:  (5,)
    [1.99990000e-04 2.99970001e-04 1.00000000e-04 1.99990000e-04
     1.99990000e-04]
    weights shape :  (5,)
    [0.500025   0.33336667 1.         0.500025   0.500025  ]
    weights shape :  (5,)
    [0.88236332 0.58827163 1.76463841 0.88236332 0.88236332]
    labels_one_hot shape:  torch.Size([10, 5])
    tensor([[1., 0., 0., 0., 0.],
            [0., 0., 0., 1., 0.],
            [0., 0., 0., 0., 1.],
            [0., 0., 1., 0., 0.],
            [0., 0., 1., 0., 0.],
            [0., 1., 0., 0., 0.],
            [0., 0., 0., 0., 1.],
            [1., 0., 0., 0., 0.],
            [0., 1., 0., 0., 0.],
            [0., 0., 1., 0., 0.]])
    --------------------------------------------------
    unsqueeze weights shape :  torch.Size([1, 5])
    tensor([[0.8824, 0.5883, 1.7646, 0.8824, 0.8824]])
    tensor([[0.8824, 0.5883, 1.7646, 0.8824, 0.8824],
            [0.8824, 0.5883, 1.7646, 0.8824, 0.8824],
            [0.8824, 0.5883, 1.7646, 0.8824, 0.8824],
            [0.8824, 0.5883, 1.7646, 0.8824, 0.8824],
            [0.8824, 0.5883, 1.7646, 0.8824, 0.8824],
            [0.8824, 0.5883, 1.7646, 0.8824, 0.8824],
            [0.8824, 0.5883, 1.7646, 0.8824, 0.8824],
            [0.8824, 0.5883, 1.7646, 0.8824, 0.8824],
            [0.8824, 0.5883, 1.7646, 0.8824, 0.8824],
            [0.8824, 0.5883, 1.7646, 0.8824, 0.8824]])
    repeat weights shape :  torch.Size([10, 5])
    tensor([[0.8824, 0.0000, 0.0000, 0.0000, 0.0000],
            [0.0000, 0.0000, 0.0000, 0.8824, 0.0000],
            [0.0000, 0.0000, 0.0000, 0.0000, 0.8824],
            [0.0000, 0.0000, 1.7646, 0.0000, 0.0000],
            [0.0000, 0.0000, 1.7646, 0.0000, 0.0000],
            [0.0000, 0.5883, 0.0000, 0.0000, 0.0000],
            [0.0000, 0.0000, 0.0000, 0.0000, 0.8824],
            [0.8824, 0.0000, 0.0000, 0.0000, 0.0000],
            [0.0000, 0.5883, 0.0000, 0.0000, 0.0000],
            [0.0000, 0.0000, 1.7646, 0.0000, 0.0000]])
    sum weights shape :  torch.Size([10])
    tensor([0.8824, 0.8824, 0.8824, 1.7646, 1.7646, 0.5883, 0.8824, 0.8824, 0.5883,
            1.7646])
    unsqueeze weights shape :  torch.Size([10, 1])
    tensor([[0.8824],
            [0.8824],
            [0.8824],
            [1.7646],
            [1.7646],
            [0.5883],
            [0.8824],
            [0.8824],
            [0.5883],
            [1.7646]])
    repeat weights shape :  torch.Size([10, 5])
    tensor([[0.8824, 0.8824, 0.8824, 0.8824, 0.8824],
            [0.8824, 0.8824, 0.8824, 0.8824, 0.8824],
            [0.8824, 0.8824, 0.8824, 0.8824, 0.8824],
            [1.7646, 1.7646, 1.7646, 1.7646, 1.7646],
            [1.7646, 1.7646, 1.7646, 1.7646, 1.7646],
            [0.5883, 0.5883, 0.5883, 0.5883, 0.5883],
            [0.8824, 0.8824, 0.8824, 0.8824, 0.8824],
            [0.8824, 0.8824, 0.8824, 0.8824, 0.8824],
            [0.5883, 0.5883, 0.5883, 0.5883, 0.5883],
            [1.7646, 1.7646, 1.7646, 1.7646, 1.7646]])
    --------------------------------------------------
    tensor(1.9583)
    View Code

    可见在代码中能够使用二分类求损失主要是因为将labels转换成了ont-hot格式

    labels_one_hot = F.one_hot(labels, no_of_classes).float()

    主要比较复杂的就是focal loss的实现:

    1)BCLoss = F.binary_cross_entropy_with_logits()

    2) modulator

    3)weight 即传进来的参数alpha

  • 相关阅读:
    ICO图标的制作、下载
    网页Loading,让页面加载完再显示
    鹏城之行
    The shit live,learn to give up!
    缘分
    一篇网络文章,能否让我真正顿悟???
    李开复给中国学生的第三封信:成功、自信、快乐
    Uncountable missing,missing...
    李开复给中国学生的第五封信:做个积极主动的你
    李开复给中国学生的第四封信:大学应这样过
  • 原文地址:https://www.cnblogs.com/wanghui-garcia/p/12196918.html
Copyright © 2020-2023  润新知