分割中的评价指标

Posted on

IOU

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
def normalize(mask, threshold):
    """
    对mask进行二值化，
    Args:
        mask:
        threshold: 二值化选择的阈值

    Returns: 二值化后的mask

    """
    mask[mask <= threshold] = 1
    mask[mask > threshold] = 0
    return mask

def iou(gt, pred):
    """
    计算真实mask和估计mask之间的iou
    Args:
        gt: 真实mask
        pred: 估计mask

    Returns: IOU(gt, pred)

    """
    normalized_gt = normalize(gt, 127)
    normalized_pred = normalize(pred, 127)
    
    and_area = cv2.bitwise_and(gt, pred)
    or_area = cv2.bitwise_or(gt, pred)
    return np.sum(and_area) / np.sum(or_area)

MAE

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
def normalize2(mask):
    """
    对mask进行二值化，直接除以最大值
    Args:
        mask:

    Returns:

    """
    return mask / np.amax(mask + 1e-8)
  

def mae(gt, pred):
    """
    计算两张mask之间的MAE，平均绝对误差
    Args:
        gt: 真实mask
        pred: 预测mask

    Returns:

    """
    if len(gt.shape) < 2 or len(pred.shape) < 2:
        print("ERROR: Mask1 or mask2 is not matrix!")
        exit()
    if len(gt.shape) > 2:
        gt = gt[:, :, 0]
    if len(pred.shape) > 2:
        pred = pred[:, :, 0]
    if gt.shape != pred.shape:
        print("ERROR: The shapes of mask1 and mask2 are different!")
        exit()

    h, w = gt.shape[:2]
    gt = normalize2(gt)
    pred = normalize2(pred)
    sum_error = np.sum(np.absolute(gt.astype(np.float32) - pred.astype(np.float32)))
    mae_error = sum_error / (float(h) * float(w) + 1e-8)

    return mae_error

Precision & Recall

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
def precision_recall(gt, pred):
    """
    计算两张mask之间在不同阈值下的precision和recall
    Args:
        gt:
        pred:

    Returns:

    """
    if len(gt.shape) < 2 or len(pred.shape) < 2:
        print("ERROR: Mask1 or mask2 is not matrix!")
        exit()
    if len(gt.shape) > 2:
        gt = gt[:, :, 0]
    if len(pred.shape) > 2:
        pred = pred[:, :, 0]
    if gt.shape != pred.shape:
        print("ERROR: The shapes of mask1 and mask2 are different!")
        exit()
        
    gt_num = gt[gt > 128].size  # 统计gt中一共有多少个前景点
    pp = pred[gt > 128]  # 统计在gt为前景区域的像素点处，预测的mask的像素值
    nn = pred[gt <= 128]  # 统计在gt为背景区域的像素点处，预测的mask的像素值

    mybins = np.arange(0, 256)  # 将分布图的横坐标划分成【0，255】

    pp_hist, pp_edges = np.histogram(pp, bins=mybins)  # 绘制在gt前景区域部分，预测mask在[0, 255]区间内的分布情况
    assert pp.size == pp_hist.sum()

    nn_hist, nn_edges = np.histogram(nn, bins=mybins)  # 绘制在gt背景区域部分，预测mask在[0, 255]区间内的分布情况
    assert nn.size == nn_hist.sum()

    pp_hist_flip = np.flipud(pp_hist)  # 将区间倒置变为[255, 254], [254, 253], ..., [1, 0] 主要是因为cumsum不能从后往前
    nn_hist_flip = np.flipud(nn_hist)  # 将区间倒置变为[255, 254], [254, 253], ..., [1, 0]

    pp_hist_flip_cumsum = np.cumsum(pp_hist_flip)  # 将区间像素数进行累加 [255, 254], [254, 253], ..., [1, 0]
    nn_hist_flip_cumsum = np.cumsum(nn_hist_flip)  # 将区间像素数进行累加 [255, 254], [254, 253], ..., [1, 0]

    precision = pp_hist_flip_cumsum / (pp_hist_flip_cumsum + nn_hist_flip_cumsum + 1e-8)  # TP/(TP+FP)
    recall = pp_hist_flip_cumsum / (gt_num + 1e-8)  # TP/(TP+FN)

    precision[np.isnan(precision)] = 0.0
    recall[np.isnan(recall)] = 0.0

    return precision[::-1], recall[::-1]

F_Measure

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
def F_measure(gt, pred, beta=pow(3, 0.5)):

    if len(gt.shape) < 2 or len(pred.shape) < 2:
        print("ERROR: Mask1 or mask2 is not matrix!")
        exit()
    if len(gt.shape) > 2:
        gt = gt[:, :, 0]
    if len(pred.shape) > 2:
        pred = pred[:, :, 0]
    if gt.shape != pred.shape:
        print("ERROR: The shapes of mask1 and mask2 are different!")
        exit()

    pr, re = precision_recall(gt, pred)
    mean_pr = np.mean(pr)  # average pre over the whole dataset at every threshold
    mean_re = np.mean(re)  # average pre over the whole dataset at every threshold
    f_measure = (1 + pow(beta, 2)) * mean_pr * mean_re / (pow(beta, 2) * mean_pr + mean_re + 1e-8)
    return f_measure

Weighted F_Measure

参考论文：https://arxiv.org/pdf/1708.06433.pdf

代码链接：https://github.com/jiwei0921/Saliency-Evaluation-Toolbox/blob/master/saliency_evaluation/WFb.m

Relax F_Measure

参考链接：https://github.com/NathanUA/BASNet/issues/20

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
def boundary_extraction(mask, k_size=2):
    """
    对结果进行腐蚀，进而得到边缘
    """

    (h, w) = mask.shape
    mask_pad = np.zeros((h+10, w+10))
    mask_pad[5:h+5, 5:w+5] = mask

    mask_pad_erd = ndimage.grey_erosion(mask_pad, size=(k_size, k_size))
    mask_pad_edge = np.logical_xor(mask_pad, mask_pad_erd)

    return mask_pad_edge[5:h+5, 5:w+5]


def precision_recall_f1(gt, pred):
    gt[gt <= 2*np.mean(gt)] = 0
    gt[gt > 2*np.mean(gt)] = 1

    pred[pred <= 127] = 0
    pred[pred > 127] = 1

    pred_edge = boundary_extraction(pred)
    gt_edge = boundary_extraction(gt)

    # distance_transform_edt 用于距离转换，计算图像中非零点到最近背景点（即0）的距离
    # 这里先将背景和前景互换，即背景变成1，前景变成0，则变为计算背景中距离最近边缘的距离
    pred_dist = distance_transform_edt(np.logical_not(pred_edge))
    gt_dist = distance_transform_edt(np.logical_not(gt_edge))

    # The relaxed boundary recall (relaxRecallb) measures the fraction of ground truth boundary pixels 
    # that are within ρ pixels of predicted boundary pixels. Here ρ sets to 3.
    buffer = 3
    pred_edge_buffer = np.zeros_like(pred_edge)
    pred_edge_buffer = pred_edge.copy()

    # precision = TP / 总数
    try:
        pred_edge_buffer[gt_dist > buffer] = 0  # buffer 之外的点（即距离最近的边缘超过3个像素的点不考虑在内）
    except:
        return 0.0, 0.0, 0.0
    
    precision_edge = np.sum(pred_edge_buffer).astype(np.float)/(np.sum(pred_edge).astype(np.float) + 1e-8)

    # recall = TP / (TP + FN)
    gt_edge_buffer = np.zeros_like(gt_edge)
    gt_edge_buffer = gt_edge.copy()
    
    try:
        gt_edge_buffer[pred_dist > buffer] = 0  # 
    except:
        return 0.0, 0.0, 0.0
    
    recall_edge = np.sum(gt_edge_buffer).astype(np.float)/(np.sum(gt_edge).astype(np.float) + 1e-8)

    # f1 = (1 + \{beta}2) * precision * recall / \{beta}2 * precision + recall
    f1_edge = (1 + 0.3) * precision_edge * recall_edge / (0.3 * precision_edge + recall_edge + 1e-8)
    
    return precision_edge, recall_edge, f1_edge

S_Measure

参考论文：https://arxiv.org/pdf/1708.00786.pdf

代码链接：https://github.com/zzhanghub/eval-co-sod/blob/7dd23832f84c01ccc2f876b4ce73b7acd8b82224/evaluator.py#L278

Mean Boundary Accuracy

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
def get_disk_kernel(radius):
    return cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (radius * 2 + 1, radius * 2 + 1))  # 获得一个圆形的核
    
def mean_boundary_acc(gt, pred):
    if len(gt.shape) < 2 or len(pred.shape) < 2:
        print("ERROR: Mask1 or mask2 is not matrix!")
        exit()
    if len(gt.shape) > 2:
        gt = gt[:, :, 0]
    if len(pred.shape) > 2:
        pred = pred[:, :, 0]
    if gt.shape != pred.shape:
        print("ERROR: The shapes of mask1 and mask2 are different!")
        exit()

    gt = normalize(gt)
    pred = normalize(pred)

    h, w = gt.shape

    min_radius = 1
    max_radius = (w + h) / 300
    num_steps = 5

    mask_acc = [0] * num_steps

    for i in range(num_steps):
        cur_radius = min_radius + int((max_radius - min_radius) / num_steps * i)
        kernel = get_disk_kernel(cur_radius)
        boundary_area = cv2.morphologyEx(gt, cv2.MORPH_GRADIENT, kernel)

        gt_bound_area = gt[boundary_area]
        pred_bound_area = pred[boundary_area]

        num_edges = np.sum(boundary_area)
        num_true_pos = np.sum((gt_bound_area * pred_bound_area) + (1 - gt_bound_area) * (1 - pred_bound_area))

        mask_acc[i] = num_true_pos / num_edges

    return sum(mask_acc) / num_steps