<jupyter_start><jupyter_text>Load model<jupyter_code>def get_model(checkpoint, effnet):
    model = timm.create_model(effnet, pretrained=False)
    if effnet=="efficientnet_b0" or effnet=="efficientnet_b1" or effnet=="efficientnet_b2":
        model.conv_stem = torch.nn.Conv2d(27, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
    elif effnet=="efficientnet_b3":
        model.conv_stem = torch.nn.Conv2d(27, 40, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
    else:
        model.conv_stem = torch.nn.Conv2d(27, 48, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
    in_features = model.classifier.in_features
    model.classifier = nn.Linear(in_features, 1)

    checkpoint = torch.load(checkpoint)
    model.load_state_dict(checkpoint['model_state_dict'])

    model.to("cuda").eval()
    return model
model = get_model(checkpoint_path, "efficientnet_b0")
model2 = get_model(checkpoint_path2, "efficientnet_b0")
model3 = get_model(checkpoint_path3, "efficientnet_b4")
# Do tta
import ttach as tta
transforms = tta.Compose(
    [
        tta.HorizontalFlip(),   
    ]
)

model = tta.ClassificationTTAWrapper(model, transforms)
model2 = tta.ClassificationTTAWrapper(model2, transforms)
model3 = tta.ClassificationTTAWrapper(model3, transforms)

models = [model, model2, model3]<jupyter_output><empty_output><jupyter_text>Test<jupyter_code>targets = []
for img in tqdm(val_loader):
    img = img.to("cuda")
    for i,model in enumerate(models):
        if i == 0:
            target = torch.sigmoid(model(img))
        else:
            target = torch.sigmoid(model(img)) + target
    target = target/len(models)
    targets.append(target.cpu().detach().numpy())
ts = []
for t in targets:
    ts.append(t[0])
targets = np.array(ts).reshape(-1)
preds_df["classification"] = np.array(targets)
preds_df.head()
positive_df = preds_df[preds_df["classification"]>0.3]
positive_df.head()
len(positive_df)<jupyter_output><empty_output><jupyter_text>Evaluate<jupyter_code>def calc_metric(tps,fps,fns):
    tps = np.sum(tps)
    fps = np.sum(fps)
    fns = np.sum(fns)
    precision = tps / (tps + fps + 1e-6)
    recall =  tps / (tps + fns +1e-6)
    f1_score = 2*(precision*recall)/(precision+recall+1e-6)
    return precision,recall,f1_score

def iou(bbox1, bbox2):
    bbox1 = [float(x) for x in bbox1]
    bbox2 = [float(x) for x in bbox2]

    (x0_1, y0_1, x1_1, y1_1) = bbox1
    (x0_2, y0_2, x1_2, y1_2) = bbox2

    # get the overlap rectangle
    overlap_x0 = max(x0_1, x0_2)
    overlap_y0 = max(y0_1, y0_2)
    overlap_x1 = min(x1_1, x1_2)
    overlap_y1 = min(y1_1, y1_2)

    # check if there is an overlap
    if overlap_x1 - overlap_x0 <= 0 or overlap_y1 - overlap_y0 <= 0:
            return 0

    # if yes, calculate the ratio of the overlap to each ROI size and the unified size
    size_1 = (x1_1 - x0_1) * (y1_1 - y0_1)
    size_2 = (x1_2 - x0_2) * (y1_2 - y0_2)
    size_intersection = (overlap_x1 - overlap_x0) * (overlap_y1 - overlap_y0)
    size_union = size_1 + size_2 - size_intersection

    return size_intersection / size_union

def find_best_match(gts, predd, threshold=0.35, frame=0.4):
    best_match_iou = -np.inf
    best_match_idx = -1
    best_dist = 100
    for gt_idx, ggt in enumerate(gts):
        dist = abs(ggt[4]-predd[4])
        if dist > frame:
            continue
        iou_score = iou(ggt[0:4], predd[0:4])
        
        if iou_score < threshold:
            continue
        
        if dist<=best_dist:
            if iou_score > best_match_iou:
                best_dist = dist
                best_match_iou = iou_score
                best_match_idx = gt_idx

    return best_match_idx

def precision_calc(gt_boxes, preds, val_thresh=0.25, frame=4): #frame_index, xmin, ymin, xmax, ymax
    tp = 0
    fp = 0
    fn = 0

    for pred_idx, pred in enumerate(preds):
        best_match_gt_idx = find_best_match(gt_boxes, pred, threshold=val_thresh, frame=frame)

        if best_match_gt_idx >= 0:
            # True positive: The predicted box matches a gt box with an IoU above the threshold.
            tp += 1

            # Remove the matched GT box
            gt_boxes = np.delete(gt_boxes, best_match_gt_idx, axis=0)

        else:
            # No match
            # False positive: indicates a predicted box had no associated gt box.
            fp += 1

    # False negative: indicates a gt box had no associated predicted box.
    fn = len(gt_boxes)

    precision = tp / (tp + fp + 1e-6)
    recall =  tp / (tp + fn +1e-6)
    f1_score = 2*(precision*recall)/(precision+recall+1e-6)

    return tp, fp, fn, precision, recall, f1_score

from scipy.optimize import linear_sum_assignment
def precision_calc(gt_boxes, pred_boxes, val_thresh=0.35, frame=4):
    cost_matix = np.ones((len(gt_boxes), len(pred_boxes)))
    for i, box1 in enumerate(gt_boxes):
        for j, box2 in enumerate(pred_boxes):
            dist = abs(box1[4]-box2[4])
            if dist > 4:
                continue
            iou_score = iou(box1[:4], box2[:4])

            if iou_score < val_thresh:
                continue
            else:
                cost_matix[i,j]=0

    row_ind, col_ind = linear_sum_assignment(cost_matix)
    fn = len(gt_boxes) - row_ind.shape[0]
    fp = len(pred_boxes) - col_ind.shape[0]
    tp=0
    for i, j in zip(row_ind, col_ind):
        if cost_matix[i,j]==0:
            tp+=1
        else:
            fp+=1
            fn+=1
            
    precision = tp / (tp + fp + 1e-6)
    recall =  tp / (tp + fn +1e-6)
    f1_score = 2*(precision*recall)/(precision+recall+1e-6)

    return tp, fp, fn, precision, recall, f1_score

def bb_intersection_over_union(A, B) -> float:
    xA = max(A[0], B[0])
    yA = max(A[1], B[1])
    xB = min(A[2], B[2])
    yB = min(A[3], B[3])

    # compute the area of intersection rectangle
    interArea = max(0, xB - xA) * max(0, yB - yA)

    if interArea == 0:
        return 0.0

    # compute the area of both the prediction and ground-truth rectangles
    boxAArea = (A[2] - A[0]) * (A[3] - A[1])
    boxBArea = (B[2] - B[0]) * (B[3] - B[1])

    iou = interArea / float(boxAArea + boxBArea - interArea)
    return iou

def find_matching_box(boxes_list, new_box, idx, target_iou):
    best_index = []
    for i in range(len(boxes_list)):
        box = boxes_list[i]

        iou = bb_intersection_over_union(box, new_box)
        if iou > target_iou:
            best_index.append(box)

    return best_index

def df2box(df):
    return np.array([df["x"],df["y"],df["w"]+df["x"],df["h"]+df["y"],df["frame"],df["classification"]]).T

def time_nms(box_df, step):
    start = box_df["frame"].min()
    end = box_df["frame"].max()
    nmsiou=0.3
    offset = 0
    outs = []
    boxlist = df2box(box_df)
    remove = []
    for t in np.arange(start, end, step):
        df = box_df
        if len(remove)>0:
            r = np.array(remove)        
            df = df.drop(df.index[r])

        box_df2 = df[(t <= df["frame"]) & (df["frame"]<= t+step+offset)]           
        orig_box = df2box(box_df2)
        box = orig_box
        order = len(orig_box)
        if len(orig_box)>0:
            for i in range(order):
                if i not in set(remove):
                    #try:
                    match_box = find_matching_box(box, orig_box[i], None, nmsiou)
                    outs.append(match_box)
                    #remove.extend(match_box)

    return outs

def threshold_opt(preds_df, video, thresh, nmax=25, nmin=15, timenms=False):
    run_th = thresh
    numbox = 0
    tries = 0
    while (numbox>nmax or numbox<nmin) and tries<=20:
        positive_df = preds_df[(preds_df["classification"]>run_th)&(preds_df["video"]==video)]
        if timenms:
            # nms!
            outs = time_nms(positive_df, 30)
            output = []
            for out in outs:
                bscore = 0
                for o in out:
                    if bscore < o[5]:
                        best = o
                        bscore = o[5]
                output.append(best)
            pred = np.array(output)[:,:5]
        else:
            pred = positive_df[positive_df["video"]==video][['x', 'y', 'w', 'h', 'frame']].values
            pred[:,2] = pred[:,2] + pred[:,0]
            pred[:,3] = pred[:,3] + pred[:,1]
        numbox = len(pred)
        if numbox>nmax:
            run_th += 0.02
        elif numbox<nmin:
            run_th -= 0.02
        tries += 1
    return pred
    
for thresh in np.arange(0.3,0.9,0.1):
    print("classification thr:", thresh)
    tps = []; fps = []; fns = [];
    tps1 = []; fps1 = []; fns1 = [];
    tps2 = []; fps2 = []; fns2 = [];
    for video in val_videos:
        #print(video)
        pred = threshold_opt(preds_df, video, thresh)
        
        #pred = positive_df[positive_df["video"]==video][['x', 'y', 'w', 'h', 'frame']].values
        #pred[:,2] = pred[:,2] + pred[:,0]
        #pred[:,3] = pred[:,3] + pred[:,1]
        #break

        gt = gt_df[gt_df["video"]==video][['x', 'y', 'w', 'h', 'frame']].values
        gt[:,2] = gt[:,0] + gt[:,2]
        gt[:,3] = gt[:,1] + gt[:,3]
        #print("pred len:{}, gt len:{}".format(len(pred), len(gt)))

        tp, fp, fn, precision, recall, f1_score = precision_calc(gt, pred)
        #print("tp:{}, fp:{}, fn:{}, precision:{}, recall:{}, f1_score:{}".format(tp, fp, fn, precision, recall, f1_score)) 
        tps.append(tp)
        fns.append(fn)
        fps.append(fp)
        if "Side" in video:
            tps2.append(tp)
            fns2.append(fn)
            fps2.append(fp)
        else:
            tps1.append(tp)
            fns1.append(fn)
            fps1.append(fp)

    # calculate final metrics
    tps = np.sum(tps)
    fps = np.sum(fps)
    fns = np.sum(fns)
    precision = tps / (tps + fps + 1e-6)
    recall =  tps / (tps + fns +1e-6)
    f1_score = 2*(precision*recall)/(precision+recall+1e-6)
    precision1,recall1,f1_score1 = calc_metric(tps1, fps1,fns1) 
    precision2,recall2,f1_score2 = calc_metric(tps2, fps2,fns2) 
    result = {"P1":precision1, "R1":recall1, "F1_1":f1_score1,
                                  "P2":precision2, "R2":recall2, "F1_2":f1_score2,}
    
    print("*"*30)
    print("precision: {}, recall:{}, f1:{}".format(precision, recall, f1_score))
    print("tps:{}, fps:{}, fns:{}".format(tps, fps, fns))
    print(result)
    print("*"*30)
pred
positive_df = preds_df[preds_df["classification"]>0.4]
positive_df.reset_index(inplace=True)
positive_df[:20]
positive_df.to_csv("class.csv")
val_dataset = HeadClassificationDataset(positive_df, image_dir=IMAGE_DIR, img_size=IMG_SIZE, transforms=get_valid_transforms(IMG_SIZE))
val_loader = torchdata.DataLoader(val_dataset, batch_size=BS, shuffle=False, num_workers=16)

cnt = 0
for i in val_loader:
    cnt += 1
    plt.imshow(i[0].numpy().transpose([1,2,0])[:,:,12:15])
    plt.show()
    if cnt == 10:
        break<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14460582808075273, 'fraction_numerical': 0.03288572211564812, 'mean_word_length': 2.516864760681015, 'pattern_counts': {'":': 8, '<': 24, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '514826', 'license': 'MIT', 'n_tokens_mistral': 4252, 'n_tokens_neox': 4099, 'n_words': 850, 'path': '2nd-stage-Inference.ipynb', 'repo_name': 'kentaroy47/kaggle-nflimpact-13thplace', 'size': '10947'}