<jupyter_start><jupyter_text>![](https://images.unsplash.com/photo-1602084551218-a28205125639?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=2070&q=80) <div class = 'alert alert-block alert-info'     style = 'background-color:4c1c84;              color:eeebf1;              border-width:5px;              border-color:4c1c84;              font-family:Comic Sans MS;              border-radius: 50px 50px'>    Ensemble    <a href = "Config"       style = "color:eeebf1;                font-size:14px">1.Config    <a href = "Settings"       style = "color:eeebf1;                font-size:14px">2.Settings    <a href = "Data-Load"       style = "color:eeebf1;                font-size:14px">3.Data Load    <a href = "Pytorch-Settings"       style = "color:eeebf1;                font-size:14px">4.Pytorch Settings    <a href = "Training"       style = "color:eeebf1;                font-size:14px">5.Training<p style = 'font-size:24px;            color:4c1c84'>    実施したこと    <li style = "color:4c1c84;                font-size:14px">Ensemble     Config<jupyter_code>import sys
sys.path.append("../src/utils/iterative-stratification/")
sys.path.append("../src/utils/detoxify")
sys.path.append("../src/utils/coral-pytorch/")
sys.path.append("../src/utils/pyspellchecker")
!pip install --no-index --find-links ../src/utils/faiss/ faiss-gpu==1.6.3
import warnings
warnings.simplefilter('ignore')

import os
import gc
gc.enable()
import sys
import glob
import copy
import math
import time
import random
import string
import psutil
import pathlib
from pathlib import Path
from contextlib import contextmanager
from collections import defaultdict
from box import Box
from typing import Optional
from pprint import pprint

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import japanize_matplotlib

from tqdm.auto import tqdm as tqdmp
from tqdm.autonotebook import tqdm as tqdm
tqdmp.pandas()

## Model
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import StratifiedKFold, KFold, GroupKFold
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader
from transformers import AutoTokenizer, AutoModel, AdamW, AutoModelForSequenceClassification
from transformers import RobertaModel, RobertaForSequenceClassification
from transformers import RobertaTokenizer
from transformers import LukeTokenizer, LukeModel, LukeConfig
from transformers import get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup
from transformers import BertTokenizer, BertForSequenceClassification, BertForMaskedLM
from transformers import RobertaTokenizer, RobertaForSequenceClassification
from transformers import XLMRobertaTokenizer, XLMRobertaForSequenceClassification
from transformers import DebertaTokenizer, DebertaModel
from transformers import DistilBertTokenizer, DistilBertModel, DistilBertForSequenceClassification
from transformers import AlbertTokenizer, AlbertModel

# Pytorch Lightning
import pytorch_lightning as pl
from pytorch_lightning.utilities.seed import seed_everything
from pytorch_lightning import callbacks
from pytorch_lightning.callbacks.progress import ProgressBarBase
from pytorch_lightning import LightningDataModule, LightningDataModule
from pytorch_lightning import Trainer
from pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping, LearningRateMonitor
from pytorch_lightning.loggers import WandbLogger
from pytorch_lightning.loggers.csv_logs import CSVLogger
from pytorch_lightning.callbacks import RichProgressBar

from sklearn.linear_model import Ridge
from sklearn.svm import SVC, SVR
from sklearn.feature_extraction.text import TfidfVectorizer
from scipy.stats import rankdata
from sklearn.preprocessing import StandardScaler, MinMaxScaler
from cuml.svm import SVR as cuml_SVR
from cuml.linear_model import Ridge as cuml_Ridge
import cudf
from detoxify import Detoxify
from iterstrat.ml_stratifiers import MultilabelStratifiedKFold

from ast import literal_eval
from nltk.tokenize import TweetTokenizer
import spacy
from scipy.stats import sem
from copy import deepcopy
from spellchecker import SpellChecker
from typing import Text, Set, List

import faiss
import cudf, cuml, cupy
from cuml.feature_extraction.text import TfidfVectorizer as cuTfidfVectorizer
from cuml.neighbors import NearestNeighbors as cuNearestNeighbors
from cuml.decomposition.tsvd import TruncatedSVD as cuTruncatedSVD
import torch

config = {
    "exp_comment":"Psuedo Labeling",
    "seed": 42,
    "root": "/content/drive/MyDrive/kaggle/Jigsaw/raw",
    "n_fold": 10,
    "epoch": 5,
    "max_length": 256,
    "environment": "AWS",
    "project": "Jigsaw",
    "entity": "dataskywalker",
    "exp_name": "Ensemble",
    "margin": 0.5,
    "train_fold": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],

    "trainer": {
        "gpus": 1,
        "accumulate_grad_batches": 8,
        "progress_bar_refresh_rate": 1,
        "fast_dev_run": True,
        "num_sanity_val_steps": 0,
    },

    "train_loader": {
        "batch_size": 8,
        "shuffle": True,
        "num_workers": 1,
        "pin_memory": True,
        "drop_last": True,
    },

    "valid_loader": {
        "batch_size": 2,
        "shuffle": False,
        "num_workers": 1,
        "pin_memory": True,
        "drop_last": False,
    },

    "test_loader": {
        "batch_size": 2,
        "shuffle": False,
        "num_workers": 1,
        "pin_memory": True,
        "drop_last": False,
    },

    "backbone": {
        "name": "albert-base-v2",
        "output_dim": 1,
    },

    "optimizer": {
        "name": "torch.optim.AdamW",
        "params": {
            "lr": 1e-6,
        },
    },

    "scheduler": {
        "name": "torch.optim.lr_scheduler.CosineAnnealingWarmRestarts",
        "params": {
            "T_0": 20,
            "eta_min": 0,
        },
    },

    "loss": "nn.MarginRankingLoss",
}

config = Box(config)
# pprint(config)
# 個人的にAWSやKaggle環境やGoogle Colabを行ったり来たりしているのでまとめています
import os
import sys
from pathlib import Path

if config.environment == 'AWS':
    
    INPUT_DIR = Path('/mnt/work/data/kaggle/Jigsaw/')
    MODEL_DIR = Path(f'../models/{config.exp_name}/')
    OUTPUT_DIR = Path(f'../data/interim/{config.exp_name}/')
    UTIL_DIR = Path('/mnt/work/shimizu/kaggle/PetFinder/src/utils')
    
    os.makedirs(MODEL_DIR, exist_ok=True)
    os.makedirs(OUTPUT_DIR, exist_ok=True)
    
    print(f"Your environment is 'AWS'.\nINPUT_DIR is {INPUT_DIR}\nMODEL_DIR is {MODEL_DIR}\nOUTPUT_DIR is {OUTPUT_DIR}\nUTIL_DIR is {UTIL_DIR}")
    
    
elif config.environment == 'Kaggle':
    INPUT_DIR = Path('../input/*****')
    MODEL_DIR = Path('./')
    OUTPUT_DIR = Path('./')
    print(f"Your environment is 'Kaggle'.\nINPUT_DIR is {INPUT_DIR}\nMODEL_DIR is {MODEL_DIR}\nOUTPUT_DIR is {OUTPUT_DIR}")

    
elif config.environment == 'Colab':
    INPUT_DIR = Path('/content/drive/MyDrive/kaggle/Jigsaw/raw')
    BASE_DIR = Path("/content/drive/MyDrive/kaggle/Jigsaw/interim")

    MODEL_DIR = BASE_DIR / f'{config.exp_name}'
    OUTPUT_DIR = BASE_DIR / f'{config.exp_name}/'

    os.makedirs(MODEL_DIR, exist_ok=True)
    os.makedirs(OUTPUT_DIR, exist_ok=True)
    
    if not os.path.exists(INPUT_DIR):
        print('Please Mount your Google Drive.')
    else:
        print(f"Your environment is 'Colab'.\nINPUT_DIR is {INPUT_DIR}\nMODEL_DIR is {MODEL_DIR}\nOUTPUT_DIR is {OUTPUT_DIR}")
        
else:
    print("Please choose 'AWS' or 'Kaggle' or 'Colab'.\nINPUT_DIR is not found.")
# Seed固定
seed_everything(config.seed)
## 処理時間計測
@contextmanager
def timer(name:str, slack:bool=False):
    t0 = time.time()
    p = psutil.Process(os.getpid())
    m0 = p.memory_info()[0] / 2. ** 30
    print(f'<< {name} >> Start')
    yield
    
    m1 = p.memory_info()[0] / 2. ** 30
    delta = m1 - m0
    sign = '+' if delta >= 0 else '-'
    delta = math.fabs(delta)
    
    print(f"<< {name} >> {m1:.1f}GB({sign}{delta:.1f}GB):{time.time() - t0:.1f}sec", file=sys.stderr)<jupyter_output><empty_output><jupyter_text>Data Load<jupyter_code>## Data Check
for dirnames, _, filenames in os.walk(INPUT_DIR):
    
    for filename in filenames:

        print(f'{dirnames}/{filename}')
val_df = pd.read_csv("/mnt/work/data/kaggle/Jigsaw/validation_data.csv")
test_df = pd.read_csv("/mnt/work/data/kaggle/Jigsaw/comments_to_score.csv")

display(val_df.head())
display(test_df.head())
with timer("Count less text & more text"):
    less_df = val_df.groupby(["less_toxic"])["worker"].agg("count").reset_index()
    less_df.columns = ["text", "less_count"]

    more_df = val_df.groupby(["more_toxic"])["worker"].agg("count").reset_index()
    more_df.columns = ["text", "more_count"]
    
    text_df = pd.merge(
        less_df,
        more_df,
        on="text",
        how="outer"
    )

    text_df["less_count"] = text_df["less_count"].fillna(0)
    text_df["more_count"] = text_df["more_count"].fillna(0)
    
    text_df["target"] = text_df["more_count"]/(text_df["less_count"] + text_df["more_count"])
    
    display(text_df)<jupyter_output><< Count less text & more text >> Start<jupyter_text>OOF LOAD<jupyter_code>eval_df = val_df.copy()
rank_df = val_df.copy()
# exp_list = ["038_exp", "040_exp", "ohashi_040_exp"]
exp_list = [
#     "038_exp",
    "040_exp", 
    "041_exp", 
#     "047_exp",
#     "048_exp",
    "049_exp",
#     "051_exp",
    "052_exp",
#     "053_exp",

#     "035_exp_DeBERTa_Base__Jigsaw_Unbiased",
#     "035_exp_HATE_BERT__Jigsaw_classification",
#     "035_exp_HATE_BERT__Jigsaw_Unbiased",
#     "035_exp_Luke_Base__Wiki_Attack",
#     "035_exp_RoBERTa_Base__Jigsaw_Unbiased",
#     "035_exp_TFIDF_Ridge__Jigsaw_classification",
#     "035_exp_TFIDF_Ridge__Ruddit",
]

ohashi_list = ["ohashi_040_exp"]
thajime_list = ["thajime_0206_exp"]

exp_list += ohashi_list
exp_list += thajime_list
OOF_DF = pd.DataFrame()
OOF_DF["text"] = text_df["text"]

for exp in tqdm(exp_list, total=len(exp_list)):
    
    if ("ohashi" not in exp) and ("thajime" not in exp):
        
        oof = pd.read_pickle(f"../data/interim/{exp[:7]}/{exp}__text_df.pkl")
        oof.columns = ["text", f"{exp}__pred"]

        sc = StandardScaler()
        oof[f"{exp}__pred"] = sc.fit_transform(oof[f"{exp}__pred"].values.reshape(-1, 1))[:, 0] 
        
#         mm = MinMaxScaler()
#         oof[f"{exp}__pred"] = mm.fit_transform(oof[f"{exp}__pred"].values.reshape(-1, 1))[:, 0] 

        OOF_DF = pd.merge(
            OOF_DF,
            oof,
            on="text",
            how="left"
        )
        OOF_DF[f"{exp}__Rank"] = rankdata(OOF_DF[f"{exp}__pred"], method="ordinal")
        
    else:
        
        oof = pd.read_pickle(f"../data/interim/{exp}/{exp}__text_df.pkl")
        oof.columns = ["text", f"{exp}__pred"]
        
        sc = StandardScaler()
        oof[f"{exp}__pred"] = sc.fit_transform(oof[f"{exp}__pred"].values.reshape(-1, 1))[:, 0] 
        
#         mm = MinMaxScaler()
#         oof[f"{exp}__pred"] = mm.fit_transform(oof[f"{exp}__pred"].values.reshape(-1, 1))[:, 0] 

        OOF_DF = pd.merge(
            OOF_DF,
            oof,
            on="text",
            how="left"
        )
        OOF_DF[f"{exp}__Rank"] = rankdata(OOF_DF[f"{exp}__pred"], method="ordinal")
    
OOF_DF.head()
# plt.figure(figsize=(20, 5))

# plt.subplot(1, 2, 1)
# sns.scatterplot(OOF_DF[f"{exp_list[0]}__pred"], OOF_DF[f"{exp_list[1]}__pred"])
# plt.grid()
# plt.title("Score")

# plt.subplot(1, 2, 2)
# sns.scatterplot(OOF_DF[f"{exp_list[0]}__Rank__pred"], OOF_DF[f"{exp_list[-1]}__Rank__pred"])
# plt.grid()
# plt.title("Rank")

# plt.show()
## Mean Average
pred_cols = [col for col in OOF_DF.columns.tolist() if "__pred" in col]
rank_cols = [col for col in OOF_DF.columns.tolist() if "__Rank" in col]

print(len(pred_cols))
print(len(rank_cols))

OOF_DF["mean__pred"] = OOF_DF[pred_cols].mean(axis=1)
OOF_DF["mean__rank"] = OOF_DF[rank_cols].mean(axis=1)

display(OOF_DF.head())
def calculate_acc_bce(eval_df:pd.DataFrame, OOF_DF:pd.DataFrame, exp_list:list):
    
    print("| Exp Name | Accuracy | BCE Loss |")
    print("|:--------:|:--------:|:--------:|")
    
    __eval_df = eval_df.copy()
    exp_list += ["mean"]
    
    for exp in exp_list:
        
        __eval_df = pd.merge(
            __eval_df,
            OOF_DF[["text", f"{exp}__pred"]],
            left_on = "less_toxic",
            right_on = "text",
            how = "left"
        )
        __eval_df = __eval_df.rename(columns={
            f"{exp}__pred": f"less__{exp}__pred"
        })
        __eval_df = __eval_df.drop("text", axis=1)

        __eval_df = pd.merge(
            __eval_df,
            OOF_DF[["text", f"{exp}__pred"]],
            left_on = "more_toxic",
            right_on = "text",
            how = "left"
        )
        __eval_df = __eval_df.rename(columns={
            f"{exp}__pred": f"more__{exp}__pred"
        })
        __eval_df = __eval_df.drop("text", axis=1)
        
        
        __eval_df[f"Diff__{exp}__pred"] = __eval_df[f"more__{exp}__pred"] - __eval_df[f"less__{exp}__pred"]
        tmp_acc = __eval_df[__eval_df[f"Diff__{exp}__pred"]>0][f"Diff__{exp}__pred"].count()/len(eval_df)

        logits = torch.tensor(__eval_df[f"Diff__{exp}__pred"].values, dtype=torch.float)
        target = torch.tensor(np.zeros(len(__eval_df))+1., dtype=torch.float)
        tmp_logloss = nn.BCEWithLogitsLoss()(logits, target).numpy()
        
        print(f"| {exp} | {tmp_acc:.5f} | {tmp_logloss:.5f} |")    
calculate_acc_bce(eval_df, OOF_DF, exp_list)
OOF_DF
OOF_DF.to_pickle(OUTPUT_DIR/"oof_text_df.pkl")
OOF_DF.to_csv(OUTPUT_DIR/"oof_text_df.csv", index=False)
plt.figure(figsize=(20, 5))

plt.subplot(1, 2, 1)
sns.scatterplot(OOF_DF["040_exp__pred"], OOF_DF["052_exp__pred"])
plt.grid()
plt.title("Score")

plt.subplot(1, 2, 2)
sns.scatterplot(OOF_DF["040_exp__Rank"], OOF_DF["052_exp__Rank"])
plt.grid()
plt.title("Rank")

plt.show()<jupyter_output><empty_output><jupyter_text>Rank Distribution<jupyter_code>rank_cols = [col for col in OOF_DF.columns.tolist() if "__Rank" in col]
print(len(rank_cols))
RANK_DF = OOF_DF[["text"]+rank_cols].copy()
RANK_DF["std"] = RANK_DF[rank_cols].std(axis=1)
RANK_DF["mean"] = RANK_DF[rank_cols].mean(axis=1)

plt.figure(figsize=(15, 3))
plt.hist(RANK_DF["std"], bins=30)
plt.grid()
plt.show()
RANK_DF.sort_values("std", ascending=False).head(20)
RANK_DF.sort_values("052_exp__Rank").head(2)
num = 8975
display(RANK_DF.loc[num, "text"])
display(RANK_DF.loc[num, rank_cols+["mean"]])
tmp_df = val_df[val_df["more_toxic"]==RANK_DF.loc[num, "text"]]
tmp_df = pd.concat(
    [
        tmp_df,
        val_df[val_df["less_toxic"]==RANK_DF.loc[num, "text"]]
    ],
    axis=0
)
tmp_df
val_df.loc[28948, "more_toxic"]
tmp_df = val_df[val_df["more_toxic"]==val_df.loc[28948, "more_toxic"]]
tmp_df = pd.concat(
    [
        tmp_df,
        val_df[val_df["less_toxic"]==val_df.loc[28948, "more_toxic"]]
    ],
    axis=0
)
tmp_df<jupyter_output><empty_output><jupyter_text>Each Ensemble<jupyter_code>eval_df = val_df.copy()
rank_df = val_df.copy()
# exp_list = ["038_exp", "040_exp", "ohashi_040_exp"]
exp_list = [
#     "038_exp",
    "040_exp", 
    "041_exp", 
#     "047_exp",
#     "048_exp",
    "049_exp",
#     "051_exp",
    "052_exp",
#     "053_exp",

#     "035_exp_DeBERTa_Base__Jigsaw_Unbiased",
#     "035_exp_HATE_BERT__Jigsaw_classification",
#     "035_exp_HATE_BERT__Jigsaw_Unbiased",
#     "035_exp_Luke_Base__Wiki_Attack",
#     "035_exp_RoBERTa_Base__Jigsaw_Unbiased",
#     "035_exp_TFIDF_Ridge__Jigsaw_classification",
#     "035_exp_TFIDF_Ridge__Ruddit",
]

ohashi_list = ["ohashi_040_exp"]
thajime_list = ["thajime_0206_exp"]

# exp_list += ohashi_list
# exp_list += thajime_list
OOF_DF = pd.DataFrame()
OOF_DF["text"] = text_df["text"]

for exp in tqdm(exp_list, total=len(exp_list)):
    
    oof = pd.read_pickle(f"../data/interim/{exp}/{exp}__text_df.pkl")
    oof.columns = ["text", f"{exp}__pred"]
    sc = StandardScaler()
    oof[f"{exp}__pred"] = sc.fit_transform(oof[f"{exp}__pred"].values.reshape(-1, 1))[:, 0] 
    
    OOF_DF = pd.merge(
        OOF_DF,
        oof,
        on="text",
        how="left"
    )
    OOF_DF[f"{exp}__Rank__pred"] = rankdata(OOF_DF[f"{exp}__pred"], method="ordinal")
## Mean Average
pred_cols = [col for col in OOF_DF.columns.tolist() if "exp__pred" in col]
rank_cols = [col for col in OOF_DF.columns.tolist() if "exp__Rank__pred" in col]

OOF_DF["mean_pred"] = OOF_DF[pred_cols].mean(axis=1)
OOF_DF["mean_rank"] = OOF_DF[rank_cols].mean(axis=1)

display(OOF_DF.head())
OHASHI_DF = pd.DataFrame()
OHASHI_DF["text"] = text_df["text"]

for exp in tqdm(ohashi_list, total=len(ohashi_list)):
    
    oof = pd.read_pickle(f"../data/interim/{exp}/{exp}__text_df.pkl")
    oof.columns = ["text", f"{exp}__pred"]
    sc = StandardScaler()
    oof[f"{exp}__pred"] = sc.fit_transform(oof[f"{exp}__pred"].values.reshape(-1, 1))[:, 0] 
    
    OHASHI_DF = pd.merge(
        OHASHI_DF,
        oof,
        on="text",
        how="left"
    )
    OHASHI_DF[f"{exp}__Rank__pred"] = rankdata(OHASHI_DF[f"{exp}__pred"], method="ordinal")
## Mean Average
pred_cols = [col for col in OHASHI_DF.columns.tolist() if "exp__pred" in col]
rank_cols = [col for col in OHASHI_DF.columns.tolist() if "exp__Rank__pred" in col]

OHASHI_DF["ohashi_mean_pred"] = OHASHI_DF[pred_cols].mean(axis=1)
OHASHI_DF["ohashi_mean_rank"] = OHASHI_DF[rank_cols].mean(axis=1)

display(OHASHI_DF.head())
TAMURA_DF = pd.DataFrame()
TAMURA_DF["text"] = text_df["text"]

for exp in tqdm(thajime_list, total=len(thajime_list)):
    
    oof = pd.read_pickle(f"../data/interim/{exp}/{exp}__text_df.pkl")
    oof.columns = ["text", f"{exp}__pred"]
    sc = StandardScaler()
    oof[f"{exp}__pred"] = sc.fit_transform(oof[f"{exp}__pred"].values.reshape(-1, 1))[:, 0] 
    
    TAMURA_DF = pd.merge(
        TAMURA_DF,
        oof,
        on="text",
        how="left"
    )
    TAMURA_DF[f"{exp}__Rank__pred"] = rankdata(TAMURA_DF[f"{exp}__pred"], method="ordinal")
## Mean Average
pred_cols = [col for col in TAMURA_DF.columns.tolist() if "exp__pred" in col]
rank_cols = [col for col in TAMURA_DF.columns.tolist() if "exp__Rank__pred" in col]

TAMURA_DF["thajime_mean_pred"] = TAMURA_DF[pred_cols].mean(axis=1)
TAMURA_DF["thajime__mean_rank"] = TAMURA_DF[rank_cols].mean(axis=1)

display(TAMURA_DF.head())
OOF_DF["each_mean_pred"] = (OHASHI_DF["ohashi_mean_pred"] + TAMURA_DF["thajime_mean_pred"]+ OOF_DF["mean_pred"])/3.
OOF_DF["each_mean_rank"] = (OHASHI_DF["ohashi_mean_rank"] + TAMURA_DF["thajime__mean_rank"]+ OOF_DF["mean_rank"])/3.
rank_df = pd.merge(
    rank_df,
    OOF_DF[["text", "each_mean_rank"]],
    left_on = "less_toxic",
    right_on = "text",
    how = "left"
)
rank_df = rank_df.rename(columns={
    f"each_mean_rank": f"less__mean__rank"
})
rank_df = rank_df.drop("text", axis=1)

rank_df = pd.merge(
    rank_df,
    OOF_DF[["text", "each_mean_rank"]],
    left_on = "more_toxic",
    right_on = "text",
    how = "left"
)
rank_df = rank_df.rename(columns={
    f"each_mean_rank": f"more__mean__rank"
})
rank_df = rank_df.drop("text", axis=1)

eval_df = pd.merge(
    eval_df,
    OOF_DF[["text", "each_mean_pred"]],
    left_on = "less_toxic",
    right_on = "text",
    how = "left"
)
eval_df = eval_df.rename(columns={
    f"each_mean_pred": f"less__mean__pred"
})
eval_df = eval_df.drop("text", axis=1)

eval_df = pd.merge(
    eval_df,
    OOF_DF[["text", "each_mean_pred"]],
    left_on = "more_toxic",
    right_on = "text",
    how = "left"
)
eval_df = eval_df.rename(columns={
    f"each_mean_pred": f"more__mean__pred"
})
eval_df = eval_df.drop("text", axis=1)
display(eval_df.head())
display(rank_df.head())
eval_df[f"Diff__mean_pred"] = eval_df["more__mean__pred"] - eval_df["less__mean__pred"]
tmp_score = eval_df[eval_df[f"Diff__mean_pred"]>0][f"Diff__mean_pred"].count()/len(eval_df)
print(f"Mean Pred Score: {tmp_score:.6f}")
rank_df[f"Diff__mean_rank"] = rank_df["more__mean__rank"] - rank_df["less__mean__rank"]
tmp_score = rank_df[rank_df[f"Diff__mean_rank"]>0][f"Diff__mean_rank"].count()/len(rank_df)
print(f"Mean Rank Score: {tmp_score:.6f}")<jupyter_output>Mean Rank Score: 0.725455
{'fraction_non_alphanumeric': 0.15094059405940594, 'fraction_numerical': 0.019702970297029703, 'mean_word_length': 3.809761904761905, 'pattern_counts': {'":': 55, '<': 33, '<?xml version=': 0, '>': 37, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '280112', 'license': 'RSA-MD', 'n_tokens_mistral': 8264, 'n_tokens_neox': 7891, 'n_words': 1284, 'path': 'notebooks/***_ensemble.ipynb', 'repo_name': 'skywalker0415/jigsaw', 'size': '20200'}