<jupyter_start><jupyter_text>![](https://images.unsplash.com/photo-1602084551218-a28205125639?ixlib=rb-1.2.1&ixid=MnwxMjA3fDB8MHxwaG90by1wYWdlfHx8fGVufDB8fHx8&auto=format&fit=crop&w=2070&q=80) <div class = 'alert alert-block alert-info'     style = 'background-color:4c1c84;              color:eeebf1;              border-width:5px;              border-color:4c1c84;              font-family:Comic Sans MS;              border-radius: 50px 50px'>    Exp 020    <a href = "Config"       style = "color:eeebf1;                font-size:14px">1.Config    <a href = "Settings"       style = "color:eeebf1;                font-size:14px">2.Settings    <a href = "Data-Load"       style = "color:eeebf1;                font-size:14px">3.Data Load    <a href = "Pytorch-Settings"       style = "color:eeebf1;                font-size:14px">4.Pytorch Settings    <a href = "Training"       style = "color:eeebf1;                font-size:14px">5.Training<p style = 'font-size:24px;            color:4c1c84'>    実施したこと    <li style = "color:4c1c84;                font-size:14px">過去コンペの予測ラベル    <li style = "color:4c1c84;                font-size:14px">TFIDF    <li style = "color:4c1c84;                font-size:14px">SVR     Config<jupyter_code>import sys
sys.path.append("../src/utils/iterative-stratification/")
import warnings
warnings.simplefilter('ignore')

import os
import gc
gc.enable()
import sys
import glob
import copy
import math
import time
import random
import string
import psutil
import pathlib
from pathlib import Path
from contextlib import contextmanager
from collections import defaultdict
from box import Box
from typing import Optional
from pprint import pprint

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import japanize_matplotlib

from tqdm.auto import tqdm as tqdmp
from tqdm.autonotebook import tqdm as tqdm
tqdmp.pandas()

## Model
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import StratifiedKFold, KFold
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader
from transformers import AutoTokenizer, AutoModel, AdamW
from transformers import RobertaModel, RobertaForSequenceClassification
from transformers import RobertaTokenizer
from transformers import LukeTokenizer, LukeModel, LukeConfig
from transformers import get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup

# Pytorch Lightning
import pytorch_lightning as pl
from pytorch_lightning.utilities.seed import seed_everything
from pytorch_lightning import callbacks
from pytorch_lightning.callbacks.progress import ProgressBarBase
from pytorch_lightning import LightningDataModule, LightningDataModule
from pytorch_lightning import Trainer
from pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping, LearningRateMonitor
from pytorch_lightning.loggers import WandbLogger
from pytorch_lightning.loggers.csv_logs import CSVLogger
from pytorch_lightning.callbacks import RichProgressBar

from sklearn.linear_model import Ridge
from sklearn.svm import SVC, SVR
from sklearn.feature_extraction.text import TfidfVectorizer
from scipy.stats import rankdata
from cuml.svm import SVR
from cuml.linear_model import Ridge as cuml_Ridge
import cudf
import torch

config = {
    "seed": 42,
    "root": "/content/drive/MyDrive/kaggle/Jigsaw/raw",
    "n_fold": 5,
    "epoch": 5,
    "max_length": 128,
    "environment": "AWS",
    "project": "Jigsaw",
    "entity": "dataskywalker",
    "exp_name": "020_exp",
    "margin": 0.5,
    "train_fold": [0, 1, 2, 3, 4],

    "trainer": {
        "gpus": 1,
        "accumulate_grad_batches": 8,
        "progress_bar_refresh_rate": 1,
        "fast_dev_run": False,
        "num_sanity_val_steps": 0,
    },

    "train_loader": {
        "batch_size": 32,
        "shuffle": True,
        "num_workers": 1,
        "pin_memory": True,
        "drop_last": True,
    },

    "valid_loader": {
        "batch_size": 64,
        "shuffle": False,
        "num_workers": 1,
        "pin_memory": True,
        "drop_last": False,
    },

    "test_loader": {
        "batch_size": 32,
        "shuffle": False,
        "num_workers": 1,
        "pin_memory": True,
        "drop_last": False,
    },

    "backbone": {
        "name": "studio-ousia/luke-base",
        "output_dim": 1,
    },

    "optimizer": {
        "name": "torch.optim.AdamW",
        "params": {
            "lr": 1e-6,
        },
    },

    "scheduler": {
        "name": "torch.optim.lr_scheduler.CosineAnnealingWarmRestarts",
        "params": {
            "T_0": 20,
            "eta_min": 0,
        },
    },

    "loss": "nn.MarginRankingLoss",
}

config = Box(config)
pprint(config)
# 個人的にAWSやKaggle環境やGoogle Colabを行ったり来たりしているのでまとめています
import os
import sys
from pathlib import Path

if config.environment == 'AWS':
    
    INPUT_DIR = Path('/mnt/work/data/kaggle/Jigsaw/')
    MODEL_DIR = Path(f'../models/{config.exp_name}/')
    OUTPUT_DIR = Path(f'../data/interim/{config.exp_name}/')
    UTIL_DIR = Path('/mnt/work/shimizu/kaggle/PetFinder/src/utils')
    
    os.makedirs(MODEL_DIR, exist_ok=True)
    os.makedirs(OUTPUT_DIR, exist_ok=True)
    
    print(f"Your environment is 'AWS'.\nINPUT_DIR is {INPUT_DIR}\nMODEL_DIR is {MODEL_DIR}\nOUTPUT_DIR is {OUTPUT_DIR}\nUTIL_DIR is {UTIL_DIR}")
    
    
elif config.environment == 'Kaggle':
    INPUT_DIR = Path('../input/*****')
    MODEL_DIR = Path('./')
    OUTPUT_DIR = Path('./')
    print(f"Your environment is 'Kaggle'.\nINPUT_DIR is {INPUT_DIR}\nMODEL_DIR is {MODEL_DIR}\nOUTPUT_DIR is {OUTPUT_DIR}")

    
elif config.environment == 'Colab':
    INPUT_DIR = Path('/content/drive/MyDrive/kaggle/Jigsaw/raw')
    BASE_DIR = Path("/content/drive/MyDrive/kaggle/Jigsaw/interim")

    MODEL_DIR = BASE_DIR / f'{config.exp_name}'
    OUTPUT_DIR = BASE_DIR / f'{config.exp_name}/'

    os.makedirs(MODEL_DIR, exist_ok=True)
    os.makedirs(OUTPUT_DIR, exist_ok=True)
    
    if not os.path.exists(INPUT_DIR):
        print('Please Mount your Google Drive.')
    else:
        print(f"Your environment is 'Colab'.\nINPUT_DIR is {INPUT_DIR}\nMODEL_DIR is {MODEL_DIR}\nOUTPUT_DIR is {OUTPUT_DIR}")
        
else:
    print("Please choose 'AWS' or 'Kaggle' or 'Colab'.\nINPUT_DIR is not found.")
# Seed固定
seed_everything(config.seed)
## 処理時間計測
@contextmanager
def timer(name:str, slack:bool=False):
    t0 = time.time()
    p = psutil.Process(os.getpid())
    m0 = p.memory_info()[0] / 2. ** 30
    print(f'<< {name} >> Start')
    yield
    
    m1 = p.memory_info()[0] / 2. ** 30
    delta = m1 - m0
    sign = '+' if delta >= 0 else '-'
    delta = math.fabs(delta)
    
    print(f"<< {name} >> {m1:.1f}GB({sign}{delta:.1f}GB):{time.time() - t0:.1f}sec", file=sys.stderr)<jupyter_output><empty_output><jupyter_text>Data Load<jupyter_code>## Data Check
for dirnames, _, filenames in os.walk(INPUT_DIR):
    
    for filename in filenames:

        print(f'{dirnames}/{filename}')
val_df = pd.read_csv("/mnt/work/data/kaggle/Jigsaw/validation_data.csv")
test_df = pd.read_csv("/mnt/work/data/kaggle/Jigsaw/comments_to_score.csv")

display(val_df.head())
display(test_df.head())<jupyter_output><empty_output><jupyter_text>Jigsaw 1st<jupyter_code>train_df = pd.read_csv("../data/external/jigsaw-classification/train.csv.zip")
print(train_df.shape)
display(train_df.head())
def sample_df(df:pd.DataFrame, frac=0.2):
    
    '''
    train_dfからtoxicとnon_toxicを抽出
    non_toxicの割合をfracで調整
    '''
    
    label_cols = ["toxic", "severe_toxic", "obscene", "threat", "insult", "identity_hate"]
    df["y"] = df[label_cols].sum(axis=1)
    df["y"] = df["y"]/df["y"].max()

    toxic_df = df[df["y"]>0].reset_index(drop=True)
    nontoxic_df = df[df["y"]==0].reset_index(drop=True)
    nontoxic_df = nontoxic_df.sample(frac=frac, random_state=config.seed)
    
    df = pd.concat([toxic_df, nontoxic_df], axis=0).sample(frac=1).reset_index(drop=True)
    
    return df
with timer("sampling df"):
    
    train_df = sample_df(train_df, frac=0.1)
    display(train_df.head())<jupyter_output><< sampling df >> Start<jupyter_text>Text Cleaning<jupyter_code>def clean(data, col):

    data[col] = data[col].str.lower()
    # Clean some punctutation
    data[col] = data[col].str.replace('\n', ' \n ')
    data[col] = data[col].str.replace(r'([a-zA-Z]+)([/!?.])([a-zA-Z]+)',r'\1 \2 \3')
    # Replace repeating characters more than 3 times to length of 3
    data[col] = data[col].str.replace(r'([*!?\'])\1\1{2,}',r'\1\1\1')    
    # Add space around repeating characters
    data[col] = data[col].str.replace(r'([*!?\']+)',r' \1 ')    
    # patterns with repeating characters 
    data[col] = data[col].str.replace(r'([a-zA-Z])\1{2,}\b',r'\1\1')
    data[col] = data[col].str.replace(r'([a-zA-Z])\1\1{2,}\B',r'\1\1\1')
    data[col] = data[col].str.replace(r'[ ]{2,}',' ').str.strip()
    
    # removing characters
    data[col] = data[col].str.replace(r"[^a-zA-Z0-9]",' ')    
    
    return data
with timer("sampling df"):
    
    train_df = clean(train_df, "comment_text")
    val_df = clean(val_df, "more_toxic")
    val_df = clean(val_df, "less_toxic")
    test_df = clean(test_df, "text")
    display(train_df.head())
    display(val_df.head())
    display(test_df.head())<jupyter_output><< sampling df >> Start<jupyter_text>TFIDF + Ridge<jupyter_code>## tfidf
vec = TfidfVectorizer(
    analyzer="char_wb",
    max_df=0.7, 
    min_df=1, 
    ngram_range=(2, 6)
)
X = vec.fit_transform(train_df["comment_text"])
print(X.shape)
ridge_model = Ridge(alpha=0.5)
ridge_model.fit(X, train_df["y"].values)
less_vec = vec.transform(val_df["less_toxic"])
more_vec = vec.transform(val_df["more_toxic"])

less_pred = ridge_model.predict(less_vec)
more_pred = ridge_model.predict(more_vec)

test_vec = vec.transform(test_df["text"])
test_pred = ridge_model.predict(test_vec)
OOF_DF = pd.DataFrame()
OOF_DF["Ridge_less_pred"] = less_pred
OOF_DF["Ridge_more_pred"] = more_pred
OOF_DF["Ridge_diff"] = OOF_DF["Ridge_more_pred"] - OOF_DF["Ridge_less_pred"]

ridge_score = OOF_DF[OOF_DF["Ridge_diff"]>0]["Ridge_diff"].count()/len(OOF_DF)
print(f"TFIDF + Ridge Model Score: {ridge_score:.6f}")
test_df["jigsaw1st_tfidf_ridge"] = test_pred<jupyter_output><empty_output><jupyter_text>Ruddit + TFIDF + Ridge<jupyter_code>ruddit_df = pd.read_csv("../data/external/Ruddit/Dataset/ruddit_with_text.csv")
print(ruddit_df.shape)
display(ruddit_df.head())
ruddit_df = ruddit_df[ruddit_df["txt"]!="[deleted]"].reset_index(drop=True)
ruddit_df = clean(ruddit_df, "txt")
display(ruddit_df.head())
## tfidf
rud_vec = TfidfVectorizer(
    analyzer="char_wb",
    max_df=0.7, 
    min_df=3, 
    ngram_range=(3, 4)
)
X = rud_vec.fit_transform(ruddit_df["txt"])
rud_ridge_model = Ridge(alpha=0.5)
rud_ridge_model.fit(X, ruddit_df["offensiveness_score"].values)
less_vec = rud_vec.transform(val_df["less_toxic"])
more_vec = rud_vec.transform(val_df["more_toxic"])

less_pred = rud_ridge_model.predict(less_vec)
more_pred = rud_ridge_model.predict(more_vec)

test_vec = rud_vec.transform(test_df["text"])
test_pred = rud_ridge_model.predict(test_vec)
OOF_DF["Ruddit_Ridge_less_pred"] = less_pred
OOF_DF["Ruddit_Ridge_more_pred"] = more_pred
OOF_DF["Ruddit_Ridge_diff"] = OOF_DF["Ruddit_Ridge_more_pred"] - OOF_DF["Ruddit_Ridge_less_pred"]

ridge_score = OOF_DF[OOF_DF["Ruddit_Ridge_diff"]>0]["Ruddit_Ridge_diff"].count()/len(OOF_DF)
print(f"Ruddit + TFIDF + Ridge Model Score: {ridge_score:.6f}")
test_df["ruddit_tfidf_ridge"] = test_pred
plt.figure(figsize=(12, 4))
sns.distplot(test_df["jigsaw1st_tfidf_ridge"], label="jigsaw1st_tfidf_ridge")
sns.distplot(test_df["ruddit_tfidf_ridge"], label="ruddit_tfidf_ridge")
plt.legend()
plt.show()
plt.figure(figsize=(12, 4))
plt.scatter(test_df["jigsaw1st_tfidf_ridge"], test_df["ruddit_tfidf_ridge"])
plt.xlabel("jigsaw1st_tfidf_ridge")
plt.ylabel("ruddit_tfidf_ridge")
plt.grid()
plt.show()<jupyter_output><empty_output><jupyter_text>Detoxify <h2 style = "font-size:45px;             font-family:Comic Sans MS ;             font-weight : normal;              background-color: eeebf1 ;             color : 4c1c84;             text-align: center;             border-radius: 100px 100px;">    Original<jupyter_code>loaded = torch.load("../data/external/detoxify_ckpt/toxic_original-c1212f89.ckpt")
loaded["config"]["arch"]["args"]
from transformers import BertTokenizer, BertForSequenceClassification
from transformers import RobertaTokenizer, RobertaForSequenceClassification
from transformers import XLMRobertaTokenizer, XLMRobertaForSequenceClassification

bert_tokeninzer = BertTokenizer.from_pretrained('roberta-base')
bert_tokeninzer.save_pretrained(OUTPUT_DIR/f"roberta-base")
bert_model = BertForSequenceClassification.from_pretrained('roberta-base')
bert_model.save_pretrained(OUTPUT_DIR/'roberta-base')
sys.path.append("../src/utils/detoxify")

from detoxify import Detoxify

detox_model = Detoxify(
    'original', 
    checkpoint="../data/external/detoxify_ckpt/toxic_original-c1212f89.ckpt",
    device="cuda"
)
detox_model.predict(train_df["comment_text"].tolist()[0])
val_df['more_detoxify_score'] = val_df['more_toxic'].progress_map(lambda line: detox_model.predict(line))
val_df['less_detoxify_score'] = val_df['less_toxic'].progress_map(lambda line: detox_model.predict(line))

more_detoxify_df = val_df["more_detoxify_score"].apply(pd.Series)
less_detoxify_df = val_df["less_detoxify_score"].apply(pd.Series)
OOF_DF["more_detoxify_unbiased_score"] = more_detoxify_df[detoxify_cols].sum(axis=1)
OOF_DF["less_detoxify_unbiased_score"] = less_detoxify_df[detoxify_cols].sum(axis=1)
OOF_DF["detoxify_diff"] = OOF_DF["more_detoxify_score"] - OOF_DF["less_detoxify_score"]
detoxify_score = OOF_DF[OOF_DF["detoxify_diff"]>0]["detoxify_diff"].count()/len(OOF_DF)
print(f"Detoxify Model Score: {detoxify_score:.6f}")
OOF_DF.head()<jupyter_output><empty_output><jupyter_text><h2 style = "font-size:45px;             font-family:Comic Sans MS ;             font-weight : normal;              background-color: eeebf1 ;             color : 4c1c84;             text-align: center;             border-radius: 100px 100px;">    Unbiased<jupyter_code>loaded = torch.load("../data/external/detoxify_ckpt/toxic_debiased-c7548aa0.ckpt")
loaded["config"]["arch"]["args"]
from transformers import BertTokenizer, BertForSequenceClassification
from transformers import RobertaTokenizer, RobertaForSequenceClassification
from transformers import XLMRobertaTokenizer, XLMRobertaForSequenceClassification

bert_tokeninzer = RobertaTokenizer.from_pretrained('roberta-base')
bert_tokeninzer.save_pretrained(OUTPUT_DIR/f"roberta-base")
bert_model = RobertaForSequenceClassification.from_pretrained('roberta-base')
bert_model.save_pretrained(OUTPUT_DIR/'roberta-base')
detox_model = Detoxify(
    'unbiased', 
    checkpoint="../data/external/detoxify_ckpt/toxic_debiased-c7548aa0.ckpt",
    device="cuda"
)
detox_model.predict(train_df["comment_text"].tolist()[0])
val_df['more_detoxify_unbiased_score'] = val_df['more_toxic'].progress_map(lambda line: detox_model.predict(line))
val_df['less_detoxify_unbiased_score'] = val_df['less_toxic'].progress_map(lambda line: detox_model.predict(line))

more_detoxify_df = val_df["more_detoxify_unbiased_score"].apply(pd.Series)
less_detoxify_df = val_df["less_detoxify_unbiased_score"].apply(pd.Series)
detoxify_cols = more_detoxify_df.columns.tolist()


OOF_DF["more_detoxify_unbiased_score"] = more_detoxify_df[detoxify_cols].sum(axis=1)
OOF_DF["less_detoxify_unbiased_score"] = less_detoxify_df[detoxify_cols].sum(axis=1)
OOF_DF["detoxify_unbiased_diff"] = OOF_DF["more_detoxify_unbiased_score"] - OOF_DF["less_detoxify_unbiased_score"]
detoxify_score = OOF_DF[OOF_DF["detoxify_unbiased_diff"]>0]["detoxify_unbiased_diff"].count()/len(OOF_DF)
print(f"Detoxify Unbiased Model Score: {detoxify_score:.6f}")

# OOF_DF["more_detoxify_unbiased_score"] = more_detoxify_df[detoxify_cols].sum(axis=1)
# OOF_DF["less_detoxify_unbiased_score"] = less_detoxify_df[detoxify_cols].sum(axis=1)
OOF_DF["detoxify_unbiased_diff"] = OOF_DF["more_detoxify_unbiased_score"] - OOF_DF["less_detoxify_unbiased_score"]
detoxify_score = OOF_DF[OOF_DF["detoxify_unbiased_diff"]>0]["detoxify_unbiased_diff"].count()/len(OOF_DF)
print(f"Detoxify Unbiased Model Score: {detoxify_score:.6f}")<jupyter_output>Detoxify Unbiased Model Score: 0.694832<jupyter_text><h2 style = "font-size:45px;             font-family:Comic Sans MS ;             font-weight : normal;              background-color: eeebf1 ;             color : 4c1c84;             text-align: center;             border-radius: 100px 100px;">    Multilingual<jupyter_code>loaded = torch.load("../data/external/detoxify_ckpt/multilingual_debiased-0b549669.ckpt")
loaded["config"]["arch"]["args"]
from transformers import BertTokenizer, BertForSequenceClassification
from transformers import RobertaTokenizer, RobertaForSequenceClassification
from transformers import XLMRobertaTokenizer, XLMRobertaForSequenceClassification

bert_tokeninzer = XLMRobertaTokenizer.from_pretrained('xlm-roberta-base')
bert_tokeninzer.save_pretrained(OUTPUT_DIR/f"xlm-roberta-base")
bert_model = XLMRobertaForSequenceClassification.from_pretrained('xlm-roberta-base')
bert_model.save_pretrained(OUTPUT_DIR/'xlm-roberta-base')
detox_model = Detoxify(
    'multilingual', 
    checkpoint="../data/external/detoxify_ckpt/multilingual_debiased-0b549669.ckpt",
    device="cuda"
)
detox_model.predict(train_df["comment_text"].tolist()[0])
val_df['more_detoxify_multi_score'] = val_df['more_toxic'].progress_map(lambda line: detox_model.predict(line))
val_df['less_detoxify_multi_score'] = val_df['less_toxic'].progress_map(lambda line: detox_model.predict(line))

more_detoxify_df = val_df["more_detoxify_multi_score"].apply(pd.Series)
less_detoxify_df = val_df["less_detoxify_multi_score"].apply(pd.Series)
detoxify_cols = more_detoxify_df.columns.tolist()

OOF_DF["more_detoxify_multi_score"] = more_detoxify_df[detoxify_cols].sum(axis=1)
OOF_DF["less_detoxify_multi_score"] = less_detoxify_df[detoxify_cols].sum(axis=1)
OOF_DF["detoxify_multi_diff"] = OOF_DF["more_detoxify_multi_score"] - OOF_DF["less_detoxify_multi_score"]
detoxify_score = OOF_DF[OOF_DF["detoxify_multi_diff"]>0]["detoxify_multi_diff"].count()/len(OOF_DF)
print(f"Detoxify Multilingual Model Score: {detoxify_score:.6f}")<jupyter_output>Detoxify Multilingual Model Score: 0.691976<jupyter_text><h2 style = "font-size:45px;             font-family:Comic Sans MS ;             font-weight : normal;              background-color: eeebf1 ;             color : 4c1c84;             text-align: center;             border-radius: 100px 100px;">    Original-Small<jupyter_code>loaded = torch.load("../data/external/detoxify_ckpt/original-albert-0e1d6498.ckpt")
loaded["config"]["arch"]["args"]
from transformers import BertTokenizer, BertForSequenceClassification
from transformers import RobertaTokenizer, RobertaForSequenceClassification
from transformers import XLMRobertaTokenizer, XLMRobertaForSequenceClassification
from transformers import AlbertTokenizer, AlbertForSequenceClassification

bert_tokeninzer = AlbertTokenizer.from_pretrained('albert-base-v2')
bert_tokeninzer.save_pretrained(OUTPUT_DIR/f"albert-base-v2")
bert_model = AlbertForSequenceClassification.from_pretrained('albert-base-v2')
bert_model.save_pretrained(OUTPUT_DIR/'albert-base-v2')
detox_model = Detoxify(
    'original-small', 
    checkpoint="../data/external/detoxify_ckpt/original-albert-0e1d6498.ckpt",
    device="cuda"
)
detox_model.predict(train_df["comment_text"].tolist()[0])
val_df['more_detoxify_ori_small_score'] = val_df['more_toxic'].progress_map(lambda line: detox_model.predict(line))
val_df['less_detoxify_ori_small_score'] = val_df['less_toxic'].progress_map(lambda line: detox_model.predict(line))

more_detoxify_df = val_df["more_detoxify_ori_small_score"].apply(pd.Series)
less_detoxify_df = val_df["less_detoxify_ori_small_score"].apply(pd.Series)

detoxify_cols = more_detoxify_df.columns.tolist()

OOF_DF["more_detoxify_ori_small_score"] = more_detoxify_df[detoxify_cols].sum(axis=1)
OOF_DF["less_detoxify_ori_small_score"] = less_detoxify_df[detoxify_cols].sum(axis=1)
OOF_DF["detoxify_ori_small_diff"] = OOF_DF["more_detoxify_ori_small_score"] - OOF_DF["less_detoxify_ori_small_score"]
detoxify_score = OOF_DF[OOF_DF["detoxify_ori_small_diff"]>0]["detoxify_ori_small_diff"].count()/len(OOF_DF)
print(f"Detoxify Original Small Model Score: {detoxify_score:.6f}")<jupyter_output>Detoxify Original Small Model Score: 0.696127<jupyter_text><h2 style = "font-size:45px;             font-family:Comic Sans MS ;             font-weight : normal;              background-color: eeebf1 ;             color : 4c1c84;             text-align: center;             border-radius: 100px 100px;">    Unbiased-Small<jupyter_code>loaded = torch.load("../data/external/detoxify_ckpt/unbiased-albert-c8519128.ckpt")
loaded["config"]["arch"]["args"]
from transformers import BertTokenizer, BertForSequenceClassification
from transformers import RobertaTokenizer, RobertaForSequenceClassification
from transformers import XLMRobertaTokenizer, XLMRobertaForSequenceClassification
from transformers import AlbertTokenizer, AlbertForSequenceClassification

bert_tokeninzer = AlbertTokenizer.from_pretrained('albert-base-v2')
bert_tokeninzer.save_pretrained(OUTPUT_DIR/f"albert-base-v2")
bert_model = AlbertForSequenceClassification.from_pretrained('albert-base-v2')
bert_model.save_pretrained(OUTPUT_DIR/'albert-base-v2')
detox_model = Detoxify(
    'unbiased-small', 
    checkpoint="../data/external/detoxify_ckpt/unbiased-albert-c8519128.ckpt",
    device="cuda"
)
detox_model.predict(train_df["comment_text"].tolist()[0])
val_df['more_detoxify_unb_small_score'] = val_df['more_toxic'].progress_map(lambda line: detox_model.predict(line))
val_df['less_detoxify_unb_small_score'] = val_df['less_toxic'].progress_map(lambda line: detox_model.predict(line))

more_detoxify_df = val_df["more_detoxify_unb_small_score"].apply(pd.Series)
less_detoxify_df = val_df["less_detoxify_unb_small_score"].apply(pd.Series)

detoxify_cols = more_detoxify_df.columns.tolist()

OOF_DF["more_detoxify_unb_small_score"] = more_detoxify_df[detoxify_cols].sum(axis=1)
OOF_DF["less_detoxify_unb_small_score"] = less_detoxify_df[detoxify_cols].sum(axis=1)
OOF_DF["detoxify_unb_small_diff"] = OOF_DF["more_detoxify_unb_small_score"] - OOF_DF["less_detoxify_unb_small_score"]
detoxify_score = OOF_DF[OOF_DF["detoxify_unb_small_diff"]>0]["detoxify_unb_small_diff"].count()/len(OOF_DF)
print(f"Detoxify Unbiased Small Model Score: {detoxify_score:.6f}")

more_cols = [
#     "Ridge_more_pred",
#     "Ruddit_Ridge_more_pred",
    "more_detoxify_score",
    "more_detoxify_unbiased_score",
    "more_detoxify_multi_score",
    "more_detoxify_ori_small_score",
    "more_detoxify_unb_small_score"
]

less_cols = [
#     "Ridge_less_pred",
#     "Ruddit_Ridge_less_pred",
    "less_detoxify_score",
    "less_detoxify_unbiased_score",
    "less_detoxify_multi_score",
    "less_detoxify_ori_small_score",
    "less_detoxify_unb_small_score"
]

OOF_DF["ensemble_more"] = OOF_DF[more_cols].mean(axis=1)
OOF_DF["ensemble_less"] = OOF_DF[less_cols].mean(axis=1)

OOF_DF["ensemble_diff"] = OOF_DF["ensemble_more"] - OOF_DF["ensemble_less"]
detoxify_score = OOF_DF[OOF_DF["ensemble_diff"]>0]["ensemble_diff"].count()/len(OOF_DF)
print(f"Ensemble Score: {detoxify_score:.6f}")
whole_text = list(set(val_df["less_toxic"].unique().tolist())&set(val_df["more_toxic"].unique().tolist()))
print(len(whole_text))<jupyter_output>8945
{'fraction_non_alphanumeric': 0.13501376826943445, 'fraction_numerical': 0.01779284049989409, 'mean_word_length': 5.279861665336526, 'pattern_counts': {'":': 48, '<': 62, '<?xml version=': 0, '>': 73, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '201384', 'license': 'RSA-MD', 'n_tokens_mistral': 9262, 'n_tokens_neox': 8893, 'n_words': 1211, 'path': 'notebooks/020_exp.ipynb', 'repo_name': 'skywalker0415/jigsaw', 'size': '23605'}