<jupyter_start><jupyter_text>Feature extraction<jupyter_code>def chunks(lst, n):
    """Yield successive n-sized chunks from lst."""
    for i in range(0, len(lst), n):
        yield lst[i:i + n]


def get_pretrained_detoxify_model ( model_type, device):
    detox = Detoxify(model_type= model_type, device=device)
    return detox


def detoxify_predict (model_type, texts, batch_size, device):
    model = get_pretrained_detoxify_model ( model_type, device)
    
    for i ,chunk in enumerate(chunks(texts, batch_size)):
        if i == 0:
            results = pd.DataFrame( model.predict(chunk) )
        else:
            results = results.append(pd.DataFrame( model.predict(chunk) ))  

    results.columns = [f"{model_type}_{c}" for c in results.columns ]        
            
    return results.reset_index(drop = True)
def tf_idf_feature (texts, feature_name, path, folds=5):
    pred = np.zeros( (len(texts), ) )
    for fold in range(folds):
        model = pickle.load(open(f"{path}/{feature_name}_{fold}.pkl", 'rb'))
        vec = pickle.load(open(f"{path}/{feature_name}_vec_{fold}.pkl", 'rb'))
        X = vec.transform ( texts )
        pred += model.predict_proba ( X )[:,1]
    
    return pred/folds<jupyter_output><empty_output><jupyter_text>Load Test Data<jupyter_code>df_submisison = pd.read_csv("../input/jigsaw-toxic-severity-rating/comments_to_score.csv")

texts = list(df_submisison["text"].values)

# Detoxify features

original_small = detoxify_predict ("original-small", texts, device="cuda", batch_size=64 ) 
unbiased_small = detoxify_predict ("unbiased-small", texts, device="cuda", batch_size=64 ) 
original = detoxify_predict ("original", texts, device="cuda", batch_size=64 )
unbiased = detoxify_predict ("unbiased", texts, device="cuda", batch_size=64 )
multilingual = detoxify_predict ("multilingual", texts, device="cuda", batch_size=64 )

df_submisison = pd.concat ([df_submisison, original, unbiased, multilingual, original_small, unbiased_small], axis=1)

# tf-idf features (jc)

features = ["toxic","severe_toxic","obscene","threat","insult","identity_hate"]
path = "../models/"
for feat in features:
    feat_name = f"jc_tfidf_{feat}"
    print(feat_name)
    df_submisison[feat_name] = tf_idf_feature(texts, feat_name, path=path, folds=5)

df_submisison.head()<jupyter_output>jc_tfidf_toxic
jc_tfidf_severe_toxic
jc_tfidf_obscene
jc_tfidf_threat
jc_tfidf_insult
jc_tfidf_identity_hate<jupyter_text>Predict<jupyter_code>cols = ['original_toxicity', 'original_severe_toxicity', 
        'unbiased_toxicity', 'unbiased_severe_toxicity', 'unbiased_identity_attack', 'unbiased_insult', 
        'multilingual_toxicity', 'multilingual_sexual_explicit', 
        'original-small_toxicity', 
        'unbiased-small_severe_toxicity', 'unbiased-small_obscene', 
        'jc_tfidf_toxic']

pred = np.zeros( (df_submisison.shape[0], ))
for col in cols:
    pred += rankdata(df_submisison[col].values, method='ordinal')

df_submisison['score'] = rankdata(pred, method='ordinal')

df_submisison[['comment_id', 'score']].to_csv("submission.csv", index=False)

df_submisison.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1357120101943294, 'fraction_numerical': 0.00541573749601784, 'mean_word_length': 4.99236641221374, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '197297', 'license': 'Apache-2.0', 'n_tokens_mistral': 1187, 'n_tokens_neox': 1098, 'n_words': 192, 'path': 'notebooks/09-jrsotc-sol-0-80901-11-detoxify-1-jc-feats.ipynb', 'repo_name': 'steubk/jrsotc-3rdplace', 'size': '3139'}