<jupyter_start><jupyter_text>6 февраля, подбор параметров для случайного леса  Загрузка и обработка данных<jupyter_code>import pandas as pd
path_to_data = './data/data_with_features'
from utils.make_vectoizer import make_vectorizer
from utils.io_custom import dump_pickle_object


train = pd.read_csv('./data/clear_data/train_data.csv')
new_vectorizer = make_vectorizer(train_data=train,
                                 ngram_range=(3,4),
                                 max_features=4000,
                                 analyzer='char_wb')
dump_pickle_object('./data/redir/vectorizer.obj', new_vectorizer)
from utils.feature_generation import get_data_with_feature


train = get_data_with_feature(train, './data/redir')
train['cosine_dist'] = train['cosine_dist'].fillna(train['cosine_dist'].mean())
validate = pd.read_csv('./data/clear_data/validate_data.csv')
validate = get_data_with_feature(validate, './data/redir')
validate['cosine_dist'] = validate['cosine_dist'].fillna(validate['cosine_dist'].mean())

X_validate = validate.drop(columns=['is_redirect'])
y_validate = validate['is_redirect']
from sklearn.model_selection import train_test_split


X_train, X_test, y_train, y_test = train_test_split(
    train.drop(columns=['is_redirect']),
    train['is_redirect'],
    stratify=train['is_redirect'],
    random_state=0,
    shuffle=True,
    test_size=0.2
)<jupyter_output><empty_output><jupyter_text>Подбор параметров для случайного леса<jupyter_code>from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV
base_rf = RandomForestClassifier(random_state=1)

parameters = {
    'n_estimators': range(10, 100, 10),
    'max_depth': range(1, 12),
    'min_samples_leaf': range(1, 7),
    'min_samples_split': range(2, 9),
    'criterion': ['gini', 'entropy'],
    'class_weight': ['balanced', 'balanced_subsample']
}

grid_searh_cv_clf = GridSearchCV(base_rf,
                                 parametrs,
                                 cv=5)

grid_searh_cv_clf.fit(X_train, y_train)
grid_searh_cv_clf.best_estimator_
best_rf = grid_searh_cv_clf.best_estimator_<jupyter_output><empty_output><jupyter_text>Топ 5 используемых признаков<jupyter_code># важность фич (как часто данная фича использовалась и давала максимальное снижеие неопределенности)
feature_importances_df = pd.DataFrame({
        'features': list(X_train),
        'feature_importances': best_rf.feature_importances_
}).sort_values('feature_importances', ascending=False)

feature_importances_df[:5]<jupyter_output><empty_output><jupyter_text>Метрики для обучающей выборки<jupyter_code>from sklearn.metrics import roc_auc_score, f1_score, accuracy_score, confusion_matrix


predicts_for_train = best_rf.predict(X_train)
predicts_for_test = best_rf.predict(X_test)

print("RF from grid searcher cv:")
print("\nMetrics for train:")
for metric in [roc_auc_score, f1_score, accuracy_score]:
    print(metric.__name__, round(metric(y_train, predicts_for_train), 4))

print("\nMetrics for test:")
for metric in [roc_auc_score, f1_score, accuracy_score]:
    print(metric.__name__, round(metric(y_test, predicts_for_test), 4))<jupyter_output>RF from grid searcher cv:

Metrics for train:
roc_auc_score 0.9728
f1_score 0.9714
accuracy_score 0.9769

Metrics for test:
roc_auc_score 0.9637
f1_score 0.9603
accuracy_score 0.9678<jupyter_text>Метрики для валидационной выборки<jupyter_code>predicts_for_validate = best_rf.predict(X_validate)

print("\nMetrics for validate with default cutoff:")
for metric in [roc_auc_score, f1_score, accuracy_score]:
    print(metric.__name__, round(metric(y_validate, predicts_for_validate), 4))
    
print('Confusion matrix for validate:')
print(confusion_matrix(y_validate, predicts_for_validate))
class_probabilities = best_rf.predict_proba(X_validate)
custom_cutoff = 0.5
predicts_with_custom_cutoff = []

for zero_prob, one_prob in class_probabilities:
        predicts_with_custom_cutoff.append(1 if one_prob > custom_cutoff else 0)
print("\nMetrics for validate with custom cutoff:")
for metric in [roc_auc_score, f1_score, accuracy_score]:
    print(metric.__name__, round(metric(y_validate, predicts_with_custom_cutoff), 4))
    
print('Confusion matrix for validate:')
print(confusion_matrix(y_validate, predicts_with_custom_cutoff))<jupyter_output>Metrics for validate with custom cutoff: 0.5
roc_auc_score 0.5743
f1_score 0.6054
accuracy_score 0.5085
Confusion matrix for validate:
[[ 31 109]
 [  7  89]]<jupyter_text>Подбор cutoff<jupyter_code>import numpy as np


class_probabilities = best_rf.predict_proba(X_validate)
all_f1_scores = []
all_cutoffs = list(np.arange(0, 1, 0.0001))

for current_cutoff in all_cutoffs:
    predicts_with_current_cutoff = []

    for zero_prob, one_prob in class_probabilities:
        predicts_with_current_cutoff.append(1 if one_prob > current_cutoff else 0)

    all_f1_scores.append(f1_score(y_validate, np.array(predicts_with_current_cutoff)))
    
max_f1_score = max(all_f1_scores)
cutoff_for_max_f1_score = all_cutoffs[np.argmax(all_f1_scores)]

print("Max f1_score is:", max_f1_score)
print("Cutoff is:", cutoff_for_max_f1_score)<jupyter_output>Max f1_score is: 0.6164874551971327
Cutoff is: 0.6<jupyter_text>Метрики для валидационной выборки<jupyter_code>predicts_with_custom_cutoff = []

for zero_prob, one_prob in class_probabilities:
        predicts_with_custom_cutoff.append(1 if one_prob > cutoff_for_max_f1_score else 0)

print('\nMetrics for validate with custom cutoff:', cutoff_for_max_f1_score)
for metric in [roc_auc_score, f1_score, accuracy_score]:
    print(metric.__name__, round(metric(y_validate, predicts_with_custom_cutoff), 4))
    
print('Confusion matrix for validate with custom cutoff:', cutoff_for_max_f1_score)
print(confusion_matrix(y_validate, predicts_with_custom_cutoff))<jupyter_output>Metrics for validate with custom cutoff: 0.6
roc_auc_score 0.6015
f1_score 0.6165
accuracy_score 0.5466
Confusion matrix for validate with custom cutoff: 0.6
[[43 97]
 [10 86]]
{'fraction_non_alphanumeric': 0.09788227447056862, 'fraction_numerical': 0.028847757211939305, 'mean_word_length': 5.974418604651163, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '232633', 'license': 'MIT', 'n_tokens_mistral': 2237, 'n_tokens_neox': 2149, 'n_words': 396, 'path': 'select_cutoff_for_rf.ipynb', 'repo_name': 'comptech-winter-school/online-store-redirects', 'size': '5997'}