<jupyter_start><jupyter_text>Built-in scikit importances<jupyter_code>base_rf = RandomForestClassifier(n_estimators=100,
                                 min_samples_leaf=5,
                                 n_jobs=-1,
                                 oob_score=True)

X_train, y_train = df.drop('interest_level',axis=1), df['interest_level']

rf = clone(base_rf)
rf.fit(X_train, y_train)
print(rf.oob_score_)
I = mkdf(X_train.columns,rf.feature_importances_)            
viz = plot_importances(I, imp_range=(0,.4), title="Feature importance via avg drop in variance (sklearn)")
viz.save('../article/images/cls_dflt.svg')
viz.close()

X_train2 = X_train.copy()
X_train2['random'] = np.random.random(size=len(X_train2))
rf2 = clone(base_rf)
rf2.fit(X_train2, y_train)
print(rf2.oob_score_)
I = mkdf(X_train2.columns,rf2.feature_importances_)
viz = plot_importances(I, imp_range=(0,.4), title="Feature importance via avg drop in variance (sklearn)")
viz.save('../article/images/cls_dflt_random.svg')
viz<jupyter_output>0.7190184794942455
0.7137704652293727<jupyter_text>Importances via dropping columns and retraining<jupyter_code>def dropcol_importances(rf, X_train, y_train):
    rf_ = clone(rf)
    rf_.random_state = 999
    rf_.fit(X_train, y_train)
    baseline = rf_.oob_score_
    imp = []
    for col in X_train.columns:
        X = X_train.drop(col, axis=1)
        rf_ = clone(rf)
        rf_.random_state = 999
        rf_.fit(X, y_train)
        o = rf_.oob_score_
        imp.append(baseline - o)
    return np.array(imp)

def classifier_dropping_importances_cv(rf, X_train, y_train, k=5):
    rf_ = clone(rf)
    rf.random_state = 999
    rf_.fit(X_train, y_train)
    
    cvscore = cross_val_score(clone(rf), X_train, y_train, cv=k, scoring='accuracy')
    m_base = cvscore.mean()
    print(f"{k}-fold cross-validation accuracy {m_base:.4f}")
                              
    imp = []
    for col in X_train.columns:
        X = X_train.drop(col, axis=1)
        rf_ = clone(rf)
        rf_.random_state = 999
        rf_.fit(X, y_train)
        
        cvscore = cross_val_score(clone(rf_), X, y_train, cv=5, scoring='accuracy')
        m = cvscore.mean()
        print(f"drop {col} {k}-fold cross-validation accuracy {m:.4f}")
        imp.append(m_base - m)
        
    return np.array(imp)
# max_features=n_features for regressors but sqrt for classifiers

X_train, y_train = df.drop('interest_level',axis=1), df['interest_level']
rf = clone(base_rf)
imp = dropcol_importances(rf, X_train, y_train)
I = mkdf(X_train.columns,imp)
viz = plot_importances(I)
viz.save('../article/images/cls_dropcol.svg')

X_train2 = X_train.copy()
X_train2['random'] = np.random.random(size=len(X_train2))
rf2 = clone(base_rf)
imp = dropcol_importances(rf2, X_train2, y_train)
I = mkdf(X_train2.columns,imp)
viz = plot_importances(I)
viz.save('../article/images/cls_dropcol_random.svg')
viz<jupyter_output><empty_output><jupyter_text>Importances via column permutation<jupyter_code>from sklearn.ensemble.forest import _generate_unsampled_indices

# TODO: add arg for subsample size to compute oob score

def oob_classifier_accuracy(rf, X_train, y_train):
    X = X_train.values
    y = y_train.values

    n_samples = len(X)
    n_classes = len(np.unique(y))
    predictions = np.zeros((n_samples, n_classes))
    for tree in rf.estimators_:
        unsampled_indices = _generate_unsampled_indices(tree.random_state, n_samples)
        tree_preds = tree.predict_proba(X[unsampled_indices, :])
        predictions[unsampled_indices] += tree_preds

    predicted_class_indexes = np.argmax(predictions, axis=1)
    predicted_classes = [rf.classes_[i] for i in predicted_class_indexes]

    oob_score = np.mean(y == predicted_classes)
    return oob_score

def permutation_importances(rf, X_train, y_train, metric):
    """
    Return importances from pre-fit rf; metric is function
    that measures accuracy or R^2 or similar. This function
    works for regressors and classifiers.
    """
    baseline = metric(rf, X_train, y_train)
    imp = []
    for col in X_train.columns:
        save = X_train[col].copy()
        X_train[col] = np.random.permutation(X_train[col])
        m = metric(rf, X_train, y_train)
        X_train[col] = save
        imp.append(baseline - m)
    return np.array(imp)
X_train, y_train = df.drop('interest_level',axis=1), df['interest_level']
rf = clone(base_rf)
rf.fit(X_train, y_train)
oob = oob_classifier_accuracy(rf, X_train, y_train)
print("oob accuracy",oob)

imp = permutation_importances(rf, X_train, y_train,
                              oob_classifier_accuracy)
I = mkdf(X_train.columns,imp)
viz = plot_importances(I)
viz.save('../article/images/cls_permute.svg')

X_train2 = X_train.copy()
X_train2['random'] = np.random.random(size=len(X_train))
rf2 = clone(base_rf)
rf2.fit(X_train2, y_train)

imp = permutation_importances(rf2, X_train2, y_train,
                            oob_classifier_accuracy)
I = mkdf(X_train2.columns,imp)
viz = plot_importances(I)
viz.save('../article/images/cls_permute_random.svg')
viz<jupyter_output>oob accuracy 0.717012481763657
{'fraction_non_alphanumeric': 0.12044653349001175, 'fraction_numerical': 0.020759890325107718, 'mean_word_length': 3.6596715328467155, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '203022', 'license': 'MIT', 'n_tokens_mistral': 1954, 'n_tokens_neox': 1883, 'n_words': 336, 'path': 'notebooks/permutation-importances-classifier.ipynb', 'repo_name': 'escherba/random-forest-importances', 'size': '5106'}