<jupyter_start><jupyter_text>Feature Selection using Permutation Importances<jupyter_code>import pandas as pd

df = pd.read_csv('mushrooms_mapped.csv')
# Split data into train, val, and test
from sklearn.model_selection import train_test_split

train, test = train_test_split(df, random_state=42)
train, val = train_test_split(train, random_state=42)

train.shape, val.shape, test.shape
target = 'class'
features = train.columns.drop([target] + ['Unnamed: 0', 'veil_type'])
X_train = train[features]
y_train = train[target]
X_val = val[features]
y_val = val[target]
import category_encoders as ce
from sklearn.pipeline import make_pipeline
from sklearn.ensemble import RandomForestClassifier

transformers = make_pipeline(
    ce.OrdinalEncoder()
)

X_train_transformed = transformers.fit_transform(X_train, y_train)
X_val_transformed = transformers.transform(X_val)

model = RandomForestClassifier(n_estimators=100, random_state=42, n_jobs=-1)
model.fit(X_train_transformed, y_train)
import eli5
from eli5.sklearn import PermutationImportance

permuter = PermutationImportance(    
    model, 
    scoring='accuracy', 
    n_iter=5, 
    random_state=42
)

permuter.fit(X_val_transformed, y_val)
feature_names = X_val.columns.tolist()
eli5.show_weights(
    permuter,
    top=None, # show permutation importances for all features
    feature_names=feature_names
)<jupyter_output><empty_output><jupyter_text>Without Odor We Can Build A Model Dependent On More Than Three Features<jupyter_code>features = train.columns.drop([target] + ['Unnamed: 0', 'veil_type', 'odor'])
X_train = train[features]
y_train = train[target]
X_val = val[features]
y_val = val[target]
transformers = make_pipeline(
    ce.OrdinalEncoder()
)

X_train_transformed = transformers.fit_transform(X_train, y_train)
X_val_transformed = transformers.transform(X_val)

model = RandomForestClassifier(n_estimators=100, random_state=42, n_jobs=-1)
model.fit(X_train_transformed, y_train)
permuter = PermutationImportance(    
    model, 
    scoring='accuracy', 
    n_iter=5, 
    random_state=42
)

permuter.fit(X_val_transformed, y_val)
feature_names = X_val.columns.tolist()
eli5.show_weights(
    permuter,
    top=None, # show permutation importances for all features
    feature_names=feature_names
)
minimum_importance = 0
mask = permuter.feature_importances_ > minimum_importance

X_train.columns[mask].tolist()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.0964004964832437, 'fraction_numerical': 0.011998345055854365, 'mean_word_length': 6.371951219512195, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '434608', 'license': 'MIT', 'n_tokens_mistral': 869, 'n_tokens_neox': 830, 'n_words': 165, 'path': 'notebooks/Feature_Selection.ipynb', 'repo_name': 'dggasque/mushroom_classifier', 'size': '2417'}