<jupyter_start><jupyter_text>1. Reading Data<jupyter_code>train_df = pd.read_csv('./dataset/train.csv', index_col='id')
test_df = pd.read_csv('./dataset/test.csv', index_col='id')
submission_df = pd.read_csv('./dataset/sample_submission.csv', index_col='id')
train_df.head()
train_df.info()
for i in train_df.columns.values:
    print(i, 'has', train_df[i].value_counts().shape[0], 'unique values')
# Checking if dataset is unbalanced
train_df['target'].value_counts()<jupyter_output><empty_output><jupyter_text>2. Data Preprocessing<jupyter_code>from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier

from sklearn.metrics import roc_auc_score, mean_squared_error, confusion_matrix, accuracy_score

import seaborn as sns
def handle_binary_values(df):
    df['bin_3'] = df['bin_3'].map({'T':1, 'F':0})
    df['bin_4'] = df['bin_4'].map({'Y':1, 'N':0})
def handle_categorial_features(df, encoder):
    
    X = df.drop('target', axis=1)
    y = df['target']
    
    categorical_features=[]
    for i in X.columns:
        if 'bin' not in i:
            categorical_features.append(i)
    
    #encoder = ce.OrdinalEncoder()
    df = encoder.fit_transform(X, y).reset_index(drop=True)
    
    return pd.concat([df,y], axis=1)
#temp = handle_categorial_features(train_df)
#temp.head(5)
handle_binary_values(train_df)
train_df.head()
def train_encoding_on_model(train_df):
    #handle_binary_values(train_df)
    #The commented ones cause MemoryError
    encoders = [
        ce.OrdinalEncoder(), 
        #ce.OneHotEncoder(),
        #ce.BinaryEncoder(),
        #ce.BaseNEncoder(base=10),
        ce.HashingEncoder(),
        ce.CountEncoder(),

        #ce.HelmertEncoder(),
        #ce.SumEncoder(),
        #ce.BackwardDifferenceEncoder(),
        #ce.PolynomialEncoder(),

        ce.TargetEncoder(),
        ce.LeaveOneOutEncoder(),
        ce.WOEEncoder()
    ]
    
    lst=[]
    for en in encoders:
        print(f'\nUsing {type(en).__name__}')
        df = handle_categorial_features(train_df, en)
        X_train, X_test, y_train, y_test = train_test_split(df.drop('target', axis=1), 
                                                            df['target'], test_size=0.33, random_state=42)
        print('Starting model building')
        model = DecisionTreeClassifier()
        model.fit(X_train, y_train)
        y_pred = model.predict(X_test)
        
        acc = accuracy_score(y_pred, y_test)
        roc = roc_auc_score(y_pred, y_test)
        mse = mean_squared_error(y_pred, y_test)
        
        print('accuracy=', acc)
        print('roc_auc score=', roc)
        print('mse=', mse)
        sns.heatmap(confusion_matrix(y_pred, y_test), annot=True, fmt='d', cmap='Greys', cbar=False, linewidths = 1, square= True)
        plt.xlabel('Actual')
        plt.ylabel('Predicted')
        plt.title('Confusion Matrix')
        plt.show()           
        lst.append((type(en).__name__, acc, roc, mse))
    return lst
lst = train_encoding_on_model(train_df)
for i in lst:
    print(i)
temp = pd.DataFrame(lst, columns=['encoding', 'accuracy', 'roc', 'mse'])
temp
temp.transpose().iloc[1:]
temp
df = handle_categorial_features(train_df, ce.LeaveOneOutEncoder())
X_train, X_test, y_train, y_test = train_test_split(df.drop('target', axis=1), 
                                                    df['target'], test_size=0.33, random_state=42)
print('Starting model building')
model = DecisionTreeClassifier()
model.fit(X_train, y_train)
y_pred = model.predict(X_test)

acc = accuracy_score(y_pred, y_test)
roc = roc_auc_score(y_pred, y_test)
mse = mean_squared_error(y_pred, y_test)
print('accuracy=', acc)
print('roc_auc score=', roc)
print('mse=', mse)
sns.heatmap(confusion_matrix(y_pred, y_test), annot=True, fmt='d', cmap='Greys', cbar=False, linewidths = 1, square= True)
plt.xlabel('Actual')
plt.ylabel('Predicted')
plt.title('Confusion Matrix')
plt.show()           
y_pred = model.predict(X_train)
print('Training Model')
print('accuracy=', accuracy_score(y_pred, y_train))
print('roc_auc score=', roc_auc_score(y_pred, y_train))
print('mse=', mean_squared_error(y_pred, y_train))

y_pred = model.predict(X_test)
print('Training Model')
print('accuracy=', accuracy_score(y_pred, y_test))
print('roc_auc score=', roc_auc_score(y_pred, y_test))
print('mse=', mean_squared_error(y_pred, y_test))
from sklearn.model_selection import StratifiedKFold
def train_model(train, test, fold_no):
    X_train = train.drop('target', axis=1)
    y_train = train['target']
    X_test = test.drop('target', axis=1)
    y_test = test['target']
    
    model.fit(X_train,y_train)
    predictions = model.predict(X_test)
    print('Fold',str(fold_no),'Accuracy:',accuracy_score(y_test,predictions))
fold_no = 1
df = handle_categorial_features(train_df, ce.LeaveOneOutEncoder())

model = DecisionTreeClassifier()
skf = StratifiedKFold(n_splits=5)
for train_index, test_index in skf.split(df, df['target']):
    train = df.loc[train_index,:]
    test = df.loc[test_index,:]
    train_model(train,test,fold_no)
    fold_no += 1
from sklearn.linear_model import LogisticRegression
fold_no = 1
df = handle_categorial_features(train_df, ce.LeaveOneOutEncoder())

model = LogisticRegression()
skf = StratifiedKFold(n_splits=5)
for train_index, test_index in skf.split(df, df['target']):
    train = df.loc[train_index,:]
    test = df.loc[test_index,:]
    train_model(train,test,fold_no)
    fold_no += 1
handle_binary_values(test_df)
test_df.head()
df = handle_categorial_features(train_df, ce.LeaveOneOutEncoder())<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14061104162944435, 'fraction_numerical': 0.006968018581382884, 'mean_word_length': 4.107664233576642, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '121373', 'license': 'MIT', 'n_tokens_mistral': 2043, 'n_tokens_neox': 1988, 'n_words': 346, 'path': 'categorical-feature-encoding-02.ipynb', 'repo_name': 'ankitchobdar/categorical-feature-encoding', 'size': '5597'}