<jupyter_start><jupyter_text>Importação de Libs<jupyter_code>import pandas as pd
import numpy as np
from category_encoders import OneHotEncoder
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from sklearn.impute import KNNImputer
from lightgbm import LGBMClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier
from skopt import gp_minimize
from sklearn.model_selection import cross_val_score
from mlxtend.classifier import EnsembleVoteClassifier
from mlxtend.evaluate import confusion_matrix
import matplotlib.pyplot as plt
from mlxtend.plotting import plot_confusion_matrix
import plotly.express as px<jupyter_output><empty_output><jupyter_text>Making Functions and Most Important variables/constants<jupyter_code>train = pd.read_csv("train.csv")
test = pd.read_csv("test.csv")
ID = test['PassengerId']
SEED = 4
IMPUTER = KNNImputer(n_neighbors=10, weights='distance')
SCALER = MinMaxScaler()
def search_title(x):
    y = list(x)
    if y.index(','):
        y = y[y.index(',') + 2:y.index(" ", y.index(',') + 2)]
        return "".join(y)
def to_miss(x):
    arr = ['Mlle.', 'Mme.', 'Lady.', 'Ms.', 'Dona.']
    if x in arr:
        x = 'Miss.'
    return x
def to_mr(x):
    arr = ['Dr.', 'Rev.', 'Col.', 'Major.', 'Capt.', 'the.', 'Jonkheer.', 'Sir.', 'Don.', 'the' , 'Master.']
    if x in arr:
        x = 'Mr.'
    return x
def is_married(x):
    if x == 'Mrs.':
        return 1
    return 0
def is_alone(x):
    if x == 0:
        return 1
    return 0
def below_age(x):
    if x < 18:
        return 1
    return 0<jupyter_output><empty_output><jupyter_text>Imputing Age<jupyter_code>train[['Age']] = IMPUTER.fit_transform(train[['Age']])<jupyter_output><empty_output><jupyter_text>Feature Engineering<jupyter_code>train['Title'] = train['Name'].apply(search_title) #make a column of titles
train['Title'] = train['Title'].apply(to_miss) #separate titles that can go to Miss
train['Title'] = train['Title'].apply(to_mr) #separate titles that can go to Mr.
train['is_married'] = train['Title'].apply(is_married) #check if the person are married or not
train['below_age'] = train['Age'].apply(below_age) #check if the passanger is a child or not
train['family_size'] = train['SibSp'] + train['Parch'] #make a column about the family size on Titanic
train['is_alone'] = train['family_size'].apply(is_alone) #check if the passanger are alone
train['group_by_age'] = pd.qcut(train['Age'], q=4, labels=[1,2,3,4]).astype(int) #divide age by groups of Quantiles
train['group_by_fare'] = pd.qcut(train['Fare'], q=4, labels=[1,2,3,4]).astype(int) #divide fare by groups of Quantiles<jupyter_output><empty_output><jupyter_text>Dropping NaN and Unwanted Columns<jupyter_code>train.drop(columns=['PassengerId', 'Cabin', 'Ticket','Name'], inplace=True)
embarked_rows_to_drop = train[train[['Embarked']].isnull().any(axis=1)].index.values
train.drop(index=embarked_rows_to_drop, inplace=True)<jupyter_output><empty_output><jupyter_text>OneHotEncoding<jupyter_code>ohe = OneHotEncoder(use_cat_names=True)
ohe_train = ohe.fit_transform(train)
X = ohe_train.drop(columns='Survived')
y = ohe_train['Survived']<jupyter_output><empty_output><jupyter_text>Scaling<jupyter_code>X = SCALER.fit_transform(X)
X = pd.DataFrame(X, columns=ohe_train.columns[1:])
X<jupyter_output><empty_output><jupyter_text>Train Test Split<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(
X, 
y, 
random_state=SEED,
test_size=0.20, 
stratify=y)<jupyter_output><empty_output><jupyter_text>Stacking Classifier<jupyter_code>clfs = [LogisticRegression(random_state=SEED),
       RandomForestClassifier(random_state=SEED), 
       LGBMClassifier(random_state=SEED, subsample_freq=1)]

eclf = EnsembleVoteClassifier(clfs=clfs, weights=[1,1,2], voting='hard')

arr = clfs + [eclf]

labels = [str(x) for x in arr]

labels = [x[:x.index("(")] for x in labels]
for clf, label in zip(arr, labels):

    scores = cross_val_score(clf, X, y, cv=5, scoring='accuracy')
    print("Accuracy: %0.2f (+/- %0.2f) [%s]" 
          % (scores.mean(), scores.std(), label))<jupyter_output>Accuracy: 0.79 (+/- 0.03) [LogisticRegression]
Accuracy: 0.80 (+/- 0.04) [RandomForestClassifier]
Accuracy: 0.83 (+/- 0.03) [LGBMClassifier]
Accuracy: 0.83 (+/- 0.03) [EnsembleVoteClassifier]<jupyter_text>Tuning<jupyter_code>def search_hyperparams(params):
    SEED = 4

    max_iter = params[0]
    C = params[1]
    n_estimators_random_forest = params[2]
    learning_rate = params[3]
    num_leaves = params[4]
    min_child_samples = params[5]
    subsample = params[6]
    colsample_bytree = params[7]
    n_estimators_lgbm = params[8]
    
    clf1 = LogisticRegression(random_state=SEED, 
                              max_iter=max_iter, 
                              C=C)
    
    clf2 = RandomForestClassifier(random_state=SEED, 
                                  n_estimators= n_estimators_random_forest)
    
    clf3= LGBMClassifier(random_state=SEED, 
                         subsample_freq=1, 
                         learning_rate=learning_rate,
                        num_leaves=num_leaves,
                        min_child_samples= min_child_samples,
                        subsample= subsample,
                        colsample_bytree = colsample_bytree,
                        n_estimators = n_estimators_lgbm)
 
    eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], weights=[1,1,2], voting='soft')
    
    scores = cross_val_score(eclf, X, y, cv=5, scoring='accuracy')
    
    return -scores.mean()

space = [
(100, 5000), #max_iter Logistic Regression
(1e-4, 1e-2, 'log-uniform'), #C Logistic Regression
(20, 200), #n_estimators RandomForest
(1e-3, 1e-1, 'log-uniform'), #learning rate LGBM
(2, 128), #num_leaves LGBM
(1, 100), #min_child_samples LGBM
(0.05, 1.0), #subsample LGBM
(0.1, 1.0), #colsample_bytree LGBM
(100, 1000)] #n_estimators LGBM

result = gp_minimize(search_hyperparams, 
space,
random_state=SEED, 
verbose=1,
n_calls = 30)
max_iter, C, n_estimators_random_forest, learning_rate, num_leaves, min_child_samples, subsample, colsample_bytree, n_estimators_lgbm = result.x
clf1 = LogisticRegression(random_state=SEED, 
                              max_iter=max_iter, 
                              C=C)
    
clf2 = RandomForestClassifier(random_state=SEED, 
                                  n_estimators= n_estimators_random_forest)
    
clf3= LGBMClassifier(random_state=SEED, 
                         subsample_freq=1, 
                         learning_rate=learning_rate,
                        num_leaves=num_leaves,
                        min_child_samples= min_child_samples,
                        subsample= subsample,
                        colsample_bytree = colsample_bytree,
                        n_estimators = n_estimators_lgbm)
 
eclf = EnsembleVoteClassifier(clfs=[clf1, clf2, clf3], weights=[1,1,2], voting='hard')<jupyter_output><empty_output><jupyter_text>Validating<jupyter_code>eclf.fit(X_train, y_train)
eclf.score(X_test, y_test)<jupyter_output><empty_output><jupyter_text>Confusion Matrix<jupyter_code>y_pred = eclf.predict(X_test)
cm = confusion_matrix(y_test, y_pred)

fig, ax = plot_confusion_matrix(conf_mat=cm)

plt.show()<jupyter_output><empty_output><jupyter_text>Definitive Model<jupyter_code>eclf.fit(X, y)<jupyter_output><empty_output><jupyter_text>Imputing Age and Fare<jupyter_code>test[['Age']] = IMPUTER.fit_transform(test[['Age']])
test[['Fare']] = IMPUTER.fit_transform(test[['Fare']])<jupyter_output><empty_output><jupyter_text>Feature Engineering on Df Set<jupyter_code>test['Title'] = test['Name'].apply(search_title) #make a column of titles
test['Title'] = test['Title'].apply(to_miss) #separate titles that can go to Miss
test['Title'] = test['Title'].apply(to_mr) #separate titles that can go to Mr.
test['is_married'] = test['Title'].apply(is_married) #check if the person are married or not
test['below_age'] = test['Age'].apply(below_age) #check if the passanger is a child or not
test['family_size'] = test['SibSp'] + test['Parch'] #make a column about the family size on Titanic
test['is_alone'] = test['family_size'].apply(is_alone) #check if the passanger are alone
test['group_by_age'] = pd.qcut(test['Age'], q=4, labels=[1,2,3,4]).astype(int) #divide age by groups of Quantiles
test['group_by_fare'] = pd.qcut(test['Fare'], q=4, labels=[1,2,3,4]).astype(int) #divide fare by groups of Quantiles<jupyter_output><empty_output><jupyter_text>Dropping Unwanted Columns<jupyter_code>test.drop(columns=['PassengerId', 'Ticket', 'Cabin', 'Name'], inplace=True)<jupyter_output><empty_output><jupyter_text>OneHotEncoding, Scaling and Predicting<jupyter_code>test = ohe.fit_transform(test)
test = SCALER.fit_transform(test)
y_pred = eclf.predict(test)
y_test = pd.read_csv('test_results.csv')
y_test = y_test['Survived']
from sklearn.metrics import accuracy_score
accuracy_score(y_test, y_pred)<jupyter_output><empty_output><jupyter_text>Creating results.csv<jupyter_code>sub = pd.Series(y_pred, index=ID, name='Survived')
sub.to_csv("results.csv", header=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13301299552255105, 'fraction_numerical': 0.016380910778639294, 'mean_word_length': 4.550303030303031, 'pattern_counts': {'":': 0, '<': 73, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '552289', 'license': 'MIT', 'n_tokens_mistral': 3469, 'n_tokens_neox': 3300, 'n_words': 622, 'path': 'Titanic/.ipynb_checkpoints/titanic-checkpoint.ipynb', 'repo_name': 'HeyLucasLeao/kaggle', 'size': '9157'}