<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
import warnings
warnings.filterwarnings('ignore')

train = pd.read_csv('/content/sample_data/train.csv')
test = pd.read_csv('/content/sample_data/test.csv')
X = train[list(test.columns)]
y = train[train.columns[~train.columns.isin(test.columns)]]
X
X["Cabin"].unique()
from sklearn.base import BaseEstimator, TransformerMixin
import math

def getCabinLetter(x):
    x = str(x)
    if x != 'nan':
        cabin = x[0]
    else:
        cabin = 'X'
    return cabin

def hasCabin(x):
    if x == 'nan':
        return 0
    else:
        return 1

def criaColunaHasFamily(sibsp, parch):
    if sibsp + parch > 0:
        x = 1
    else:
        x = 0
    return x

def criaColunaTicketLen(ticket):
    return len(ticket)


def extraiPronome(nome):
    return nome.split(',')[1].split('.')[0].strip()

class AtributosDesejados(BaseEstimator, TransformerMixin):
    def __init__(self, excluirName=True):
        self.excluirName = excluirName
    def fit(self, X, y=None):
        self.colunasIndesejadas = ['PassengerId', 'Ticket', 'Embarked', "SibSp", "Parch", "Cabin"]
        if self.excluirName:
            self.colunasIndesejadas.append('Name')
        return self
    def transform(self, X, y=None):
        # Adiciona coluna 'hasFamily' se SibSp ou Parch são maiores que zero
        X['hasFamily'] = X.apply(lambda x: criaColunaHasFamily(x["SibSp"], x["Parch"]),axis=1)
        # Cria coluna Ticketlen
        X['TicketLen'] = X["Ticket"].apply(criaColunaTicketLen)
        # trata coluna de Cabin
        X['hasCabin'] = X['Cabin'].apply(lambda x: 0 if pd.isnull(x) else 1)
        X["Cabin"] = X["Cabin"].apply(getCabinLetter)
        # Remove Colunas indesejadas
        Xdrop = X.drop(self.colunasIndesejadas,axis=1)
        # Cria coluna com pronomes de tratamento
        if 'Name' not in self.colunasIndesejadas:
            Xdrop['Name'] = Xdrop['Name'].apply(extraiPronome)
       
        return Xdrop


class AtributosNumericos(BaseEstimator, TransformerMixin):
    def fit(self, X, y=None):
        self.colunasNumericas = X.select_dtypes(include='number').columns
        return self
    def transform(self, X, y=None):
        return X[self.colunasNumericas].to_numpy()



class AtributosCategoricos(BaseEstimator, TransformerMixin):
    def fit(self, X, y=None):
        self.colunasCategoricas = X.select_dtypes(include='object').columns
        return self
    def transform(self, X, y=None):
        return X[self.colunasCategoricas].to_numpy()

from sklearn.impute import SimpleImputer
from sklearn.preprocessing import OneHotEncoder, StandardScaler
from sklearn.pipeline import Pipeline, FeatureUnion

trataAtributos = Pipeline([
    ('unecaracteristicas', FeatureUnion([
        ('pipenum', Pipeline([
            ('atributos_numericos', AtributosNumericos()),
            ('imputer', SimpleImputer(strategy='median')),
            ('scaler', StandardScaler())
        ])),
        ('pipecat', Pipeline([
            ('atributos_categoricos', AtributosCategoricos()),
            ('imputer', SimpleImputer(strategy='most_frequent')),
            ('encoder', OneHotEncoder(handle_unknown='ignore'))
        ]))
    ])),
])

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV, cross_validate, RepeatedKFold
import numpy as np


pipetotal = Pipeline([
    ('atributosDesejados', AtributosDesejados()),
    ('trataAtributos', trataAtributos),
    ('classificador', RandomForestClassifier())
])

parametros = {
    'atributosDesejados__excluirName': [False, True],
    'classificador__criterion': ['entropy'],
    'classificador__max_depth': [None], 
    'classificador__min_samples_leaf': [1,2],
    'classificador__min_samples_split': [2,5],
    'classificador__n_estimators': [30],
    'classificador__max_features': ['auto'],
    'classificador__bootstrap': [True]
}
modelo = GridSearchCV(pipetotal, param_grid=parametros,n_jobs=-1)

scores = cross_validate(modelo, X, y, cv=RepeatedKFold(n_splits=5, n_repeats=5, random_state=10), verbose=10)
scores['test_score'], np.mean(scores['test_score']), np.std(scores['test_score'])
modelo.fit(X,y)
y_pred = modelo.predict(test)
result = test[['PassengerId']]
result['Survived'] = y_pred
result.to_csv('submission.csv',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13469951646327424, 'fraction_numerical': 0.005756389592447617, 'mean_word_length': 3.8863892013498313, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '544403', 'license': 'MIT', 'n_tokens_mistral': 1541, 'n_tokens_neox': 1505, 'n_words': 283, 'path': 'ajuste_de_caracteristicas.ipynb', 'repo_name': 'glima91/mestrado-reconhecimento-de-padroes', 'size': '4343'}