<jupyter_start><jupyter_text>Fixing Fare<jupyter_code>data[data['Fare'].isnull()]
data[(data["Pclass"] == 3) & (data["Embarked"] == "S")]["Fare"].median()
data.loc[data['Fare'].isnull(), "Fare"] = 8.05
data.loc["test"].loc[1044]<jupyter_output><empty_output><jupyter_text>Fixing Embarked<jupyter_code>data[(data['Embarked'].isnull())]
data[data["Cabin"].str.startswith('B', na=False)][["Cabin", "Embarked"]].sort_values(['Cabin','Embarked'], ascending=[True,True])
data.loc[data['Embarked'].isnull(), "Embarked"] = "S"
data.loc["train"].loc[[62,830]]
data.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
MultiIndex: 1309 entries, (train, 1) to (test, 1309)
Data columns (total 11 columns):
Age         1046 non-null float64
Cabin       295 non-null object
Embarked    1309 non-null object
Fare        1309 non-null float64
Name        1309 non-null object
Parch       1309 non-null int64
Pclass      1309 non-null int64
Sex         1309 non-null object
SibSp       1309 non-null int64
Survived    891 non-null float64
Ticket      1309 non-null object
dtypes: float64(3), int64(3), object(5)
memory usage: 126.7+ KB<jupyter_text>Fixing Age<jupyter_code>data[(data['Age'].isnull())]
data[(data['Age'].notnull())].groupby(['Parch', 'Sex'])["Age"].mean()
data[(data['Age'].isnull())].groupby(['Parch', 'Sex']).count()
selection = (data['Age'].isnull()) & (data['Parch'] == 0) & (data['Sex'] == 'female')
data.loc[selection, "Age"] = 31.116379
selection = (data['Age'].isnull()) & (data['Parch'] == 0) & (data['Sex'] == 'male')
data.loc[selection, "Age"] = 32.338619
selection = (data['Age'].isnull()) & (data['Parch'] == 1) & (data['Sex'] == 'female')
data.loc[selection, "Age"] = 25.608434
selection = (data['Age'].isnull()) & (data['Parch'] == 1) & (data['Sex'] == 'male')
data.loc[selection, "Age"] = 24.272727
selection = (data['Age'].isnull()) & (data['Parch'] == 2) & (data['Sex'] == 'female')
data.loc[selection, "Age"] = 20.645593
selection = (data['Age'].isnull()) & (data['Parch'] == 2) & (data['Sex'] == 'male')
data.loc[selection, "Age"] = 16.383684
selection = (data['Age'].isnull()) & (data['Parch'] == 4) & (data['Sex'] == 'female')
data.loc[selection, "Age"] = 44.666667
selection = (data['Age'].isnull()) & (data['Parch'] == 9) & (data['Sex'] == 'female')
data.loc[selection, "Age"] = 43.000000
selection = (data['Age'].isnull()) & (data['Parch'] == 9) & (data['Sex'] == 'male')
data.loc[selection, "Age"] = 40.000000<jupyter_output><empty_output><jupyter_text>DF to Numpy<jupyter_code>data.info()
dropped_features = ["Name","Cabin","Ticket"]
data_encoded = data.drop(dropped_features, 1)
data_encoded = pd.get_dummies(data_encoded)
train_encoded = data_encoded.loc["train"].copy()
test_encoded = data_encoded.loc["test"].copy()

labels = train_encoded["Survived"]

train_encoded.drop(["Survived"], 1, inplace=True)
test_encoded.drop(["Survived"], 1, inplace=True)
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import VotingClassifier


from sklearn.model_selection import cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import f1_score
from sklearn.feature_selection import RFECV

def display_scores(scores):
    print("Scores:", scores)
    print("Mean:", scores.mean())
    print("Standard deviation:", scores.std())

def test_models(X, y):
    clf_1 = RandomForestClassifier(random_state=42)
    rfecv = RFECV(estimator=clf_1, cv=5, scoring='f1')
    scores = cross_val_score(clf_1, rfecv.fit_transform(X, y), y, scoring="f1", cv=5)
    print("\nRandomForestClassifier:%d" % rfecv.n_features_)
    display_scores(scores)
    
    clf_2 = ExtraTreesClassifier(random_state=42)
    rfecv = RFECV(estimator=clf_2, cv=5, scoring='f1')
    scores = cross_val_score(clf_2, rfecv.fit_transform(X, y), y, scoring="f1", cv=5)
    print("\nExtraTreesClassifier:%d" % rfecv.n_features_)
    display_scores(scores)
    
    clf_3 = GradientBoostingClassifier(random_state=42)
    rfecv = RFECV(estimator=clf_3, cv=5, scoring='f1')
    scores = cross_val_score(clf_3, rfecv.fit_transform(X, y), y, scoring="f1", cv=5)
    print("\nGradientBoostingClassifier:%d" % rfecv.n_features_)
    display_scores(scores)
    
    clf_4 = AdaBoostClassifier(random_state=42)
    rfecv = RFECV(estimator=clf_4, cv=5, scoring='f1')
    scores = cross_val_score(clf_4, rfecv.fit_transform(X, y), y, scoring="f1", cv=5)
    print("\nAdaBoostClassifier:%d" % rfecv.n_features_)
    display_scores(scores)
    
    eclf2 = VotingClassifier(estimators=[('random', clf_1), ('extra', clf_2), ('gnb', clf_3), ('ada', clf_4)],voting='soft')
    scores = cross_val_score(eclf2, X, y, scoring="f1", cv=5)
    print("\nVoting")
    display_scores(scores)
test_models(train_encoded, labels)
from sklearn.model_selection import KFold
from sklearn.metrics import accuracy_score
def best_slice_model(clf, X, y):
    best_accuracy = -1
    best_clf = clf
    kf = KFold(n_splits=5)
    for train_index, test_index in kf.split(X):
        X_train, X_test = X[train_index], X[test_index]
        y_train, y_test = y[train_index], y[test_index]
        clf.fit(X_train, y_train)
        l_accuracy = accuracy_score(y_test, clf.predict(X_test))
        if l_accuracy > best_accuracy:
            best_accuracy = l_accuracy
            best_clf = clf
            print(best_clf.__class__.__name__, " Best accuracy:",best_accuracy)
    return best_clf
def build_voting_clf(X, y):
    clf_1 = RandomForestClassifier(random_state=42)
    clf_1 = best_slice_model(clf_1, X, y)
    
    clf_2 = ExtraTreesClassifier(random_state=42)
    clf_2 = best_slice_model(clf_2, X, y)
    
    clf_3 = GradientBoostingClassifier(random_state=42)
    clf_3 = best_slice_model(clf_3, X, y)
    
    clf_4 = AdaBoostClassifier(random_state=42)
    clf_4 = best_slice_model(clf_4, X, y)
    
    eclf2 = VotingClassifier(estimators=[('random', clf_1), ('extra', clf_2), ('gnb', clf_3), ('ada', clf_4)],voting='soft')
    eclf2.fit(X, y)#Is this really necessary?
    return eclf2
    
clf = build_voting_clf(train_encoded.as_matrix(), labels.as_matrix())<jupyter_output>RandomForestClassifier  Best accuracy: 0.782122905028
RandomForestClassifier  Best accuracy: 0.803370786517
RandomForestClassifier  Best accuracy: 0.837078651685
RandomForestClassifier  Best accuracy: 0.842696629213
ExtraTreesClassifier  Best accuracy: 0.776536312849
ExtraTreesClassifier  Best accuracy: 0.808988764045
GradientBoostingClassifier  Best accuracy: 0.798882681564
GradientBoostingClassifier  Best accuracy: 0.831460674157
GradientBoostingClassifier  Best accuracy: 0.859550561798
AdaBoostClassifier  Best accuracy: 0.765363128492
AdaBoostClassifier  Best accuracy: 0.820224719101
AdaBoostClassifier  Best accuracy: 0.859550561798<jupyter_text>Voting has the best results.<jupyter_code>clf = VotingClassifier(estimators=[('random', RandomForestClassifier(random_state=42)), ('extra', ExtraTreesClassifier(random_state=42)), ('gnb', GradientBoostingClassifier(random_state=42)), ('ada', AdaBoostClassifier(random_state=42))],voting='soft')
clf.fit(train_encoded, labels)
test["Survived"] = clf.predict(test_encoded)
test['Survived'].to_csv("result_2.csv")<jupyter_output><empty_output><jupyter_text>Setting some hyperparameterswork in progress...<jupyter_code>from sklearn.model_selection import GridSearchCV
params = {'random__max_features': [0.2, 0.4, 0.6], 'random__criterion': ['gini', 'entropy'], 'random__bootstrap': [True, False], 'random__n_estimators': [10,15,20,25],
         'extra__max_features': [0.2, 0.4, 0.6], 'extra__criterion': ['gini', 'entropy'], 'extra__bootstrap': [True, False], 'extra__n_estimators': [10,15,20,25],
         'gnb__max_features': [0.2, 0.4, 0.6], 'gnb__loss': ['deviance', 'exponential'], 'gnb__learning_rate': [0.9, 0.1,0.15,0.2], 'gnb__n_estimators': [100,120,140,150],}
grid_search = GridSearchCV(clf, params, cv=5,scoring='f1')
grid_search.fit(train_encoded, labels)
grid_search.best_estimator_
test["Survived"] = grid_search.best_estimator_.predict(test_encoded)
test['Survived'].to_csv("result_2.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16298709132585354, 'fraction_numerical': 0.05670165279285801, 'mean_word_length': 5.960537363560033, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '304300', 'license': 'Apache-2.0', 'n_tokens_mistral': 3435, 'n_tokens_neox': 3096, 'n_words': 540, 'path': 'titanic/Titanic - Ensemble - Brute Force.ipynb', 'repo_name': 'mathcunha/ml', 'size': '8289'}