<jupyter_start><jupyter_text>Examining Data<jupyter_code>dt_train.shape
dt_train.isnull().sum()<jupyter_output><empty_output><jupyter_text>Data cleaning  Filling NAs<jupyter_code>dt_train['Age'].hist(bins=20)
# fill in missing values for Age feature
age_count = dt_train['Age'].value_counts(normalize=True)
age_missing = dt_train['Age'].isnull()
dt_train.loc[age_missing, 'Age' ] = np.random.choice(age_count.index,
                                                age_missing.sum(), 
                                                 p=age_count.values)
dt_train['Age'].hist(bins=20)
# fill in the missing values for embarked column
dt_train['Embarked'].fillna(method='ffill', inplace=True)
cols_drop = ['PassengerId', 'Name', 'Ticket', 'Cabin']
dt_train.drop(labels=cols_drop, axis=1, inplace=True)
#dt_train.head()
dt_train.columns<jupyter_output><empty_output><jupyter_text>Examining the features<jupyter_code>np.unique(dt_train['Embarked'])
agg_embarked = dt_train.groupby('Embarked')['Survived'].sum()
plt.bar(agg_embarked.index, agg_embarked.values, width=0.5)
y_col = "Survived"
X_train = dt_train.drop(labels=y_col, axis=1)
X_train['Pclass'] = X_train['Pclass'].astype('str')
X_train = pd.get_dummies(X_train)
y_train = dt_train['Survived']
# do necessary normalization o columns
cols_to_norm = ['Age', 'SibSp', 'Parch', 'Fare']
normed_cols = (X_train[cols_to_norm] - X_train[cols_to_norm].mean(axis=0))\
/X_train[cols_to_norm].std(axis=0)

X_train.drop(labels=cols_to_norm, axis=1, inplace=True)
X_train = pd.concat([normed_cols, X_train], axis=1)
X_train.head()<jupyter_output><empty_output><jupyter_text>Modelling<jupyter_code>from sklearn.model_selection import train_test_split
Xtrain, Xtest, ytrain, ytest = train_test_split(X_train, y_train, random_state=2)
# get baseline using naive bayes
from sklearn.naive_bayes import GaussianNB
model = GaussianNB()
model.fit(Xtrain, ytrain)
pred = model.predict(Xtest) 

np.mean(pred == ytest)
from sklearn.svm import SVC
svc = SVC(kernel='rbf', class_weight='balanced')
from sklearn.model_selection import GridSearchCV

param_grid = {'C': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15],
             'gamma': [0.0001, 0.0005, 0.001, 0.005]}
grid = GridSearchCV(svc, param_grid)
grid.fit(X_train, y_train)
# print(grid.)
print(grid.best_params_)
grid.best_score_
# try RF
from sklearn.ensemble import RandomForestClassifier
rf = RandomForestClassifier(n_estimators=1000)
rf.fit(Xtrain, ytrain)
ypred = rf.predict(Xtest)
np.mean(ypred == ytest)
rf = RandomForestClassifier()
rf_param_grid = {'n_estimators': [5000, 7000, 10000]}
grid_rf = GridSearchCV(rf, rf_param_grid)
grid_rf.fit(X_train, y_train)
grid_rf.best_params_
grid_rf.best_score_<jupyter_output><empty_output><jupyter_text>Test Data<jupyter_code>dt_test = pd.read_csv('test.csv')
dt_test.head(3)
dt_test.isnull().sum(axis=0)
dt_test.shape
dt_test['Age'].hist(bins=20)
# check if na values are random in data
na_index = dt_test.loc[dt_test['Age'].isnull(), :].index
def fill_age_na(data):
    """"""
    age_count = data['Age'].value_counts(normalize=True)
    age_missing = data['Age'].isnull()
    data.loc[age_missing, 'Age' ] = np.random.choice(age_count.index,
                                                age_missing.sum(), 
                                                 p=age_count.values)
    return data

def otherClean(data):
    """"""
    data['Embarked'].fillna(method='ffill', inplace=True)
    cols_drop = ['PassengerId', 'Name', 'Ticket', 'Cabin']
    data = data.drop(labels=cols_drop, axis=1, inplace=False)
    data['Pclass'] = data['Pclass'].astype('str')
    data = pd.get_dummies(data)
    data['Fare'].fillna(value=np.mean(data['Fare']), inplace=True)
    
    return data
    
def get_train_test(data, y_col):
    """method for extracting the feature matrix and target column"""
    X = data.drop(labels=y_col, axis=1)
    y = data[y_col]
    
    return X, y

def standardize(data, cols):
    
    normed_cols = (data[cols] - data[cols].mean(axis=0))\
                    /data[cols].std(axis=0)

    data.drop(labels=cols_to_norm, axis=1, inplace=True)
    data = pd.concat([normed_cols, data], axis=1)
    
    return data

def pred_change(old, new, target):
    
    return np.sum(old[target] != new[target])/old.shape[0]
    
X_test = fill_age_na(dt_test)
X_test = otherClean(X_test)
X_test = standardize(X_test, ['Age', 'SibSp', 'Parch', 'Fare'])<jupyter_output><empty_output><jupyter_text>__Random Forest__<jupyter_code>rf = RandomForestClassifier(n_estimators=7000, random_state=10)
rf.fit(X_train, y_train)
rf_pred = rf.predict(X_test)

submit = pd.DataFrame({'PassengerId': dt_test['PassengerId'],
            'Survived': rf_pred})
submit.to_csv('submission.csv', index=False)<jupyter_output><empty_output><jupyter_text>__RF with grid search__<jupyter_code>rf = RandomForestClassifier(random_state=50)
rf_param_grid = {'n_estimators': [10, 50, 100, 1000, 2000, 5000]}
grid_rf = GridSearchCV(rf, rf_param_grid)
grid_rf.fit(X_train, y_train);
grid_rf.best_params_, grid_rf.best_score_

rf_new = grid_rf.best_estimator_.predict(X_test)
rf_submit = pd.DataFrame({'PassengerId': dt_test['PassengerId'],
            'Survived': rf_new})
submit.to_csv('rf_submission.csv', index=False)<jupyter_output><empty_output><jupyter_text>Check prediction change<jupyter_code>old = pd.read_csv('submission.csv')
pred_change(old, rf_submit, target='Survived')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1462874030291836, 'fraction_numerical': 0.01828592537864795, 'mean_word_length': 5.111738148984198, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '226090', 'license': 'MIT', 'n_tokens_mistral': 2136, 'n_tokens_neox': 2035, 'n_words': 327, 'path': 'TitanicDataAnalysis.ipynb', 'repo_name': 'Aroogz/kaggle-Titanic', 'size': '5414'}