<jupyter_start><jupyter_text>Data exploration<jupyter_code>import numpy as np
import pandas as pd
from sklearn.impute import SimpleImputer
from sklearn import preprocessing
from sklearn.compose import ColumnTransformer
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from scipy import stats
import seaborn as sns
df = pd.read_csv('./data/train.csv')
df<jupyter_output><empty_output><jupyter_text>Data cleaning<jupyter_code>df.isna().sum(axis=0)/len(df)
imp = SimpleImputer(missing_values=np.nan, strategy='median')
imp.fit(df.loc[:,['Age']].values)
df['Age'] = imp.transform(df.loc[:,['Age']].values)
imp2 = SimpleImputer(missing_values=np.nan, strategy='most_frequent')
imp2.fit(df['Embarked'].values.reshape((len(df['Embarked']),1)))
df.loc[:,['Embarked']] = imp2.transform(df['Embarked'].values.reshape((len(df['Embarked']),1)))
df.isna().sum(axis=0)/len(df)
df<jupyter_output><empty_output><jupyter_text>Categorical variables<jupyter_code>vars_disc = ['Survived', 'Pclass', 'Sex', 'SibSp', 'Parch', 'Embarked']
for i in vars_disc:
    print(df[i].value_counts())
labelEncoder = preprocessing.LabelEncoder()
df['Sex'] = labelEncoder.fit_transform(df['Sex'])
df
ct = ColumnTransformer(
    [('one_hot_encoder',preprocessing.OneHotEncoder(drop='first'),['Pclass', 'Embarked' ])],
    remainder='passthrough')
pd.DataFrame(ct.fit_transform(df))
df = pd.DataFrame(ct.fit_transform(df), columns = [
    'is-Pclass-2', 'is-Pclass-3', 'is-Embarked-Q', 'is-Embarked-S', 'PassengerId', 'Survived',
    'Name','Sex', 'Age', 'SibSp','Parch','Ticket','Fare','Cabin']).drop(['Cabin','Ticket','Name'], axis=1).astype(float)<jupyter_output><empty_output><jupyter_text>Choosing variables<jupyter_code>df['familysize'] = df['SibSp'] + df['Parch']
df = df.drop(['SibSp', 'Parch'], axis=1)
df
corr = df.corr()
sns.heatmap(corr,yticklabels = corr.columns, xticklabels = corr.columns,annot=True,fmt=".1f",linewidths=.5)
df.describe()
X = df[['is-Pclass-2','is-Pclass-3','is-Embarked-Q','is-Embarked-S','Sex','Age','Fare','familysize']].values
y = df['Survived'].values<jupyter_output><empty_output><jupyter_text>Random Forest<jupyter_code>classifier = RandomForestClassifier(n_estimators=1000, criterion = 'entropy', random_state=0)
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size = 0.25, random_state = 0)
classifier.fit(X_train, y_train)
y_pred = classifier.predict(X_test)
from sklearn.metrics import confusion_matrix, accuracy_score
cm = confusion_matrix(y_test,y_pred)
cm
accuracy_score(y_test,y_pred)<jupyter_output><empty_output><jupyter_text>Outliers<jupyter_code>sc_X = preprocessing.StandardScaler()
X_trans = sc_X.fit_transform(X)
pd.DataFrame(X_trans)
sns.boxplot(x=X_trans[:,5])
outliers = (abs(X_trans[:,5]) > 3) | (abs(X_trans[:,6]) > 3 ) | (abs(X_trans[:,7]) > 3 )
X_cut = X[~outliers,:]
y_cut = Y=y[~outliers]
len(X) - len(X_cut)
X_train, X_test, y_train, y_test = train_test_split(X_cut,y_cut, test_size = 0.25, random_state = 0)
classifier = RandomForestClassifier(n_estimators=1000, criterion = 'entropy', random_state=0)
classifier.fit(X_train, y_train)
y_pred = classifier.predict(X_test)
cm = confusion_matrix(y_test,y_pred)
print(cm)
print(accuracy_score(y_test,y_pred))<jupyter_output>[[101  19]
 [ 22  68]]
0.8047619047619048<jupyter_text>Variable reduction<jupyter_code>corr = df.corr()
sns.heatmap(corr,yticklabels = corr.columns, xticklabels = corr.columns,annot=True,fmt=".1f",linewidths=.5)
df.columns
X = df[['Sex','Age','Fare','familysize']].values
y = df['Survived'].values
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size = 0.25, random_state = 0)
classifier = RandomForestClassifier(n_estimators=1000, criterion = 'entropy', random_state=0)
classifier.fit(X_train, y_train)
y_pred = classifier.predict(X_test)
cm = confusion_matrix(y_test,y_pred)
print(cm)
print(accuracy_score(y_test,y_pred))
sc_X = preprocessing.StandardScaler()
X_trans = sc_X.fit_transform(X)
pd.DataFrame(X_trans)
outliers = (abs(X_trans[:,1]) > 3) | (abs(X_trans[:,2]) > 3 ) | (abs(X_trans[:,3]) > 3 )
X_cut = X[~outliers,:]
y_cut = Y=y[~outliers]
len(X) - len(X_cut)
X_train, X_test, y_train, y_test = train_test_split(X_cut,y_cut, test_size = 0.25, random_state = 0)
classifier = RandomForestClassifier(n_estimators=1000, criterion = 'entropy', random_state=0)
classifier.fit(X_train, y_train)
y_pred = classifier.predict(X_test)
cm = confusion_matrix(y_test,y_pred)
print(cm)
print(accuracy_score(y_test,y_pred))<jupyter_output>[[102  18]
 [ 22  68]]
0.8095238095238095<jupyter_text>LogisticRegression<jupyter_code>from sklearn.linear_model import LogisticRegression
df.columns
X = df[['Sex','Age','Fare','familysize']].values
y = df['Survived'].values
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size = 0.25, random_state = 0)
classifier = LogisticRegression(random_state=0)
classifier.fit(X_train, y_train)
y_pred = classifier.predict(X_test)
cm = confusion_matrix(y_test,y_pred)
print(cm)
print(accuracy_score(y_test,y_pred))<jupyter_output>[[116  23]
 [ 25  59]]
0.7847533632286996
{'fraction_non_alphanumeric': 0.1683362797092909, 'fraction_numerical': 0.029267334511883715, 'mean_word_length': 8.868217054263566, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '348996', 'license': 'MIT', 'n_tokens_mistral': 2167, 'n_tokens_neox': 2042, 'n_words': 289, 'path': 'prediction/survival-on-titanic-exploration.ipynb', 'repo_name': 'hyfi06/survival-on-titanic', 'size': '5091'}