<jupyter_start><jupyter_text>Titanic 2<jupyter_code>%load_ext autoreload
%autoreload 2

%matplotlib inline
from fastai.imports import *
from fastai.structured import *

from pandas_summary import DataFrameSummary
from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier
from IPython.display import display

from sklearn import metrics
from sklearn.model_selection import train_test_split
from sklearn.model_selection import RandomizedSearchCV
PATH = "data/titanic/"
df_raw = pd.read_csv(f'{PATH}train.csv')
df_raw.isnull().sum() / len(df_raw.isnull())
df_raw.head()
df_raw.drop('Cabin', axis=1, inplace=True)
df_raw.isnull().sum() / len(df_raw.isnull())
df_raw.Age.fillna(df_raw['Age'].mean(), inplace=True)
df_raw.dropna(inplace=True)
df_raw.isnull().sum() / len(df_raw.isnull())
df_raw.info()
df_raw.drop(['PassengerId', 'Fare', 'Ticket', 'Name'], axis=1, inplace=True)
df_raw.info()
df = pd.get_dummies(df_raw, columns=['Sex', 'Embarked'], drop_first=True)
df.head()
X = df.drop('Survived', axis=1)
y = df['Survived']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.44, random_state=42)
m = RandomForestClassifier(n_jobs=-1)
%time m.fit(X_train, y_train)
m.score(X_test, y_test)
p = {"n_estimators": np.arange(50, 1000, 50),
         "max_depth": np.arange(1, 100),
         "min_samples_split": np.arange(1, 50),
         "min_samples_leaf": np.arange(1, 10)}

tuned_model = RandomizedSearchCV(m, p, cv=5, n_jobs=-1)

%time tuned_model.fit(X_train, y_train)
tuned_model.score(X_test, y_test)
df_test = pd.read_csv(f'{PATH}test.csv')
df_test_raw = df_test
df_test.isnull().sum() / len(df_test)
df_test.Age.fillna(df_test['Age'].mean(), inplace=True)
df_test.columns
df_test.drop(['PassengerId', 'Fare', 'Ticket', 'Name', 'Cabin'], axis=1, inplace=True)

df_test.dropna(inplace=True)
df_test.isnull().sum() / len(df_test)
df_test = pd.get_dummies(df_test, columns=['Sex', 'Embarked'], drop_first=True)
y_test = m.predict(df_test)
d = {'PassengerId': df_test_raw.PassengerId, 'Survived': y_test}
pd.DataFrame(d).to_csv(f'{PATH}res2.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16454502593116455, 'fraction_numerical': 0.015558698727015558, 'mean_word_length': 7.007547169811321, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '301313', 'license': 'Apache-2.0', 'n_tokens_mistral': 898, 'n_tokens_neox': 862, 'n_words': 146, 'path': 'courses/ml1/titantic_2.ipynb', 'repo_name': 'qabbasi/fastai', 'size': '2121'}