<jupyter_start><jupyter_text>Brief data analysis<jupyter_code>data.columns
# check whether a column has a nan value
print('In training set: ')
for col in data.columns:
    if data[col].isnull().values.any():
        print('column {} has NaN value'.format(col))

print('')

print('In testing set')
for col in test_data.columns:
    if test_data[col].isnull().values.any():
        print('column {} has NaN value'.format(col))
data['Survived'].value_counts().reset_index().rename(columns={"index": "Survived_type", "Survived": "counts"})
data[data['Cabin'].apply(lambda x: not isinstance(x, float))][['Cabin', 'Survived']]['Survived'].value_counts()
data[data['Cabin'].apply(lambda x: isinstance(x, float))][['Cabin', 'Survived']]['Survived'].value_counts()<jupyter_output><empty_output><jupyter_text>* It is clear that the exitence of Cabin number can reduce the impurity of data<jupyter_code># create a new column that displays the existence of Cabin number 
data['HasCabinNum'] = data['Cabin'].apply(lambda x: isinstance(x, str))
test_data['HasCabinNum'] = test_data['Cabin'].apply(lambda x: isinstance(x, str))
# convert str type 'male/female' to int type '0/1', 0 for male ,1 for female
data['Sex'] = data['Sex'].apply(lambda x: 0 if x=='male' else 1)
test_data['Sex'] = test_data['Sex'].apply(lambda x: 0 if x=='male' else 1)
# there is a high correlation between 'Ticket' and 'Fare'
# for pair in data[['Ticket', 'Fare', 'Survived']].groupby('Ticket'):
#     display(pair[1])
(data['Age']==0).value_counts()
 (test_data['Age']==0).value_counts()<jupyter_output><empty_output><jupyter_text>* Fill NaN values in column 'Age' with int 0<jupyter_code>data['Age'] = data['Age'].fillna(0)
test_data['Age'] = test_data['Age'].fillna(0)<jupyter_output><empty_output><jupyter_text>* Fill NaN values in column 'Embarked' with 'unknown_location'<jupyter_code>data['Embarked'] = data['Embarked'].fillna('unknown_location')
test_data['Embarked'] = test_data['Embarked'].fillna('unknown_location')
display(data.loc[data['Embarked']=='unknown_location'])
display(test_data.loc[test_data['Embarked']=='unknown_location'])<jupyter_output><empty_output><jupyter_text>* Fill NaN values in column 'Fare' with -1<jupyter_code>data['Fare'] = data['Fare'].fillna(-1)
test_data['Fare'] = test_data['Fare'].fillna(-1)<jupyter_output><empty_output><jupyter_text>model training<jupyter_code>X_train = data[['PassengerId', 'Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked', 'HasCabinNum']]
# onehot encoding (column 'Embarked' is of str type)
X_train = pd.get_dummies(X_train)
y_train = data['Survived']

X_test = test_data[['PassengerId', 'Pclass', 'Sex', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked', 'HasCabinNum']]
X_test = pd.get_dummies(X_test)
# the column 'Embarked_unknown_location' to the X_test
X_test['Embarked_unknown_location'] = [0] * X_test.shape[0]
clf = DecisionTreeClassifier(random_state=0)
clf.fit(X_train.iloc[:, 1:], y_train)
clf.score(X_train.iloc[:, 1:], y_train)
result = pd.DataFrame({"PassengerId": X_test['PassengerId']})
result['Survived'] = clf.predict(X_test.iloc[:, 1:])
submission_name = 'First'
result.to_csv('{}_submission.csv'.format(submission_name), index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18551895587321318, 'fraction_numerical': 0.007147296457426973, 'mean_word_length': 8.223495702005732, 'pattern_counts': {'":': 3, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '197294', 'license': 'MIT', 'n_tokens_mistral': 1234, 'n_tokens_neox': 1184, 'n_words': 234, 'path': 'data_preprocessing.ipynb', 'repo_name': 'Murray132/Kaggle_Titanic', 'size': '3218'}