<jupyter_start><jupyter_text>Part 1:: Classification<jupyter_code># TRAIN_DATA.replace({'Sex': gender_mapping}, inplace=True)
# TEST_DATA .replace({'Sex': gender_mapping}, inplace=True)

TRAIN_DATA['is_male'] = TRAIN_DATA.apply(lambda row: row.Sex == 'male', axis=1)  
TEST_DATA ['is_male'] = TEST_DATA.apply(lambda row: row.Sex == 'male', axis=1)  

TRAIN_DATA['Embarked_C'] = TRAIN_DATA.apply(lambda row: row['Embarked'] == 'C', axis=1)  
TRAIN_DATA['Embarked_Q'] = TRAIN_DATA.apply(lambda row: row['Embarked'] == 'Q', axis=1)  
TRAIN_DATA['Embarked_S'] = TRAIN_DATA.apply(lambda row: row['Embarked'] == 'S', axis=1)  

TEST_DATA['Embarked_C'] = TEST_DATA.apply(lambda row: row.Embarked == 'C', axis=1)  
TEST_DATA['Embarked_Q'] = TEST_DATA.apply(lambda row: row.Embarked == 'Q', axis=1)  
TEST_DATA['Embarked_S'] = TEST_DATA.apply(lambda row: row.Embarked == 'S', axis=1)  


default_values = {'Age': 20, 'Fare': 20}
TRAIN_DATA.fillna(value=default_values, inplace=True)
TEST_DATA.fillna(value=default_values, inplace=True)

print('Is Data Null', TRAIN_DATA.isna(), TEST_DATA.isna())

# TRAIN_DATA['Sex'].apply(lambda x: 1 if x == 'male' else 0 )
# TEST_DATA ['Sex'].apply(lambda x: 1 if x == 'male' else 0 )
#TRAIN_DATA['Embarked'].apply(lambda x: ord(x))

print("* TRAIN_DATA.head()", TRAIN_DATA.head(), sep="\n", end="\n\n")
print("* TEST_DATA.head()" , TEST_DATA.head(), sep="\n", end="\n\n")
print("Train")
print(TRAIN_DATA)

# features = ['Pclass', 'is_male', 'Age', 'SibSp', 'Parch', 'Fare', 'Embarked_C', 'Embarked_Q', 'Embarked_S']
features = ['Pclass', 'is_male', 'Age', 'Fare', 'Embarked_C', 'Embarked_Q', 'Embarked_S']



y = TRAIN_DATA['Survived']
X = TRAIN_DATA[features]

# y_test = TEST_DATA['Survived']
# X_TEST = TEST_DATA[features]

X.fillna(X.mean(), inplace=True)
print(X.dtypes)

print(X.head())
print(y.head())
print("Nans")
print(X)
from sklearn import tree

dt1 = tree.DecisionTreeClassifier(min_samples_split=100, random_state=99, max_depth=5)
dt1.fit(X, y)



dt2 = tree.DecisionTreeClassifier()
dt2.fit(X, y)


import graphviz
targ_names = ['Yes','No']

data = tree.export_graphviz(dt1,out_file=None,feature_names=features,class_names=targ_names,   
                         filled=True, rounded=True,  
                         special_characters=True)
graph = graphviz.Source(data)
graph
# targ_names = ['Yes','No']

# data = export_graphviz(dt2,out_file=None,feature_names=features,class_names=targ_names,   
#                          filled=True, rounded=True,  
#                          special_characters=True)
# graph = graphviz.Source(data)
# graph
# y_pred=dt1.predict(X_TEST)<jupyter_output><empty_output><jupyter_text>Part 2:: Visualization<jupyter_code>TRAIN_DATA.head()
# plt.figure(figsize=(10,6))
sns.countplot(x='Embarked', data=TRAIN_DATA)
TRAIN_DATA['Age'].hist(bins=16)
corr=TRAIN_DATA.corr()
print(corr)
sns.heatmap(corr)
sns.scatterplot(x=TRAIN_DATA['Age'], y=TRAIN_DATA['Fare'], hue=TRAIN_DATA['Survived'])<jupyter_output><empty_output><jupyter_text>Part 3:: Clustering<jupyter_code>from sklearn.cluster import KMeans

kmeans = KMeans(n_clusters=30)

columns = ['Age', 'Fare']
kmeans.fit(X[columns])
y_kmeans = kmeans.predict(X[columns])

#y_kmeans

import matplotlib.pyplot as plt
import seaborn as sns; sns.set()

from sklearn.datasets.samples_generator import make_blobs

plt.scatter(X['Age'], X['Fare'],  s=10)
centers = kmeans.cluster_centers_
plt.scatter(centers[:, 0], centers[:, 1], c='black', s=200, alpha=0.5);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1782658300057045, 'fraction_numerical': 0.013690815744438107, 'mean_word_length': 6.028056112224449, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '479007', 'license': 'MIT', 'n_tokens_mistral': 1472, 'n_tokens_neox': 1389, 'n_words': 226, 'path': 'DW_DM/dw-dm-sem-7.ipynb', 'repo_name': 'shrey-c/files', 'size': '3506'}