<jupyter_start><jupyter_text>Cleaning Data<jupyter_code>keep_cols = [ 'TotalCaserPerM','DeathsPerM', 'TestPerM' ,'DeathRate%']
# keep_cols = [ 'DeathsPerM', 'TestPerM' ,'DeathRate%']

display_cols = ['Country', 'TotalCases'] + keep_cols
covidCluster = covid.copy()
covidCluster[display_cols].head()
covidCluster[covidCluster[display_cols].isna().any(axis = 1)][display_cols].head()
covidCluster = covidCluster[display_cols].dropna()
covidCluster = covidCluster[covidCluster.Country != 'San Marino']
# covidCluster = covidCluster.head(100)
# covidCluster.head(100)<jupyter_output><empty_output><jupyter_text>Visualizaing data<jupyter_code>sns.pairplot(covidCluster)<jupyter_output><empty_output><jupyter_text>Kmeans<jupyter_code>X = covidCluster[keep_cols]
# from sklearn.impute import KNNImputer
# imputer = KNNImputer(n_neighbors=2)
# covidImputed = imputer.fit_transform(covidCluster)
# pd.DataFrame(covidImputed)
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
X = scaler.fit_transform(X)
pd.DataFrame(X).head()<jupyter_output><empty_output><jupyter_text>Find optimal Number of Clusters<jupyter_code>kmeans_per_k = [KMeans( n_clusters= k, random_state= 42).fit(X) for k in range(1, 15)]
inertias = [model.inertia_ for model in kmeans_per_k]

plt.figure(figsize=(8, 3.5))
plt.plot(range(1, 15), inertias, "bo-")
plt.xlabel("$k$", fontsize=14)
plt.ylabel("Inertia", fontsize=14)
# plt.annotate('Elbow',
#              xy=(4, inertias[3]),
#              xytext=(0.55, 0.55),
#              textcoords='figure fraction',
#              fontsize=16,
#              arrowprops=dict(facecolor='black', shrink=0.1)
#             )
plt.axis([1, 15, 0, 1300])
# save_fig("inertia_vs_k_plot")
plt.show()
from sklearn.metrics import silhouette_score
silhouette_scores = [silhouette_score(X, model.labels_)
                     for model in kmeans_per_k[1:]]
plt.figure(figsize=(8, 3))
plt.plot(range(2, 15), silhouette_scores, "bo-")
plt.xlabel("$k$", fontsize=14)
plt.ylabel("Silhouette score", fontsize=14)
# plt.axis([1.8, 8.5, 0.55, 0.7])
# save_fig("silhouette_score_vs_k_plot")
plt.show()<jupyter_output><empty_output><jupyter_text>Run Kmeans for Optimal Clusters<jupyter_code>k = 3
kmeans = KMeans(n_clusters = k, random_state= 42)
y_pred = kmeans.fit_predict(X)
y_pred

covidCluster['cluster'] = y_pred
covidCluster = covidCluster.sort_values(by = ['cluster', 'TotalCases'], ascending = False)
covidCluster.head()
agg = {
        'Count': pd.NamedAgg(column = 'cluster', aggfunc = 'count'),
        'MeanTotalCaserPerM':pd.NamedAgg(column = 'TotalCaserPerM', aggfunc = 'mean'),
        'MeanDeathsPerM':pd.NamedAgg(column = 'DeathsPerM', aggfunc = 'mean'),
        'MeanTestPerM':pd.NamedAgg(column = 'TestPerM', aggfunc = 'mean'),
        'MeanDeathRate%':pd.NamedAgg(column = 'DeathRate%', aggfunc = 'mean'),
        
       
}
# group = covidCluster.groupby(['cluster']).agg({'cluster': 'count',  'TotalCaserPerM': 'mean', 'DeathsPerM': 'mean', 'TestPerM': 'mean', 'DeathRate':'mean'} ).rename(columns = ['1', '2', '3', '4', '5', '6'])
group = covidCluster.groupby('cluster').agg(**agg)
group.round(2)<jupyter_output><empty_output><jupyter_text>Cluster 0<jupyter_code>covidCluster[covidCluster.cluster == 0]<jupyter_output><empty_output><jupyter_text>Cluster 1<jupyter_code>covidCluster[covidCluster.cluster == 1]<jupyter_output><empty_output><jupyter_text>Cluster 2<jupyter_code>covidCluster[covidCluster.cluster == 2]
covidCluster[covidCluster.cluster == 3]
covidCluster.shape
covidCluster.to_csv('clusters.csv')<jupyter_output><empty_output><jupyter_text>DBSCAN<jupyter_code>from sklearn.cluster import DBSCAN
dbscan = DBSCAN(eps = 2, min_samples = 5)
dbscan.fit(X)
covidCluster['cluster'] = dbscan.labels_
covidCluster = covidCluster.sort_values(by = ['cluster', 'TotalCases'], ascending = False)
covidCluster.head()<jupyter_output><empty_output><jupyter_text>AgglomerativeClustering<jupyter_code>from sklearn.cluster import AgglomerativeClustering
aggcluster = AgglomerativeClustering(n_clusters = 3, affinity= 'euclidean', linkage = 'complete')
y_pred = aggcluster.fit_predict(X)
covidCluster['cluster'] = y_pred
covidCluster = covidCluster.sort_values(by = ['cluster', 'TotalCases'], ascending = False)
covidCluster.head()
group = covidCluster.groupby('cluster').agg(**agg)
group.round(2)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1563719862227325, 'fraction_numerical': 0.019058553386911595, 'mean_word_length': 6.83453237410072, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '442055', 'license': 'MIT', 'n_tokens_mistral': 1730, 'n_tokens_neox': 1612, 'n_words': 237, 'path': 'CovidCluster.ipynb', 'repo_name': 'ajayrawatsap/covid-19', 'size': '4355'}