<jupyter_start><jupyter_text>finding correlation among fields for checking and rooting out multi colnearity<jupyter_code>plt.figure(figsize = (9,9))
sns.heatmap(rest_numbers_list.corr(),vmax=1,annot = True)<jupyter_output><empty_output><jupyter_text>normalizing dataset<jupyter_code>#normalizing below the dataset
from sklearn.preprocessing import StandardScaler
scaler = StandardScaler()
rest_numbers_list_fit = scaler.fit_transform(rest_numbers_list)
#Improting the PCA module
from sklearn.decomposition import PCA
pca = PCA(svd_solver='randomized', random_state=42)
#Performing here the PCA on the train data
pca.fit(rest_numbers_list_fit)
pca.components_
column_names = list(rest_numbers_list.columns)
df_pca = pd.DataFrame({'PC1':pca.components_[0],'PC2':pca.components_[1],'PC3':pca.components_[2],'PC4':pca.components_[3], 'Feature':column_names})
df_pca.head()
country = pd.concat([country_data], axis=1)
country.head()
%matplotlib inline
fig = plt.figure(figsize = (8,8))
plt.scatter(df_pca.PC1, df_pca.PC2)
plt.xlabel('PC 1')
plt.ylabel('PC 2')
for i, txt in enumerate(df_pca.Feature):
    plt.annotate(txt, (df_pca.PC1[i],df_pca.PC2[i]))
plt.tight_layout()
plt.show()
pca.explained_variance_ratio_
#Making the screeplot - plotting of the cumulative variance against the no. of components
%matplotlib inline
fig = plt.figure(figsize = (8,8))
plt.plot(np.cumsum(pca.explained_variance_ratio_))
plt.xlabel('no. of Components')
plt.ylabel('cumulative explained variance')
plt.show()<jupyter_output><empty_output><jupyter_text>Looks like 4 components are enough to describe above 90% of the variance in the dataset We'll choose 4 components for our modeling<jupyter_code>#Using incremental PCA for efficiency - as we all know this saves a lot of time on larger datasets(in general)
from sklearn.decomposition import IncrementalPCA
pca_final = IncrementalPCA(n_components=4)

df_train_pca = pca_final.fit_transform(rest_numbers_list_fit)
df_train_pca.shape
x_df=pd.DataFrame(df_train_pca)

x_df.head()
df_pca2=pd.merge(x_df , pd.DataFrame(country), left_index=True,right_index=True)
df_pca2.head()
# Outlier treatment : imputing the outlier will impact the analysis; hence droping them makes more sense 
# as we used in earlier assignments; using similar function

def remove_outlier(df,col_name):
    q1 = df[col_name].quantile(0.25)
    q2 = df[col_name].quantile(0.75)
    iqr = q2-q1
    fence_low = q1-1.5*iqr
    fence_high = q2+1.5*iqr
    df_out = df.loc[(df[col_name]>fence_low) & (df[col_name]<fence_high)]
    return df_out
pcs=remove_outlier(x_df,0)
x_df
x_df=remove_outlier(x_df,1)
x_df
x_df=remove_outlier(x_df,2)
x_df
x_df=remove_outlier(x_df,3)
x_df.head()
#Set the 5 clustering

from sklearn.cluster import KMeans
kmeans=KMeans(n_clusters=5)
clusters = kmeans.fit(x_df)
clusters
x_df['ClusterId'] = pd.Series(clusters.labels_, index=x_df.index)
x_df['ClusterId'].head()
country_merged=pd.merge(pd.DataFrame(x_df['ClusterId']) , pd.DataFrame(country), left_index=True,right_index=True)
country_merged
country_updated=country_merged.drop(["country"], axis = 1, inplace = False)<jupyter_output><empty_output><jupyter_text>performing Hopkins below<jupyter_code>from sklearn.neighbors import NearestNeighbors
from random import sample
from numpy.random import uniform
import numpy as np
from math import isnan
 
def hopkins(X):
    d = X.shape[1]
    n = len(X) # rows
    m = int(0.1 * n) 
    nbrs = NearestNeighbors(n_neighbors=1).fit(X.values)
 
    rand_X = sample(range(0, n, 1), m)
 
    ud = []
    wd = []
    for j in range(0, m):
        u_dist, _ = nbrs.kneighbors(uniform(np.amin(X,axis=0),np.amax(X,axis=0),d).reshape(1, -1), 2, return_distance=True)
        ud.append(u_dist[0][1])
        w_dist, _ = nbrs.kneighbors(X.iloc[rand_X[j]].values.reshape(1, -1), 2, return_distance=True)
        wd.append(w_dist[0][1])
 
    S = sum(ud) / (sum(ud) + sum(wd))
    if isnan(S):
        print(ud, wd)
        S = 0
 
    return S
hopkins(country_updated)
## Silhouette Analysis

from sklearn.metrics import silhouette_score
sa_ = []
for k in range(2, 15):
    kmeans = KMeans(n_clusters=k).fit(x_df)
    sa_.append([k, silhouette_score(country_updated, kmeans.labels_)])
plt.plot(pd.DataFrame(sa_)[0], pd.DataFrame(sa_)[1]);
# sum of squared distances
sd = []
for num_clusters in list(range(1,21)):
    mod_clus = KMeans(n_clusters = num_clusters, max_iter=50)
    mod_clus.fit(x_df)
    sd.append(mod_clus.inertia_)

plt.plot(sd)
# analysis of clusters formed
country_updated.index = pd.RangeIndex(len(country_updated.index))
country_km = pd.concat([country_updated, pd.Series(mod_clus.labels_)], axis=1)
country_km.columns 

cluster_km_child_mort = pd.DataFrame(country_km.groupby(["ClusterId"]).child_mort.mean())
cluster_km_exports = pd.DataFrame(country_km.groupby(["ClusterId"]).exports.mean())
cluster_km_health = pd.DataFrame(country_km.groupby(["ClusterId"]).health.mean())

cluster_km_imports = pd.DataFrame(country_km.groupby(["ClusterId"]).imports.mean())
cluster_km_income = pd.DataFrame(country_km.groupby(["ClusterId"]).income .mean())
cluster_km_inflation = pd.DataFrame(country_km.groupby(["ClusterId"]).inflation.mean())

cluster_km_life_expec= pd.DataFrame(country_km.groupby(["ClusterId"]).life_expec.mean())
cluster_km_total_fer = pd.DataFrame(country_km.groupby(["ClusterId"]).total_fer .mean())
cluster_km_gdpp = pd.DataFrame(country_km.groupby(["ClusterId"]).gdpp.mean())
df = pd.concat([pd.Series([0,1,2,3,4]),cluster_km_child_mort,cluster_km_exports,cluster_km_health,cluster_km_income,cluster_km_imports,cluster_km_inflation,cluster_km_life_expec,cluster_km_total_fer,cluster_km_gdpp], axis=1)
df.columns = ['ClusterId', 'child_mort','exports', 'health', 'imports' ,'income', 'inflation', 'life_expec', 'total_fer', 'gdpp']
df.head()
#plotting graphs to see 
sns.barplot(x=df.ClusterId,y=df.child_mort)
sns.barplot(x=df.ClusterId,y=df.exports)
sns.barplot(x=df.ClusterId,y=df.health)
sns.barplot(x=df.ClusterId,y=df.income)
sns.barplot(x=df.ClusterId,y=df.inflation)
sns.barplot(x=df.ClusterId,y=df.imports)
sns.barplot(x=df.ClusterId,y=df.life_expec)
sns.barplot(x=df.ClusterId,y=df.total_fer)
sns.barplot(x=df.ClusterId,y=df.gdpp)
country_merged.loc[country_merged['ClusterId']==4]
# heirarchical clustering -- along with K meanss as per problem statement

merging = linkage(x_df, method = "single", metric='euclidean')
dendrogram(merging)
plt.show()
#Showing dendogram as per clusters formed
merging = linkage(x_df, method = "complete", metric='euclidean')
dendrogram(merging)
plt.show()
Cluster_Cut = pd.Series(cut_tree(merging, n_clusters = 5).reshape(-1,))
country_hc = pd.concat([country_merged, Cluster_Cut], axis=1)
country_hc.columns
#summarise the whole mean data as shown below

cluster_km_child_mort = pd.DataFrame(country_hc.groupby(["ClusterId"]).child_mort.mean())
cluster_km_exports = pd.DataFrame(country_hc.groupby(["ClusterId"]).exports.mean())
cluster_km_health = pd.DataFrame(country_hc.groupby(["ClusterId"]).health .mean())
cluster_km_imports = pd.DataFrame(country_hc.groupby(["ClusterId"]).imports.mean())
cluster_km_income= pd.DataFrame(country_hc.groupby(["ClusterId"]).income.mean())
cluster_km_inflation = pd.DataFrame(country_hc.groupby(["ClusterId"]).inflation .mean())
cluster_km_life_expec = pd.DataFrame(country_hc.groupby(["ClusterId"]).life_expec.mean())
cluster_km_total_fer= pd.DataFrame(country_hc.groupby(["ClusterId"]).total_fer.mean())
cluster_km_gdpp = pd.DataFrame(country_hc.groupby(["ClusterId"]).gdpp .mean())
# catcatinating newly columns with cluster means
df = pd.concat([pd.Series([0,1,2,3,4]), cluster_km_child_mort,cluster_km_exports,cluster_km_health,cluster_km_income,cluster_km_imports,cluster_km_inflation,cluster_km_life_expec,cluster_km_total_fer,cluster_km_gdpp], axis=1)
df.columns = ['ClusterId', 'child_mort','exports', 'health', 'imports' ,'income', 'inflation', 'life_expec', 'total_fer', 'gdpp']
df.head()

#plotting again
sns.barplot(x=df.ClusterId,y=df.child_mort)
sns.barplot(x=df.ClusterId,y=df.exports)
sns.barplot(x=df.ClusterId,y=df.health)
sns.barplot(x=df.ClusterId,y=df.imports)
sns.barplot(x=df.ClusterId,y=df.income)
sns.barplot(x=df.ClusterId,y=df.inflation)
sns.barplot(x=df.ClusterId,y=df.life_expec)
sns.barplot(x=df.ClusterId,y=df.total_fer)
sns.barplot(x=df.ClusterId,y=df.gdpp)
required_data=country_merged.loc[country_merged['ClusterId']==4]
required_data.shape
required_data.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.143989588263133, 'fraction_numerical': 0.012068149550402271, 'mean_word_length': 7.32807881773399, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '469059', 'license': 'MIT', 'n_tokens_mistral': 3361, 'n_tokens_neox': 3219, 'n_words': 500, 'path': 'Clustering_&_PCA_Assignment.ipynb', 'repo_name': 'saurabdongre/Clustering-and-PCA', 'size': '8452'}