<jupyter_start><jupyter_text>Normalize<jupyter_code>from sklearn.preprocessing import Normalizer
data = pd.DataFrame(Normalizer().fit_transform(df), columns=df.columns, index = df.index)
data.head()
data.describe().T
data.shape<jupyter_output><empty_output><jupyter_text>1. KMeans Clustering***<jupyter_code>from sklearn.cluster import KMeans
# Finding optimal clusters using Elbow diagram
kpoints = range(2,10)
def draw_elbow(elbow_data):
    inertias = []
    for k in kpoints:
        inertias.append(KMeans(n_clusters=k, init = "k-means++", random_state=42).fit(elbow_data).inertia_)
    plt.figure(figsize = (15, 6))
    plt.plot(kpoints, inertias, "-o")
    plt.title("Number of Clusters vs Distance")
    plt.xlabel("Number of Clusters")
    plt.ylabel("Distance")
    plt.show()
draw_elbow(data)
# Finding optimal clusters using The Silhouette Method (when elbow diagram is not clear)
# The silhouette value measures how similar a point is to its own cluster (cohesion) compared to other clusters (separation).
# The Silhouette Score reaches its global maximum at the optimal k
from sklearn.metrics import silhouette_score
def draw_sil(sil_data):
    sil_scores = []
    for k in kpoints:
        sil_scores.append(silhouette_score(data, KMeans(n_clusters=k, random_state=42, init = "k-means++").fit(sil_data).labels_ , metric = "euclidean"))
    plt.figure(figsize=(15,6))
    plt.plot(kpoints, sil_scores, "-o")
    plt.title("Clusters vs Silhouette Scores")
    plt.xlabel("Number of Clusters")
    plt.ylabel("Silhouette Score") 
    plt.show()
draw_sil(data)<jupyter_output><empty_output><jupyter_text>Optimal k value is 3<jupyter_code># Model 
model_kmeans = KMeans(n_clusters=3, init = "k-means++", random_state=42)
model_kmeans.fit(data)
kmean_clusters = model_kmeans.labels_
output = data.copy()
output["kmeans-cluster"] = kmean_clusters
output.head(3)
sns.pairplot(output, hue = "kmeans-cluster", markers = ["o", "s", "D"])<jupyter_output><empty_output><jupyter_text>2. Agglomerative Clustering***<jupyter_code>from scipy.cluster.hierarchy import dendrogram, linkage
plt.figure(figsize = (15, 6))
plt.title("Dendrogram")
dend = dendrogram(linkage(data, method = "ward"))<jupyter_output><empty_output><jupyter_text>Number of clusters = 3<jupyter_code>from sklearn.cluster import AgglomerativeClustering
model_ag = AgglomerativeClustering(n_clusters= 3, affinity= "euclidean", linkage = "ward").fit(data)
ag_clusters = model_ag.labels_
output["agg-cluster"] = ag_clusters
output.head(3)<jupyter_output><empty_output><jupyter_text>3. PCA***<jupyter_code>from sklearn.preprocessing import MinMaxScaler
scaled_data = pd.DataFrame(MinMaxScaler().fit_transform(df), columns=df.columns, index = df.index)
scaled_data.head()
from sklearn.decomposition import PCA
pca = PCA(n_components= 0.95)
pca.fit(scaled_data)
data_pca = pca.transform(scaled_data)
print(f'Number of Principal Components to explain 95% variance = {pca.n_components_}')<jupyter_output>Number of Principal Components to explain 95% variance = 3<jupyter_text>4. Explained Variance***<jupyter_code>plt.figure(figsize = (10, 6))
x = np.arange(1, pca.n_components_+1 , step = 1)
y = np.cumsum(pca.explained_variance_ratio_)
plt.plot(x, y, marker = "o", linestyle = "--", color = "b")
# horizontal line for 95% cutoff threshold
plt.axhline(y = 0.95, color = 'r', linestyle = "-")
plt.text(1.2, 0.93, "95% cut-off threshold", color = 'b', fontsize = 14)
plt.xticks(x)
plt.grid(axis = 'x')
plt.xlabel('Number of Components')
plt.ylabel('Cumulative Variance (%)')
plt.title("Components vs Explained Variance")
plt.show()<jupyter_output><empty_output><jupyter_text>5. KMeans with Principal Components***<jupyter_code>draw_elbow(data_pca)<jupyter_output><empty_output><jupyter_text>k = 4<jupyter_code># Model 
model_kmeans_pca = KMeans(n_clusters=4, init = "k-means++", random_state=42)
model_kmeans_pca.fit(data_pca)
kmean_clusters_pca = model_kmeans_pca.labels_
# plotting the PCA components
plt.figure(figsize =(8, 6))
plt.scatter(data_pca[:, 0], data_pca[:, 1], c = kmean_clusters_pca)
plt.xlabel('First Principal Component')
plt.ylabel('Second Principal Component')
plt.show()
output["pca-kmeans-cluster"] = kmean_clusters_pca
output.head(3)<jupyter_output><empty_output><jupyter_text>6. Agglomerative Clustering with PCA***<jupyter_code>plt.figure(figsize = (15, 6))
plt.title("Dendrogram")
dend = dendrogram(linkage(data_pca, method = "ward"))<jupyter_output><empty_output><jupyter_text>number of components = 4<jupyter_code>model_ag_pca = AgglomerativeClustering(n_clusters= 4, affinity= "euclidean", linkage = "ward").fit(data_pca)
ag_clusters_pca = model_ag_pca.labels_
output["pca-agg-cluster"] = ag_clusters_pca
output.head(3)<jupyter_output><empty_output><jupyter_text>Result***<jupyter_code>result = output.reset_index().groupby("pca-agg-cluster")["index"].apply(list)
for x in result:
    print(x)<jupyter_output>['Arkansas', 'Connecticut', 'Delaware', 'Hawaii', 'Indiana', 'Kansas', 'Kentucky', 'Massachusetts', 'Missouri', 'Montana', 'New Jersey', 'Ohio', 'Oklahoma', 'Oregon', 'Pennsylvania', 'Rhode Island', 'Utah', 'Virginia', 'Washington', 'Wyoming']
['Alaska', 'Arizona', 'California', 'Colorado', 'Florida', 'Illinois', 'Maryland', 'Michigan', 'Nevada', 'New Mexico', 'New York', 'Texas']
['Idaho', 'Iowa', 'Maine', 'Minnesota', 'Nebraska', 'New Hampshire', 'North Dakota', 'South Dakota', 'Vermont', 'West Virginia', 'Wisconsin']
['Alabama', 'Georgia', 'Louisiana', 'Mississippi', 'North Carolina', 'South Carolina', 'Tennessee']
{'fraction_non_alphanumeric': 0.15054347826086956, 'fraction_numerical': 0.01322463768115942, 'mean_word_length': 7.228017883755589, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '17079', 'license': 'MIT', 'n_tokens_mistral': 2151, 'n_tokens_neox': 1997, 'n_words': 401, 'path': 'notebooks/act-week14-usarrests.ipynb', 'repo_name': 'sreeh4ri/ictk_projects', 'size': '5520'}