<jupyter_start><jupyter_text>load functions<jupyter_code>import numpy as np
import pandas as pd
import umap
from sklearn.cluster import KMeans
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.manifold import TSNE
from sklearn.datasets import load_iris
from sklearn.metrics import silhouette_score
# import our modules
from utils import vis_tsne, plot_elbow, vis_umap, d3_tsne, new_d3_umap, d3_umap

def run_kmeans(X, cluster_num, init='random', n_init=10, max_iter=300,tol=1e-4,random_state=40):
    km = KMeans(
            n_clusters=cluster_num, init=init,
            n_init=n_init, max_iter=max_iter, 
            tol=tol, random_state=random_state
        )
    y_km = km.fit_predict(X)
    return km, y_km

def search_k_means_cluster(X, min_num, max_num):
    distortions = []
    silhouette_score_list = []
    for i in range(min_num,max_num):
        km, y_km = run_kmeans(X, i)
        distortions.append(km.inertia_)
        tmp_score = silhouette_score(X, y_km)
        silhouette_score_list.append(tmp_score)
        print("For n_clusters = {}, silhouette score is {}".format(i, tmp_score))
    return distortions, silhouette_score_list<jupyter_output><empty_output><jupyter_text>Select a range of clusters and plot elblow<jupyter_code>"""
min_num, max_num = 4,5
# load the PCA files
X = np.load("pca_good_projection.npy")
print(np.shape(X))
# remove the first row
#X = X[1:,:]
print(np.shape(X))
# test
distortions, silhouette_score_list =search_k_means_cluster(X, min_num, max_num)
plot_elbow(min_num, max_num, distortions, 'distortions', 'ELBOW plot')
plot_elbow(min_num, max_num, silhouette_score_list,'Silhouette', 'Silhouette score plot')
"""<jupyter_output><empty_output><jupyter_text>Visualize clusters based on the optimal number<jupyter_code>#X = np.load("pca_good_projection.npy")
optimal_num = 5
X_train = np.load("train_pca_good_projection.npy", allow_pickle=True)
X_test = np.load("test_pca_good_projection.npy",allow_pickle=True)
#print("Train shape", np.shape(X_train), "Test shape", np.shape(X_test))
print(X_test)

km, y_km_train = run_kmeans(X_train, optimal_num)
y_km_test = km.predict(X_test)
# convert to 1-indexed
y_km_train += 1
y_km_test += 1
all_y_km = np.concatenate((y_km_train, y_km_test), axis=0)
X_all = np.concatenate((X_train, X_test), axis=0)


#all_y_km
#vis_tsne(X, y_km) 
vis_umap(X_train, y_km_train)
%matplotlib notebook
#heat = np.random.normal(0, 0.2, np.shape(X)[0])
#d3_tsne(X, y_km)
reducer = d3_umap(X_train, y_km_train)
new_d3_umap(X_all, all_y_km, reducer)
#print(y_km)
np.save("new_kmeans_membership_%dclusters.npy"%optimal_num, all_y_km)
# cluster membership distribution
"""
num_bins = 50
n, bins, patches = plt.hist(y_km, num_bins, facecolor='blue', alpha=0.5)
plt.title("membership distribution")
plt.show()
"""
#np.where(y_km==2)[0]
#print(y_km)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12137110877929345, 'fraction_numerical': 0.011892270024484085, 'mean_word_length': 5.941747572815534, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '356781', 'license': 'MIT', 'n_tokens_mistral': 1187, 'n_tokens_neox': 1142, 'n_words': 219, 'path': 'src/models/kmeans.ipynb', 'repo_name': 'NCBI-Codeathons/SLEuth', 'size': '2859'}