<jupyter_start><jupyter_text>import data<jupyter_code>df = pd.read_csv('cluster.csv', header = None)

# convert dataframe into numpy array
X = df.values
X.shape<jupyter_output><empty_output><jupyter_text>Standardize the data points<jupyter_code>from sklearn.preprocessing import StandardScaler

sc = StandardScaler()
X_sc = sc.fit_transform(X)<jupyter_output><empty_output><jupyter_text>Define a function to calculate distances to centroids for all data points - can be further optimized (with the np.vectorize function etc.)<jupyter_code>def distance(points1, points2):
    distances = np.zeros(shape=(points1.shape[0], points2.shape[0]))
    distanceToCentroids = lambda point: \
        np.array([np.linalg.norm((point - centroid), ord=1) for centroid in points2],dtype = float)

    # Loop over data points:
    for i in range(points1.shape[0]):
        distances[i,] = distanceToCentroids(points1[i, ])
    
    return(distances)<jupyter_output><empty_output><jupyter_text>Define the function to iteratively get find the cluster centroids<jupyter_code>def k_centroids(X, n_clusters, init='k-means++'
                , precompute_distances='auto', n_init=10
                , max_iter=1000, tol=1e-4
#                 , random_state=None, n_jobs=1
#                 ,algorithm="auto", return_n_iter=False
               ):
    # X : array-like matrix, already standardized
    
    n_samples = X.shape[0]
    n_features = X.shape[1]
    
    # initialize the centroids
    # random sampling, can use other methods like kmeans++
    centroids = X[random.sample(range(n_samples), n_clusters), ]
    
    # calculate the distances to clusters, clustering labels and the resulting inertia
    distances = distance(X, centroids)
    labels = np.argmin(distances, axis = 1)
    inertia = np.sum((X - centroids[labels])**2, dtype=np.float64)
    
    # initiate an array to store the silhouette scores for all data points
    silhouettes = np.zeros(shape=(n_samples, )) 
    
    for iteration in range(max_iter):
        
        n_samples_in_cluster = np.bincount(labels, minlength = n_clusters)

        # calculate the new centroids by taking the medians of data points in the same cluster
       
        centroids = np.zeros(shape = (n_clusters, n_features))
        
        for j in range(n_features):
            adic = defaultdict(list)
            for i in range(n_samples):
                adic[labels[i]].append(X[i, j])
            for each_label in np.unique(labels):
                centroids[each_label, j] = statistics.median(adic[each_label])

        distances = distance(X, centroids)
        labels = np.argmin(distances, axis = 1)
        inertia_previous = inertia
        inertia = np.sum((X - centroids[labels])**2, dtype=np.float64)
        
        # calculate the silhouette scores for each data point based on the new clusters
        for i in range(n_samples):
            a_i = np.average(np.ma.masked_equal(distance(X[labels == labels[i,], ], X[i:i+1,]),0))
            b_i = np.min(distance(X[labels != labels[i,], ], X[i:i+1,]))
            silhouettes[i,] = (b_i - a_i)/np.max([a_i, b_i])

        if (np.linalg.norm(inertia - inertia_previous) < tol):
            break
        
    return(centroids, inertia, labels, silhouettes)
centroids, inertia, labels, silhouettes = k_centroids(X_sc, 3)
from sklearn.metrics import silhouette_samples, silhouette_score
silhouette_avg = silhouette_score(X_sc, labels)
sample_silhouette_values = silhouette_samples(X_sc, labels)
silhouette_avg
np.average(silhouettes)
np.vstack((labels, silhouettes))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10827318156579678, 'fraction_numerical': 0.009439200444197668, 'mean_word_length': 3.294398092967819, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '415133', 'license': 'MIT', 'n_tokens_mistral': 1209, 'n_tokens_neox': 1167, 'n_words': 267, 'path': 'algorithms/K-medians clustering.ipynb', 'repo_name': 'mbzhuang/CustomerSegmentationforVAProducts', 'size': '3602'}