<jupyter_start><jupyter_text>Aproach    1. Select the k most central points as seed    2. Select the data points which add the most distance when compared to all the points in the cluster    3. Terminate when the data points run out<jupyter_code>k = 2
C = dict()
for cluster in range(k):
    C[cluster] = pd.DataFrame()<jupyter_output><empty_output><jupyter_text>1. Intialization 1.1 Calculating the mean vector<jupyter_code># Calculating the seed vector

mean_vector = X.mean()<jupyter_output><empty_output><jupyter_text>1.2 Calculating the distance from mean vector and sorting it<jupyter_code>X['dist_mean'] = X.apply(lambda x: np.linalg.norm(np.asarray(x)- np.asarray(mean_vector)), axis=1)
dist_means = X.sort_values(by='dist_mean')
idx_to_drop = dist_means.index[:k]
dist_means.reset_index(drop=True,inplace=True)
X.drop('dist_mean',axis=1,inplace=True)
X.drop(idx_to_drop, inplace=True)
dist_means.head(2)<jupyter_output><empty_output><jupyter_text>1.3 Assigning seed points to cluster<jupyter_code>mu = list()
for cluster in range(k):
    C[cluster] = C[cluster].append(dist_means.iloc[cluster].drop('dist_mean'))
    mu.append(C[cluster].mean())
C[0]<jupyter_output><empty_output><jupyter_text>2. Cluster Assignment<jupyter_code>y = iris.target
columns=iris.feature_names
X = iris.data
df=pd.DataFrame(X,columns=columns)
df['target'] = y
X = df.drop('target',axis=1)

pval = dict()
for cluster in range(k):
    pval[cluster] = dict()
    for i in C[0].columns:
        pval[cluster][i] = list()


for i in tqdm(range(int(len(X)/k)), desc='Iterations: '):
    for cluster in range(k):

        # Calculating the distances from the mean vector of each cluster (in Descending order)
        X['dist_mean'] = X.apply(lambda x: np.linalg.norm(np.asarray(x)- np.asarray(mu[cluster])), axis=1)
        dist_means = X.sort_values(by='dist_mean', ascending=False)
        idx_to_drop = dist_means.index[0]
        dist_means.reset_index(drop=True,inplace=True)
        X.drop('dist_mean',axis=1,inplace=True)

        # Assigning the top value to the cluster
        C[cluster] = C[cluster].append(dist_means.iloc[0].drop('dist_mean'))
        C[cluster] = C[cluster].reset_index(drop=True)
        
        # Updating means of each cluster
        mu[cluster] = C[cluster].mean()

        # Remove datapoint from X?
        X.drop(idx_to_drop,inplace=True)
        
        for i in C[0].columns:
            pval[cluster][i].append(sc.ks_2samp(C[cluster][i],df.drop('target',axis=1)[i])[1])<jupyter_output><empty_output><jupyter_text>3. Plotting the result<jupyter_code>fig, axes = plt.subplots(k, len(C[0].columns), figsize=(15,5))

# Plotting the KS p-value
for cluster in range(k):
    for i,j in enumerate(C[0].columns):
        axes[cluster, i].plot(pval[cluster][j][15:])

# Setting columns headers
for ax, col in zip(axes[0], C[0].columns):
    ax.set_title(col)

# Setting row titles
for ax, row in zip(axes[:,0], range(k)):
    ax.set_ylabel("Cluster {} ".format(row), rotation=90, size='large')
        
plt.tight_layout()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14536585365853658, 'fraction_numerical': 0.012682926829268294, 'mean_word_length': 4.602914389799635, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '412734', 'license': 'MIT', 'n_tokens_mistral': 1150, 'n_tokens_neox': 1116, 'n_words': 229, 'path': 'Examples/WCA KS.ipynb', 'repo_name': 'Anirudhsekar96/WCA', 'size': '3075'}