<jupyter_start><jupyter_text>This demonstrates scikit-learn clustering for comparison with Tribuo clustering<jupyter_code>import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.cluster import KMeans
from sklearn.cluster import AgglomerativeClustering
from sklearn.cluster import MeanShift
from sklearn.cluster import DBSCAN
from sklearn.metrics import adjusted_mutual_info_score
from sklearn.metrics import normalized_mutual_info_score
# This dataset is generated in the notebook: scikit-learn Clustering - Data Setup
df = pd.read_csv('../../data/gaussianBlobs.csv')
# print(df)
df_X = df.drop(['Cluster'], axis=1)
df_y = pd.DataFrame(df[['Cluster']])

X_train, X_test, y_train, y_test = train_test_split(df_X.values, df_y.values, test_size=0.2, random_state=1)
print('Training data size = %d, number of features = %d' % (len(X_train), len(df_X.columns)))
print('Testing data size = %d, number of features = %d' % (len(X_test), len(df_X.columns)))
def evaluate(actual, predicted):
    print('Clustering Evaluation')
    print('Normalized MI = %.2f' % normalized_mutual_info_score(actual, predicted))
    print('Adjusted MI = %.2f' % adjusted_mutual_info_score(actual, predicted))
km = KMeans(n_clusters=6, max_iter=100, n_jobs=4, random_state=1, init='random')
km_plus_plus = KMeans(n_clusters=6, max_iter=100, n_jobs=4, random_state=1, init='k-means++')
# This crashes the kernel everytime
# ag = AgglomerativeClustering(n_clusters=6)
# This doesn't finish after a reasonable amount of time
# ms = MeanShift(n_jobs=4)
# This also crashes the kernel everytime
# dbscan = DBSCAN(eps=3, min_samples=50, n_jobs=4)
print(km)
print(km_plus_plus)
# print(ag)
# print(ms)
# print(dbscan)
%time km.fit(X_train)
# run 1
# time:  17.4 s

# run 2
# time:  17.7 s

# run 3
# time:  16.0 s
predicted = km.predict(X_test)
evaluate(y_test.ravel(), predicted)
# run 1
# Normalized MI = 1.00
# Adjusted MI = 1.00

# run 2
# Normalized MI = 1.00
# Adjusted MI = 1.00

# run 3
# Normalized MI = 1.00
# Adjusted MI = 1.00
%time km_plus_plus.fit(X_train)
# run 1
# time:  19.2 s

# run 2
# time:  18.6 s

# run 3
# time:  17.9 s
predicted = km_plus_plus.predict(X_test)
evaluate(y_test.ravel(), predicted)

# run 1
# Normalized MI = 1.00
# Adjusted MI = 1.00

# run 2
# Normalized MI = 1.00
# Adjusted MI = 1.00

# run 3
# Normalized MI = 1.00
# Adjusted MI = 1.00
# %time ag.fit(X_train)
# predicted = ag.predict(X_test)
# evaluate(y_test.ravel(), predicted)
# %time ms.fit(X_train)
# predicted = ms.predict(X_test)
# evaluate(y_test.ravel(), predicted)
# %time dbscan.fit(X_train)
# predicted = dbscan.predict(X_test)
# evaluate(y_test.ravel(), predicted)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13014460511679643, 'fraction_numerical': 0.03337041156840934, 'mean_word_length': 5.24537037037037, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '504344', 'license': 'Apache-2.0', 'n_tokens_mistral': 1083, 'n_tokens_neox': 1000, 'n_words': 255, 'path': 'notebooks/clustering/scikit-learn Clustering.ipynb', 'repo_name': 'geoffreydstewart/JavaVsPythonMLlibs', 'size': '2697'}