<commit_before>#%%
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np 
from sklearn import manifold

data = pd.read_csv('data/ml_requests.csv')

data.head()
#%%
items = data['items'].str.split(',')
features_names = set().union(*items)

#%%
features = pd.DataFrame(index=items.index, columns=features_names)

for i in items.index:
    features.iloc[i] = dict.fromkeys(items.iloc[i], 1)

features.fillna(0, inplace=True)
features = features.reindex_axis(sorted(features.columns), axis=1)
features

#%%
X = features
n_components=2

t0 = time()
mds = manifold.MDS(n_components, max_iter=100, n_init=1)
Y = mds.fit_transform(X)
t1 = time()
print("MDS: %.2g sec" % (t1 - t0))
plt.scatter(Y[:, 0], Y[:, 1])
plt.title("MDS (%.2g sec)" % (t1 - t0))
plt.axis('tight')
plt.show()

#%%
t0 = time()
tsne = manifold.TSNE(n_components=n_components, init='pca', random_state=0)
Y = tsne.fit_transform(X)
t1 = time()
print("t-SNE: %.2g sec" % (t1 - t0))
plt.scatter(Y[:, 0], Y[:, 1])
plt.title("t-SNE (%.2g sec)" % (t1 - t0))
plt.axis('tight')

plt.show()<commit_msg>add kmeans clustering<commit_after>#%%
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np 
from sklearn import manifold
from sklearn.cluster import KMeans
from sklearn.cluster import MeanShift 
#%%
data = pd.read_csv('data/ml_requests.csv')

data.head()
items = data['items'].str.split(',')
features_names = set().union(*items)

##%%
features = pd.DataFrame(index=items.index, columns=features_names)

for i in items.index:
    features.iloc[i] = dict.fromkeys(items.iloc[i], 1)

features.fillna(0, inplace=True)
features = features.reindex_axis(sorted(features.columns), axis=1)
features.head()

#%%
X = features
n_components=2

colors = KMeans(n_clusters=2).fit_predict(X)
colors

#%%
t0 = time()
mds = manifold.MDS(n_components, max_iter=100, n_init=1)
Y = mds.fit_transform(X)
t1 = time()
print("MDS: %.2g sec" % (t1 - t0))
plt.scatter(Y[:, 0], Y[:, 1], c=colors, cmap=plt.cm.Spectral)
plt.title("MDS (%.2g sec)" % (t1 - t0))
plt.axis('tight')
plt.show()

#%%
t0 = time()
tsne = manifold.TSNE(n_components=n_components, init='pca', random_state=0)
Y = tsne.fit_transform(X)
t1 = time()
print("t-SNE: %.2g sec" % (t1 - t0))
plt.scatter(Y[:, 0], Y[:, 1], c=colors, cmap=plt.cm.Spectral)
plt.title("t-SNE (%.2g sec)" % (t1 - t0))
plt.axis('tight')
plt.show()
{'fraction_non_alphanumeric': 0.1949765857811835, 'fraction_numerical': 0.025117071094082587, 'mean_word_length': 5.25, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7668123', 'n_tokens_mistral': 1000, 'n_tokens_neox': 966, 'n_words': 206}