<jupyter_start><jupyter_text>Clustering attempts.<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import pandas as pd
df = pd.read_csv("mc_san_gaia_lam.csv")
df
plt.plot(df.vx, df.vy, "k.", ms=1, alpha=.2)
plt.xlim(-100, 150)
plt.ylim(100, 350)
plt.xlabel("vx")
plt.ylabel("vy")
plt.plot(df.vx, df.vz, "k.", ms=1, alpha=.2)
plt.xlim(-100, 150)
plt.ylim(-100, 100)
plt.xlabel("vx")
plt.ylabel("vz")
plt.plot(df.color_teffs, df.Prot, "k.", ms=1, alpha=.2)
plt.xlim(8000, 3000)
# plt.ylim(100, 350)
plt.xlabel("Teff")
plt.ylabel("Prot")
plt.yscale("log")
print(__doc__)

# Author: <NAME> <<EMAIL>>
# License: BSD 3 clause

import numpy as np
import matplotlib.pyplot as plt

from sklearn.cluster import KMeans
from sklearn.datasets import make_blobs

plt.figure(figsize=(12, 12))

n_samples = 1500
random_state = 170
X, y = make_blobs(n_samples=n_samples, random_state=random_state)
print(np.shape(X), np.shape(y))
print(X[:1])
print(y)

# Incorrect number of clusters
y_pred = KMeans(n_clusters=2, random_state=random_state).fit_predict(X)

plt.subplot(221)
plt.scatter(X[:, 0], X[:, 1], c=y_pred)
plt.title("Incorrect Number of Blobs")

# Anisotropicly distributed data
transformation = [[0.60834549, -0.63667341], [-0.40887718, 0.85253229]]
X_aniso = np.dot(X, transformation)
y_pred = KMeans(n_clusters=3, random_state=random_state).fit_predict(X_aniso)

plt.subplot(222)
plt.scatter(X_aniso[:, 0], X_aniso[:, 1], c=y_pred)
plt.title("Anisotropicly Distributed Blobs")

# Different variance
X_varied, y_varied = make_blobs(n_samples=n_samples,
                                cluster_std=[1.0, 2.5, 0.5],
                                random_state=random_state)
y_pred = KMeans(n_clusters=3, random_state=random_state).fit_predict(X_varied)

plt.subplot(223)
plt.scatter(X_varied[:, 0], X_varied[:, 1], c=y_pred)
plt.title("Unequal Variance")

# Unevenly sized blobs
X_filtered = np.vstack((X[y == 0][:500], X[y == 1][:100], X[y == 2][:10]))
y_pred = KMeans(n_clusters=3,
                random_state=random_state).fit_predict(X_filtered)

plt.subplot(224)
plt.scatter(X_filtered[:, 0], X_filtered[:, 1], c=y_pred)
plt.title("Unevenly Sized Blobs")

plt.show()
m = np.isfinite(df.vx.values) & np.isfinite(df.vy.values) & np.isfinite(df.vz.values)
m &= (-100 < df.vx.values) & (df.vx.values < 150)
m &= (100 < df.vy.values) & (df.vy.values < 350)
m &= (-100 < df.vz.values) & (df.vz.values < 100)
vx, vy, vz = df.vx.values[m], df.vy.values[m], df.vz.values[m]
from sklearn.cluster import KMeans
from sklearn.datasets import make_blobs

plt.figure(figsize=(12, 12))

X = np.vstack((vx, vy, vz)).T
print(np.shape(X))

y_pred = KMeans(n_clusters=3, random_state=random_state).fit_predict(X)

plt.subplot(221)
plt.scatter(X[:, 0], X[:, 1], c=y_pred, s=1)
from sklearn import mixture
from matplotlib.colors import LogNorm

X_train = np.vstack((vx, vy)).T
print(np.shape(X_train))

# fit a Gaussian Mixture Model with two components
clf = mixture.GaussianMixture(n_components=4, covariance_type='full')
clf.fit(X_train)

# display predicted scores by the model as a contour plot
x = np.linspace(-100., 150.)
y = np.linspace(100., 350.)
X, Y = np.meshgrid(x, y)
XX = np.array([X.ravel(), Y.ravel()]).T
Z = -clf.score_samples(XX)
Z = Z.reshape(X.shape)

CS = plt.contour(X, Y, Z, norm=LogNorm(vmin=1.0, vmax=1000.0),
                 levels=np.logspace(0, 3, 10))
CB = plt.colorbar(CS, shrink=0.8, extend='both')
plt.scatter(X_train[:, 0], X_train[:, 1], .8)<jupyter_output>(11010, 2)
{'fraction_non_alphanumeric': 0.17648733276401934, 'fraction_numerical': 0.05636208368915457, 'mean_word_length': 4.846921797004992, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '441783', 'license': 'MIT', 'n_tokens_mistral': 1647, 'n_tokens_neox': 1465, 'n_words': 293, 'path': 'Clustering.ipynb', 'repo_name': 'RuthAngus/edr3_hack', 'size': '3531'}