<jupyter_start><jupyter_text>GMM random seed<jupyter_code>t0 = time.time()
pipe19_random = Pipeline([
  (f'avgw2v_tfidf', embedding_vectorizer()),
  ('gmm', GaussianMixture(n_components=19,
                    init_params='random',
                    verbose=10))
])
print("Computing clusters...", end='')
pipe19_random.fit(np.array(all_data))
print("done")
print("%0.2fs" % (time.time() - t0))
gmm19_rnd = pipe19_random['gmm']
X_labeled19_rnd = pipe19_random['avgw2v_tfidf'].transform(np.array(labeled.data))
probs19_rnd = gmm19_rnd.predict_proba(X_labeled19_rnd)
probs19_rnd
df = pd.DataFrame(probs19_rnd)
df['y'] = labeled.labels
candidates_probs_rnd = df.groupby('y').sum()

# Sort by dems
candidates_probs_rnd['is_dem'] = is_dem
candidates_probs_rnd.sort_values(by='is_dem', ascending=False, inplace=True)
candidates_probs_rnd.pop('is_dem')
candidates_probs_rnd
candidates_probs_rnd_norm = normalize(candidates_probs_rnd, norm='l1', axis=0)
mapping = pd.DataFrame([np.random.choice(candidates_probs_rnd.index, p=col) for col in candidates_probs_rnd_norm.T])
mapping
# candidates_probs_rnd.values
centroid_predictions = pipe19_random.predict(unlabeled.data[:10])
centroid_predictions
unlabeled_labels = pd.Series(a0).map(mapping.squeeze())
unlabeled_labels
pd.DataFrame(dict(
    fname=list(map(lambda fname: f'unlabeled/{fname}', unlabeled.fnames[:10])),
    label=unlabeled_labels
))
list(map(lambda fname: f'unlabeled/{fname}', unlabeled.fnames))
candidates_probs_rnd.sum(axis=1)
unlabeled.y = pipe19_random.predict(unlabeled.data)
unlabeled.y
t0 = time.time()
pipe19_kmeans = Pipeline([
  (f'avgw2v_tfidf', embedding_vectorizer()),
  ('kmeans', KMeans(n_clusters=19,
                    init='k-means++',
                    verbose=10,
                    n_jobs=-1))
])
print("Computing clusters...", end='')
pipe19_kmeans.fit(np.array(all_data))
print("done")
print("%0.2fs" % (time.time() - t0))
cluster_predictions_19_kmeans = pipe19_kmeans.transform(labeled.data)
cluster_predictions_19_kmeans = normalize(cluster_predictions_19_kmeans, norm='l1')
df = pd.DataFrame(cluster_predictions_19_kmeans)
df['y'] = labeled.labels
candidates_probs_kmeans = df.groupby('y').sum()
candidates_probs_kmeans

# Sort by dems
candidates_probs_kmeans['is_dem'] = is_dem
candidates_probs_kmeans.sort_values(by='is_dem', ascending=False, inplace=True)
candidates_probs_kmeans.pop('is_dem')
candidates_probs_kmeans
candidates_probs_kmeans.sum(axis=1)
fig, axs = plt.subplots(2, 1, figsize=(10,10))

im = axs[0].imshow(candidates_probs_kmeans)
fig.colorbar(im, ax=axs[0])
axs[0].set_xticks([])
axs[0].set_yticks(range(19))
axs[0].set_xlabel('centroids')
axs[0].set_ylabel('candidates')
axs[0].set_yticklabels(candidates_probs_rnd.index.values)
axs[0].set_title('K-Means')

im = axs[1].imshow(candidates_probs_rnd)
fig.colorbar(im, ax=axs[1])
axs[1].set_xticks([])
axs[1].set_yticks(range(19))
axs[1].set_xlabel('centroids')
axs[1].set_ylabel('candidates')
axs[1].set_yticklabels(candidates_probs_rnd.index.values)
axs[1].set_title('GMM')

plt.imshow(candidates_probs_kmeans)
plt.colorbar()<jupyter_output><empty_output><jupyter_text>GMM kmeans seed  GMM labeled seed<jupyter_code># Compute embeddings of labeled data to use as seed for means
embedding_vectorizer_glove = lambda : TfidfEmbeddingVectorizer(word2vec_glove, tokenizer=nltk.word_tokenize)
X_labeled_glove = embedding_vectorizer_glove().fit_transform(np.array(labeled.data))
df = pd.DataFrame(X_labeled_glove)
df['y'] = labeled.labels
labeled_centroids = df.groupby('y').mean()
labeled_centroids
cosine_similarities = pd.DataFrame(cosine_similarity(labeled_centroids.values)).set_index(labeled_centroids.index)
cosine_similarities
dotproduct_similarities = labeled_centroids.dot(labeled_centroids.T)
dotproduct_similarities
# Determine weights proportional to affiliation from labels
def weights_from_labeled():
    df = pd.DataFrame(labeled.labels)
    df['a'] = 1
    return df.groupby(0).sum() / len(labeled.labels)
pipe_19_glove_labelinit = Pipeline([
  (f'avgw2v_tfidf', embedding_vectorizer_glove()),
  ('gmm', GaussianMixture(n_components=19,
                    init_params='random',
                    weights_init=weights_from_labeled().values.ravel(),
                    means_init=labeled_centroids.values,
                    verbose=10))
])

print("Computing clusters...", end='')
pipe_19_glove_labelinit.fit(np.array(all_data))
print("done")
print("%0.2fs" % (time.time() - t0))
gmm_19_glove_labelinit = pipe_19_glove_labelinit['gmm']
X_labeled_19_glove_labelinit = pipe_19_glove_labelinit['avgw2v_tfidf'].transform(np.array(labeled.data))
probs_19_glove_labelinit = gmm_19_glove_labelinit.predict_proba(X_labeled_19_glove_labelinit)<jupyter_output><empty_output><jupyter_text>Stats GMM random seed<jupyter_code>df = pd.DataFrame(probs_19_glove_rnd)
df['y'] = labeled.labels
candidates_probs_rnd = df.groupby('y').mean()
candidates_probs_rnd
plt.imshow(candidates_probs_rnd)
plt.colorbar()
candidates_probs_rnd.std(axis=1)
print(candidates_probs_rnd.std(axis=1).values.mean())
plt.imshow(np.expand_dims(candidates_probs_rnd.std(axis=1).values, axis=0))
plt.colorbar()
df = pd.DataFrame(pipe_19_glove_rnd.predict(np.array(all_data)))
df[1] = 1
df.groupby(0).sum().plot.bar()<jupyter_output><empty_output><jupyter_text>Stats GMM labeled centroids seed<jupyter_code>df = pd.DataFrame(probs_19_glove_labelinit)
df['y'] = labeled.labels
candidates_probs_labelinit = df.groupby('y').mean()
candidates_probs_labelinit
plt.imshow(candidates_probs_labelinit)
plt.colorbar()
candidates_probs_labelinit.std(axis=1)
print(candidates_probs_rnd.std(axis=1).values.mean())
plt.imshow(np.expand_dims(candidates_probs_labelinit.std(axis=1).values, axis=0))
plt.colorbar()
df = pd.DataFrame(pipe_19_glove_rnd.predict(np.array(all_data)))
df[1] = 1
df.groupby(0).sum().plot.bar()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1415899010576595, 'fraction_numerical': 0.02524735585124531, 'mean_word_length': 7.269393511988716, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '453223', 'license': 'Apache-2.0', 'n_tokens_mistral': 2452, 'n_tokens_neox': 2280, 'n_words': 263, 'path': 'hw1/notebooks/gmm.ipynb', 'repo_name': 'zydeon/uci-statnlp', 'size': '5862'}