<jupyter_start><jupyter_text>TFIDF<jupyter_code>tfidf_vectorizer=TfidfVectorizer(stop_words='english')
df_tfidf=tfidf_vectorizer.fit_transform(df['text']) 
df_tfidf
build_model(k_means, df_tfidf.toarray(), df_labels)
build_model(agglomerative_fn, df_tfidf.toarray(), df_labels)
build_model(dbscan_fn, df_tfidf.toarray(), df_labels)
build_model(mean_shift_fn, df_tfidf.toarray(), df_labels)
build_model(birch_fn, df_tfidf.toarray(), df_labels)
build_model(affinity_propagation_fn, df_tfidf.toarray(), df_labels)
build_model(mini_batch_kmeans_fn, df_tfidf.toarray(), df_labels)<jupyter_output>homo	compl	v-meas	ARI	AMI	silhouette
--------------------------------------------------
0.660	0.743	0.699	0.536	0.698	0.022<jupyter_text>Doc2Vec<jupyter_code>from getEmbeddings import getEmbeddings
from d2v_pre import d2v_pre

d2v_pre('data/sports.csv')

x,y = getEmbeddings('data/sports_doc2vec.csv')
def doc2vec_build_model(clustering_model, data, labels):
    
    model = clustering_model(data)

    print('homo\tcompl\tv-meas\tARI\tAMI')
    print(50 * '-')
    
    print('%.3f\t%.3f\t%.3f\t%.3f\t%.3f'
          %(metrics.homogeneity_score(labels, model.labels_),
            metrics.completeness_score(labels, model.labels_),
            metrics.v_measure_score(labels, model.labels_),
            metrics.adjusted_rand_score(labels, model.labels_),
            metrics.adjusted_mutual_info_score(labels,  model.labels_)))
doc2vec_build_model(k_means, x, y)
doc2vec_build_model(agglomerative_fn, x,y)
doc2vec_build_model(dbscan_fn, x,y)
doc2vec_build_model(mean_shift_fn, x, y)
doc2vec_build_model(birch_fn,x, y)
doc2vec_build_model(affinity_propagation_fn, x, y)
doc2vec_build_model(mini_batch_kmeans_fn, x, y)<jupyter_output>homo	compl	v-meas	ARI	AMI
--------------------------------------------------
0.039	0.041	0.040	0.024	0.038
{'fraction_non_alphanumeric': 0.16648471615720525, 'fraction_numerical': 0.034934497816593885, 'mean_word_length': 4.455357142857143, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '320190', 'license': 'MIT', 'n_tokens_mistral': 783, 'n_tokens_neox': 732, 'n_words': 91, 'path': 'sport_clustering.ipynb', 'repo_name': 'singh-l/Clustering_Repo', 'size': '1832'}