<jupyter_start><jupyter_text>TFIDF<jupyter_code># loading libraries
import pandas as pd
from nltk.stem.snowball import SnowballStemmer
stemmer = SnowballStemmer("english")
import nltk
import re
from sklearn.cluster import SpectralClustering
from sklearn.metrics import silhouette_score
from sklearn.cluster import KMeans
from sklearn.model_selection import GridSearchCV
from collections import Counter
import ast
# importing data
ted_main = pd.read_csv('ted_main.csv')
ted_main['tags'] = ted_main['tags'].apply(lambda x: ast.literal_eval(x))
transcripts = pd.read_csv('transcripts.csv')
ted_merged = pd.merge(left=transcripts,
                      right=ted_main,
                      left_on='url',
                      right_on='url')
transcript = ted_merged.transcript
def tokenize(text):
    tokens = [word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]
    filtered_tokens = []
    for token in tokens:
        if re.search('[a-zA-Z]', token):
            filtered_tokens.append(token)
#    stems = [stemmer.stem(t) for t in filtered_tokens]
    return filtered_tokens
doc = transcript.tolist()
from sklearn.feature_extraction.text import TfidfVectorizer

tfidf_vectorizer = TfidfVectorizer(max_df=0.8, max_features=200000,
                                 min_df=0.2, stop_words='english',
                                 use_idf=True, tokenizer=tokenize, ngram_range=(1,3))

%time tfidf_matrix = tfidf_vectorizer.fit_transform(doc) #fit the vectorizer to synopses

print(tfidf_matrix.shape)<jupyter_output>Wall time: 1min 29s
(2467, 364)<jupyter_text>Spectural Clustering<jupyter_code>n_cluster = range(2,11)
best_param = []
list_score = []
for n in n_cluster:
    model = SpectralClustering(n_clusters=n)
    model.fit(tfidf_matrix)
    label = model.labels_
    list_score.append(silhouette_score(tfidf_matrix, label))
list_score = np.array(list_score)
best_param.append(n_cluster[list_score.argmax()])
print(best_param)
model = SpectralClustering(n_clusters=8)
model.fit(tfidf_matrix)
label = model.labels_
clusters = label.tolist()
Counter(clusters)<jupyter_output><empty_output><jupyter_text>KMeans Clustering<jupyter_code>n_cluster = list(range(2,11))
param_grid = {'n_clusters': n_cluster}
kmeans = KMeans()
kmeans_cv = GridSearchCV(kmeans, param_grid)
kmeans_cv.fit(tfidf_matrix)
print("Tuned Kmeans Parameter: {}".format(kmeans_cv.best_params_))
km_model = KMeans(n_clusters=8)
km_model.fit(tfidf_matrix)
km_label = km_model.labels_
km_clusters = km_label.tolist()
Counter(km_clusters)
import warnings
warnings.filterwarnings("ignore")
ted_merged['cluster'] = clusters
ted_w_cluster = ted_merged[['title','transcript','tags','cluster']]
ted_w_cluster[ted_w_cluster['cluster']==7][:50]
ted_w_cluster
c0_tag = [item for sub_list in ted_w_cluster[ted_w_cluster.cluster == 0]['tags'].tolist() for item in sub_list]
c1_tag = [item for sub_list in ted_w_cluster[ted_w_cluster.cluster == 1]['tags'].tolist() for item in sub_list]
c2_tag = [item for sub_list in ted_w_cluster[ted_w_cluster.cluster == 2]['tags'].tolist() for item in sub_list]
c3_tag = [item for sub_list in ted_w_cluster[ted_w_cluster.cluster == 3]['tags'].tolist() for item in sub_list]
c4_tag = [item for sub_list in ted_w_cluster[ted_w_cluster.cluster == 4]['tags'].tolist() for item in sub_list]
c5_tag = [item for sub_list in ted_w_cluster[ted_w_cluster.cluster == 5]['tags'].tolist() for item in sub_list]
c6_tag = [item for sub_list in ted_w_cluster[ted_w_cluster.cluster == 6]['tags'].tolist() for item in sub_list]
c7_tag = [item for sub_list in ted_w_cluster[ted_w_cluster.cluster == 7]['tags'].tolist() for item in sub_list]
# c8_tag = [item for sub_list in ted_w_cluster[ted_w_cluster.cluster == 8]['tags'].tolist() for item in sub_list]
# c9_tag = [item for sub_list in ted_w_cluster[ted_w_cluster.cluster == 9]['tags'].tolist() for item in sub_list]
c0_tag_stat = pd.Series(Counter(c0_tag))
c1_tag_stat = pd.Series(Counter(c1_tag))
c2_tag_stat = pd.Series(Counter(c2_tag))
c3_tag_stat = pd.Series(Counter(c3_tag))
c4_tag_stat = pd.Series(Counter(c4_tag))
c5_tag_stat = pd.Series(Counter(c5_tag))
c6_tag_stat = pd.Series(Counter(c6_tag))
c7_tag_stat = pd.Series(Counter(c7_tag))
# c8_tag_stat = pd.Series(Counter(c8_tag))
# c9_tag_stat = pd.Series(Counter(c9_tag))
print(c0_tag_stat.nlargest(10))
print ("")
print (c1_tag_stat.nlargest(10))
print ("")
print (c2_tag_stat.nlargest(10))
print ("")
print (c3_tag_stat.nlargest(10))
print ("")
print (c4_tag_stat.nlargest(10))
print ("")
print (c5_tag_stat.nlargest(10))
print ("")
print(c6_tag_stat.nlargest(10))
print ("")
print(c7_tag_stat.nlargest(10))
print ("")
# print(c8_tag_stat.nlargest(10))
# print ("")
# print(c9_tag_stat.nlargest(10))
# print ("")<jupyter_output>entertainment    87
culture          75
humor            64
technology       51
TEDx             49
science          39
performance      39
music            36
comedy           36
design           36
dtype: int64

global issues    198
business         115
economics         91
technology        75
politics          59
TEDx              54
culture           52
social change     52
health            49
society           49
dtype: int64

design           106
cities            65
architecture      55
technology        42
culture           32
art               32
collaboration     24
creativity        23
business          22
innovation        22
dtype: int64

technology       53
data             38
science          25
health           17
TEDx             16
communication    13
business         12
computers        12
global issues    12
medicine         12
dtype: int64

science          86
technology       57
environment      50
exploration      36
nature           30
TEDx             30
desi[...]
{'fraction_non_alphanumeric': 0.10751565762004175, 'fraction_numerical': 0.03583855254001392, 'mean_word_length': 3.9052901023890785, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '4915', 'license': 'MIT', 'n_tokens_mistral': 2165, 'n_tokens_neox': 2011, 'n_words': 431, 'path': 'Label Clustering.ipynb', 'repo_name': 'tienyuliu/IDS-703-Final-Project', 'size': '5748'}