<jupyter_start><jupyter_text>Кластеризация<jupyter_code>print(datetime.datetime.now())<jupyter_output><empty_output><jupyter_text>Данные<jupyter_code>df_embeddings = pd.read_csv('data/embeddings/education_field_of_study.csv', index_col=0)
vec_embedding = df_embeddings.loc[:, ~df_embeddings.columns.isin(
    ['education_field_of_study']
)]
def l2_dist(x1, x2):
    return np.linalg.norm(x2 - x1)


def cos_sim(x1, x2):
    return np.dot(x1, x2) / (np.linalg.norm(x1) * np.linalg.norm(x2))


def l2_dist_ind(ind1, ind2):
    return l2_dist(np.array(vec_embedding.loc[ind1]), np.array(vec_embedding.loc[ind2]))


def cos_sim_ind(ind1, ind2):
    return cos_sim(np.array(vec_embedding.loc[ind1]), np.array(vec_embedding.loc[ind2]))


def get_distance(profile1, profile2):
    ind1 = df_embeddings[df_embeddings['education_field_of_study'] == profile1].head(1).index[0]
    ind2 = df_embeddings[df_embeddings['education_field_of_study'] == profile2].head(1).index[0]

    print('l2 dist: ', l2_dist_ind(ind1, ind2))
    print('cos sim: ', cos_sim_ind(ind1, ind2))<jupyter_output><empty_output><jupyter_text>Свойство Word2Vec embedding-ов<jupyter_code>get_distance('Computer Science', 'Machine Learning')
get_distance('Medicine', 'Biology')
get_distance('Medicine', 'Computer Science')
get_distance('Biology', 'Computer Science')
get_distance('Biology', 'Machine Learning')
get_distance('Biochemistry', 'Computer Science')
get_distance('Biochemistry', 'Biology')
get_distance('CS', 'Computer Science')<jupyter_output>l2 dist:  12.729455672572442
cos sim:  0.3308707818486452<jupyter_text>TSNE<jupyter_code>from sklearn.manifold import TSNE

tsne = TSNE(n_components=2, n_jobs=-1)
def df_to_2_tsne(df_embeddings):
    # Сделать чтобы сохранялось в кэш
    
    X_2_tsne = pd.DataFrame(
        tsne.fit_transform(vec_embedding), 
        columns=['x', 'y']
    )
    
    X_2_tsne_labeled = pd.concat(
        [
            X_2_tsne, 
            df_embeddings[['education_field_of_study']]
        ], 
        axis=1
    )
    
    return X_2_tsne_labeled

X_2_tsne_labeled = df_to_2_tsne(df_embeddings)
fig = px.scatter(
    X_2_tsne_labeled, 
    x="x", y="y", 
    text='education_field_of_study'
)

fig.update_traces(mode="markers")

fig.show()
fig.write_html("../figures/prep-embeddings/TSNE_{0}.html".format(datetime.datetime.now()))<jupyter_output><empty_output><jupyter_text>Clusterization  k Measn<jupyter_code>from sklearn.cluster import KMeans
from sklearn import metrics
kmeans = KMeans(n_clusters=6, random_state=0).fit(vec_embedding)
labels = kmeans.labels_
centroids = kmeans.cluster_centers_
centroids.shape<jupyter_output><empty_output><jupyter_text>Clusterization metric The Silhouette Coefficient is calculated using the mean intra-cluster distance (a) and the mean nearest-cluster distance (b) for each sample. The Silhouette Coefficient for a sample is (b - a) / max(a, b). To clarify, b is the distance between a sample and the nearest cluster that the sample is not a part of.The best value is 1 and the worst value is -1. Values near 0 indicate overlapping clusters.<jupyter_code>metrics.silhouette_score(vec_embedding, labels, metric='euclidean')<jupyter_output><empty_output><jupyter_text>Группы<jupyter_code>X_kmeans_labeled = pd.concat(
    [
        df_embeddings[['education_field_of_study']], 
        pd.DataFrame(labels, columns=['label'])
    ], 
    axis=1
)
X_kmeans_labeled
X_kmeans_labeled['label'].value_counts()<jupyter_output><empty_output><jupyter_text>Vis + TSNE<jupyter_code>X_2_tsne_labeled = df_to_2_tsne(df_embeddings)

X_2_tsne_labeled = pd.concat(
    [
        X_2_tsne_labeled, 
        pd.DataFrame(labels, columns=['label'])
    ], 
    axis=1
)
X_2_tsne_labeled['label'] = 'label ' + X_2_tsne_labeled['label'].astype(str)
fig = px.scatter(
    X_2_tsne_labeled, 
    x="x", y="y", 
    text='education_field_of_study',
    color='label'
)

fig.update_traces(mode="markers")

fig.show()
fig.write_html("../figures/prep-embeddings/kMeans_{0}.html".format(datetime.datetime.now()))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1303597831444061, 'fraction_numerical': 0.02488910793494332, 'mean_word_length': 5.6, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '407411', 'license': 'MIT', 'n_tokens_mistral': 1626, 'n_tokens_neox': 1539, 'n_words': 231, 'path': 'code/prep-clustering.ipynb', 'repo_name': 'Intelligent-Systems-Phystech/2022-Project-vc', 'size': '4058'}