<jupyter_start><jupyter_text>Unsupervised learning with k-Means<jupyter_code>from pymongo import MongoClient
from gensim.models import Word2Vec
import gensim
import re
import string
from sklearn.feature_extraction.text import CountVectorizer
from nltk.tokenize import TreebankWordTokenizer
from nltk.stem import PorterStemmer
from nltk.corpus import stopwords
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA
import networkx as nx
from itertools import combinations
import community
from collections import defaultdict
import randomcolor
from sklearn.cluster import MiniBatchKMeans
from sklearn.metrics import silhouette_score
from sklearn.preprocessing import StandardScaler
import pickle
from scipy.special import comb
from scipy.spatial.distance import pdist<jupyter_output><empty_output><jupyter_text>Load reduced-dim vectors and full-dim vectors   300-dimension vectors<jupyter_code>inclusion_vectors = pickle.load(open("vectorized_criteria.p", 'rb'))
inclusion_df.shape
drop_index = np.argwhere(np.isnan(inclusion_vectors))
drop_row_index = drop_index[:,0]
inclusion_df = pd.DataFrame(inclusion_vectors)
inclusion_df = inclusion_df.dropna()
inclusion_df.shape<jupyter_output><empty_output><jupyter_text>86-dimension vectors<jupyter_code>reduced_86D_vectors = pickle.load(open("86D_threshhold_pca_features.p", 'rb'))
drop_index = np.argwhere(np.isnan(reduced_86D_vectors))
drop_row_index = drop_index[:,0]
reduced_86D_vectors = pd.DataFrame(reduced_86D_vectors)
reduced_86D_vectors = reduced_86D_vectors.dropna()
reduced_86D_vectors.shape<jupyter_output><empty_output><jupyter_text>2-dimension vectors<jupyter_code>reduced_2D_vectors = pickle.load(open("2D_pcafeatures.p", 'rb'))<jupyter_output><empty_output><jupyter_text>Clustering on the word2vec'd 300 dimensions without dimensionality reduction<jupyter_code>SSEs = []
silhouettes = []
for k in range(2,50):
    km = MiniBatchKMeans(n_clusters=k, random_state=2)
    km.fit(inclusion_df)
    labels = km.labels_
    SSEs.append(km.inertia_) 
    silhouettes.append(silhouette_score(inclusion_df, labels))
    
plt.figure(dpi=150)    
plt.xlabel("Number of clusters")  
plt.ylabel("SSE")
plt.plot(range(2,50), SSEs);
plt.figure(dpi=150)    
plt.xlabel("Number of clusters")  
plt.ylabel("Average Silhouette Score")
plt.plot(range(2,50), silhouettes);<jupyter_output><empty_output><jupyter_text>Clustering on the reduced 2-D PC space: k-Means sandbox example just to visualize it No need to scale the data because it's on the same scale (dimension-reduced vectorized criteria)<jupyter_code>km = MiniBatchKMeans(n_clusters=3)
clusts = km.fit_predict(reduced_2D_vectors)
plt.scatter(reduced_2D_vectors[:,0], reduced_2D_vectors[:,1], c=clusts, cmap=plt.cm.rainbow);<jupyter_output><empty_output><jupyter_text>86-dim k-Means<jupyter_code>SSEs = []
silhouettes = []
for k in range(2,100):
    km = MiniBatchKMeans(n_clusters=k, random_state=2)
    km.fit(reduced_86D_vectors)
    labels = km.labels_
    SSEs.append(km.inertia_) 
    silhouettes.append(silhouette_score(reduced_86D_vectors, labels))
    
plt.figure(dpi=150)    
plt.xlabel("Number of clusters")  
plt.ylabel("SSE")
plt.plot(range(2,100), SSEs);<jupyter_output><empty_output><jupyter_text>Match cluster labels to individual criteria in 86-dim space Link 'winning' cluster labels to individual criteria and studies. Need to grab study ids from the database because I didn't store it locally =/. Build this into the data cleaning earlier so I collect a list of studies to match back to the inclusion vectors later on.<jupyter_code>def connect_to_mongo(database, collection):
    
    """
    Opens a connection to a specified Mongo DB location
    
    Input Parameters:
    database: name of database to connect to or create (str)
    collection: name of collection to connect to or create (str)
    
    Returns:
    The connection object for the database without a collection specified
    The connection object for a specific Mongo location (database & collection)
    """
    
    client = MongoClient()
    db = client[database]
    mongo_loc = db[collection]
    return db, mongo_loc
trials_loc, eligibility_loc = connect_to_mongo('clinical_trials', 'eligibilities')
doc_cursor = eligibility_loc.find({"inclusion_criteria": { '$regex' : ".*asthma.*"}})

study_id = []
inclusion_text = []

for study in doc_cursor:
    for crit in study['cleaned_inclusion']:
        study_id.append(study['study_id'])
        inclusion_text.append(crit)
print(len(study_id))
pickle.dump(inclusion_text, open("inclusion_text.p", "wb"))
pickle.dump(study_id, open("study_id.p", "wb"))
len(labels)<jupyter_output><empty_output><jupyter_text>Remove study ids from criteria that were dropped due to having NaN<jupyter_code>drop_row_list = pickle.load(open("drop_row_list.p", 'rb'))
for i in sorted(drop_row_list, reverse=True):
    del study_id[i]
    del inclusion_text[i]
len(np.unique(study_id))
len(study_id)
len(inclusion_text)
clusts = labels.reshape(11235, 1)
clusts
studies = np.array(study_id)
studies = studies.reshape(11235, 1)
crit_studies_clusters = np.hstack((clusts, studies))
print(crit_studies_clusters)
crit_df = pd.DataFrame(crit_studies_clusters)
crit_df.columns = ['cluster', 'study_id']
crit_df.head()<jupyter_output><empty_output><jupyter_text>Format for network plot with criteria as nodes: 86-dim<jupyter_code>crit_group = crit_df.groupby(by='study_id')['cluster'].apply(tuple)
crit_network_input = np.array(crit_group)
crit_network_input
combinations(crit_network_input[0], 2)
network_input = []
for crit in crit_network_input:
    results = [x for x in combinations(crit, 2)]
    network_input += (results)
len(network_input)
network_input[:10]<jupyter_output><empty_output><jupyter_text>Make network plot: nodes are eligibility criteria<jupyter_code>trials = nx.Graph()
for crit_1, crit_2 in network_input:
    trials.add_edge(crit_1, crit_2)
positions = nx.spring_layout(trials)
network = nx.draw(trials, pos=positions)
labels =  nx.draw_networkx_labels(trials, pos=positions)
nx.density(trials)
nx.degree(trials)
degree_hist = nx.degree_histogram(trials)
xticks = range(len(degree_hist))
plt.bar(xticks, degree_hist, align='center')
plt.xlabel('Degree')
plt.ylabel('Number of nodes with this degree')
plt.title('An initial look into the degree distribution');<jupyter_output><empty_output><jupyter_text>Format for network plot with studies as nodes<jupyter_code>crit_group_study_node = crit_df.groupby(by='cluster')['study_id'].apply(tuple)
crit_network_input_study_node = np.array(crit_group_study_node)
network_input_study_nodes = []
for crit in crit_network_input_study_node:
    results = [x for x in combinations(crit, 2)]
    network_input_study_nodes += (results)
network_input_study_nodes[:5]<jupyter_output><empty_output><jupyter_text>Identifying communities in the criteria network<jupyter_code>modules = community.best_partition(trials)
module_ids = set(modules.values())
num_modules = len(module_ids)
num_modules<jupyter_output><empty_output><jupyter_text>Identify which criteria are in each community<jupyter_code>module_to_nodes = defaultdict(list)
for node, module_id in modules.items():
    module_to_nodes[module_id].append(node)

# module_to_nodes[4]<jupyter_output><empty_output><jupyter_text>Plot the criteria network graph with communities<jupyter_code>positions = nx.spring_layout(trials)

rand_color = randomcolor.RandomColor()
colors = rand_color.generate(count=num_modules)
for i, module in enumerate(module_ids):
    nodes_in_this_module = module_to_nodes[module]
    module_color = colors[i % len(colors)]
    nx.draw_networkx_nodes(trials,
                           positions,
                           nodes_in_this_module,
                           node_color = module_color,
                           node_size = 150)
nx.draw_networkx_edges(trials, positions, alpha=0.5)
# labels =  nx.draw_networkx_labels(study_nodes, pos=positions)<jupyter_output><empty_output><jupyter_text>Network graph: trials as nodes<jupyter_code># study_nodes = nx.Graph()
# for trial_1, trial_2 in network_input_study_nodes:
#     study_nodes.add_edge(trial_1, trial_2)
default_weight = 1
study_nodes = nx.Graph()
for nodes in network_input_study_nodes:
    n0 = nodes[0]
    n1 = nodes[1]
    if study_nodes.has_edge(n0,n1):   
        study_nodes[n0][n1]['weight'] += default_weight
    else:
        study_nodes.add_edge(n0,n1, weight=default_weight)
positions = nx.spring_layout(study_nodes)
network = nx.draw(study_nodes, pos=positions, node_size = 15)
# labels =  nx.draw_networkx_labels(study_nodes, pos=positions)
nx.write_gexf(study_nodes, "86D_kMeans.gexf")
neighbor_weights = study_nodes['NCT00862394']
# print(neighbor_weights)
nx.density(study_nodes)
degree_hist = nx.degree_histogram(study_nodes)
xticks = range(len(degree_hist)) # adjust range and widths
plt.bar(xticks, degree_hist, align='center')
plt.xlabel('Degree')
plt.ylabel('Number of study nodes with this degree')
plt.title('An initial look into the degree distribution');<jupyter_output><empty_output><jupyter_text>Identifying communities in the network<jupyter_code>modules = community.best_partition(study_nodes)
module_ids = set(modules.values())
num_modules = len(module_ids)
num_modules<jupyter_output><empty_output><jupyter_text>Identify which criteria are in each community<jupyter_code>module_to_nodes = defaultdict(list)
for node, module_id in modules.items():
    module_to_nodes[module_id].append(node)

# module_to_nodes[0]<jupyter_output><empty_output><jupyter_text>Plot the criteria network graph with communities<jupyter_code>positions = nx.spring_layout(study_nodes)

rand_color = randomcolor.RandomColor()
colors = rand_color.generate(hue='random', count=num_modules)
for i, module in enumerate(module_ids):
    nodes_in_this_module = module_to_nodes[module]
    module_color = colors[i % len(colors)]
    nx.draw_networkx_nodes(trials,
                           positions,
                           nodes_in_this_module,
                           node_color = module_color,
                           node_size = 20)
nx.draw_networkx_edges(study_nodes, positions, alpha=0.1)
# labels =  nx.draw_networkx_labels(study_nodes, pos=positions)
nx.write_gexf(study_nodes, "86D_kMeans_trial_nodes.gexf")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10071663761379043, 'fraction_numerical': 0.01346116598876622, 'mean_word_length': 6.112258953168044, 'pattern_counts': {'":': 1, '<': 77, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '275828', 'license': 'MIT', 'n_tokens_mistral': 3666, 'n_tokens_neox': 3463, 'n_words': 699, 'path': '4_unsupervised_learning_kMeans.ipynb', 'repo_name': 'cstepien/clinical_trials_word2vec_clustering', 'size': '10326'}