<jupyter_start><jupyter_text>Running kmeans in sci-kit learn<jupyter_code>n_samples = 1500
random_state = 170
X, y = make_blobs(n_samples=n_samples, random_state=random_state)

plt.scatter(X[:, 0], X[:, 1])
plt.title("Create random blobs")
# Incorrect number of clusters
y_pred = KMeans(n_clusters=2, random_state=random_state).fit_predict(X)

plt.scatter(X[:, 0], X[:, 1], c=y_pred)
plt.title("Incorrect Number of Blobs")
# Incorrect number of clusters
y_pred = KMeans(n_clusters=3, random_state=random_state).fit_predict(X)

plt.scatter(X[:, 0], X[:, 1], c=y_pred)
plt.title("correct Number of Blobs")<jupyter_output><empty_output><jupyter_text>Task1: Understanding initialization impact on convergence<jupyter_code>random_state = np.random.RandomState(0)

# Datasets generation parameters
n_samples_per_center = 100
grid_size = 3
scale = 0.1
n_clusters = grid_size ** 2

def make_data(random_state, n_samples_per_center, grid_size, scale):
    random_state = check_random_state(random_state)
    centers = np.array([[i, j]
                        for i in range(grid_size)
                        for j in range(grid_size)])
    n_clusters_true, n_features = centers.shape

    noise = random_state.normal(
        scale=scale, size=(n_samples_per_center, centers.shape[1]))

    X = np.concatenate([c + noise for c in centers])
    y = np.concatenate([[i] * n_samples_per_center
                        for i in range(n_clusters_true)])
    return shuffle(X, y, random_state=random_state)<jupyter_output><empty_output><jupyter_text>Kmeans initialisation effect<jupyter_code>X, y = make_data(random_state, n_samples_per_center, grid_size, scale)
km = KMeans(n_clusters=n_clusters, init='random', n_init=1,
                     random_state=random_state).fit(X)

plt.figure()
for k in range(n_clusters):
    my_members = km.labels_ == k
    color = cm.nipy_spectral(float(k) / n_clusters, 1)
    plt.plot(X[my_members, 0], X[my_members, 1], 'o', marker='.', c=color)
    cluster_center = km.cluster_centers_[k]
    plt.plot(cluster_center[0], cluster_center[1], 'o',
             markerfacecolor=color, markeredgecolor='k', markersize=6)
    plt.title("Example cluster allocation with a random init\n"
              "with KMeans")

plt.show()
X, y = make_data(random_state, n_samples_per_center, grid_size, scale)
km = KMeans(n_clusters=n_clusters, init='k-means++', n_init=1,
                     random_state=random_state).fit(X)

plt.figure()
for k in range(n_clusters):
    my_members = km.labels_ == k
    color = cm.nipy_spectral(float(k) / n_clusters, 1)
    plt.plot(X[my_members, 0], X[my_members, 1], 'o', marker='.', c=color)
    cluster_center = km.cluster_centers_[k]
    plt.plot(cluster_center[0], cluster_center[1], 'o',
             markerfacecolor=color, markeredgecolor='k', markersize=6)
    plt.title("Example cluster allocation with a random init\n"
              "with KMeans")

plt.show()<jupyter_output><empty_output><jupyter_text>Inertia measure<jupyter_code># Number of time the k-means algorithm will be run with different
# centroid seeds. The final results will be the best output of n_init consecutive runs in terms of inertia.
n_init_range = np.array([1, 5, 10, 15, 20])

# Number of run (with randomly generated dataset) for each strategy so as
# to be able to compute an estimate of the standard deviation
n_runs = 10

plt.figure()
plots = []
legends = []

cases = [
    (KMeans, 'k-means++', {}),
    (KMeans, 'random', {}),
]

for factory, init, params in cases:
    print("Evaluation of %s with %s init" % (factory.__name__, init))
    inertia = np.empty((len(n_init_range), n_runs))

    for run_id in range(n_runs):
        X, y = make_data(run_id, n_samples_per_center, grid_size, scale)
        for i, n_init in enumerate(n_init_range):
            km = factory(n_clusters=n_clusters, init=init, random_state=run_id,
                         n_init=n_init, **params).fit(X)
            inertia[i, run_id] = km.inertia_
    p = plt.errorbar(n_init_range, inertia.mean(axis=1), inertia.std(axis=1))
    plots.append(p[0])
    legends.append("%s with %s init" % (factory.__name__, init))

plt.xlabel('n_init')
plt.ylabel('inertia')
plt.legend(plots, legends)
plt.title("Mean inertia for various k-means init across %d runs" % n_runs)<jupyter_output>Evaluation of KMeans with k-means++ init
Evaluation of KMeans with random init<jupyter_text>Task 2: Use k-means to cluster wikipedia articles  Compute TF-IDF<jupyter_code># import text documents from wikipedia abstracts
wiki_data=pd.read_csv('../people_wiki.txt',delimiter='\t', index_col='name')['text']

wiki_data.head()
#Define the TFIDF vectorizer that will be used to process the data
tfidf_vectorizer = TfidfVectorizer()
#Apply this vectorizer to the full dataset to create normalized vectors
tf_idf = tfidf_vectorizer.fit_transform(wiki_data)<jupyter_output><empty_output><jupyter_text>Choose the right number of k<jupyter_code>def plot_k_vs_heterogeneity(k_values, heterogeneity_values):
    plt.figure(figsize=(7,4))
    plt.plot(k_values, heterogeneity_values, linewidth=4)
    plt.xlabel('K')
    plt.ylabel('Inertia')
    plt.title('K vs. Inertia')
    plt.rcParams.update({'font.size': 16})
    plt.tight_layout()
heterogeneity_values = []
k_list = [2, 10, 25, 50, 100, 300]
##### TAKES FOREVER TO RUN#####
for n_clusters in k_list:
    km = KMeans(n_clusters=n_clusters, init='k-means++').fit(tf_idf)
    heterogeneity_values.append(km.inertia_)

plot_k_vs_heterogeneity(k_list, heterogeneity_values)
# compute kmeans using random initialization
kmeans = KMeans(n_clusters=100, init='random').fit(tf_idf)  
# predict the closest cluster for each point
labels= kmeans.predict(tf_idf)
# retrieve other pages from a cluster
def get_cluster(cluster_nb, labels, dataset):
    curr_cluster=[]
    abstract=[]
    for index in range(len(labels)):
        if labels[index]==cluster_nb:
            curr_cluster.append(dataset.index.values[index,])
            abstract.append(dataset.iloc[index,])
            
    result = pd.DataFrame({'names': curr_cluster,
                           'abstract': abstract})
    return result
        
# Find the cluster in which Dr. Dre is
pos = wiki_data.index.get_loc("Dr. Dre")
labels[pos]
cluster= get_cluster(labels[pos], labels, wiki_data)
cluster
# try to get a smarter initialization?<jupyter_output><empty_output><jupyter_text>LDA modeling<jupyter_code># import text documents from wikipedia abstracts
wiki_data=pd.read_csv('../people_wiki.txt',delimiter='\t', index_col='name')['text']

wiki_data.head()
# LDA can only use raw term counts for LDA because it is a probabilistic graphical model
tf_vectorizer = CountVectorizer(max_df=0.95, min_df=2, stop_words='english')
tf = tf_vectorizer.fit_transform(wiki_data)
tf_feature_names = tf_vectorizer.get_feature_names()
no_topics = 20
# Run LDA
lda = LatentDirichletAllocation(n_topics=no_topics).fit(tf)<jupyter_output>/Users/aponsero/anaconda3/lib/python3.7/site-packages/sklearn/decomposition/online_lda.py:314: DeprecationWarning: n_topics has been renamed to n_components in version 0.19 and will be removed in 0.21
  DeprecationWarning)<jupyter_text>Display and evaluate topics<jupyter_code>def display_topics(model, feature_names, no_top_words):
    for topic_idx, topic in enumerate(model.components_):
        print("Topic %d:" % (topic_idx))
        print(" ".join([feature_names[i]
                        for i in topic.argsort()[:-no_top_words - 1:-1]]))

no_top_words = 10
display_topics(lda, tf_feature_names, no_top_words)
# transform method returns a matrix with one line per document, columns being topics weight
predict = lda.transform(tf)
result = pd.DataFrame(predict).set_index(wiki_data.reset_index()['name'])
# get distribution of topics for Juliette Binoche
d = pd.Series(result.loc["Juliette Binoche"]).sort_values(ascending=False)

ax = d.plot(kind='bar', title='Juliette Binoche Wikipedia Topic distribution',
            figsize=(10,6), width=.8, fontsize=14, rot=45 )

ax.title.set_size(20)
# Adjust (less or more topics) and rerun the LDA model

#display best word by topic

# transform method returns a matrix with one line per document, columns being topics weight

# get distribution of topics for Zendaya<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11903608621942359, 'fraction_numerical': 0.012836037781545169, 'mean_word_length': 4.595528455284553, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '64754', 'license': 'MIT', 'n_tokens_mistral': 2968, 'n_tokens_neox': 2804, 'n_words': 667, 'path': 'sprint4_clustering/sprint4_demo.ipynb', 'repo_name': 'maliaca/ML_sprint', 'size': '8258'}