<jupyter_start><jupyter_text>ANLY-501 Mod 3 DiscussionR<NAME> (rq47)2021-09-26  The setup<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
import wikipedia
import re
import glob
import os
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
from wordcloud import WordCloud<jupyter_output><empty_output><jupyter_text>Retrieve wiki page textYou don't need to run this section if the text files are already present in the directory.But no harm to run it anyways.<jupyter_code>search_list = ["Sebastian_Vettel", "Red_Bull_Racing", "Ggplot2", "Hadley_Wickham", "Michael_Jordan", "Charlotte_Hornets"]
for word in search_list:
    try:
        page_content = wikipedia.page(word, auto_suggest=False).content # auto_suggest has to be turned off to avoid incorrect results
        page_content = re.sub(r'==.*?==+', '', page_content) # Drop headers surrounded by ‘==’
        page_content = page_content.replace('\n', '') # Replace ‘\n’ (a new line) with an empty string
        text_file = open("{}.txt".format(word), "w")
        text_file.write(page_content)
        text_file.close()
    except wikipedia.exceptions.DisambiguationError as e:
        print(e.options)
        break
    except wikipedia.exceptions.PageError:
        print("Page not found")
        break<jupyter_output><empty_output><jupyter_text>Load data from .txt<jupyter_code>os.chdir(r'.')
df = pd.DataFrame()

for filename in glob.glob('*.txt'):
    with open(filename, 'r') as file:
        txt = file.read()
        file.close()
        txt = re.sub(r'[,.;@#?!&$\-\']+', ' ', txt, flags=re.IGNORECASE)
        txt = re.sub(' +', ' ', txt, flags=re.IGNORECASE)
        txt = re.sub(r'\"', ' ', txt, flags=re.IGNORECASE)
        txt = re.sub(r'[^a-zA-Z]', " ", txt, flags=re.VERBOSE)
        txt = txt.replace(',', '')
        txt = ' '.join(txt.split())
        txt = re.sub("\n|\r", "", txt)
        df = df.append({'page_title': filename.split('.')[0], 'page_content': txt}, ignore_index=True)

df
df['category'] = ['R', 'R', 'F1', 'F1', 'NBA', 'NBA']
df<jupyter_output><empty_output><jupyter_text>Text clustering<jupyter_code>vectorizer = TfidfVectorizer(stop_words='english')
X = vectorizer.fit_transform(df['page_content'])

sum_of_squared_distances = []
K = range(1, 7)

for k in K:
    model = KMeans(n_clusters=k, init='k-means++', max_iter=200, n_init=1)
    model = model.fit(X)
    sum_of_squared_distances.append(model.inertia_)

plt.plot(K, sum_of_squared_distances, 'bx-')
plt.xlabel('k')
plt.ylabel('Sum_of_squared_distances')
plt.title('Elbow Method For Optimal k')
plt.show()<jupyter_output><empty_output><jupyter_text>It's really hard to say there is a typical elbow here in the plot. Let's just use $k=3$ for now. There are too few articles here in the data.<jupyter_code>model = KMeans(n_clusters=3, init='k-means++', max_iter=200, n_init=1)
model.fit(X)
labels = model.labels_
df['label'] = labels
df
print("Top terms per cluster:")
order_centroids = model.cluster_centers_.argsort()[:, ::-1]
terms = vectorizer.get_feature_names()
for i in range(3):
    print("Cluster %d:" % i),
    for ind in order_centroids[i, :10]:
        print(' %s' % terms[ind]),
    print<jupyter_output>Top terms per cluster:
Cluster 0:
 vettel
 red
 prix
 bull
 grand
 race
 season
 team
 webber
 pole
Cluster 1:
 ggplot
 wickham
 data
 hadley
 graphics
 statistical
 plotting
 plot
 packages
 isbn
Cluster 2:
 jordan
 nba
 hornets
 season
 game
 team
 bobcats
 charlotte
 basketball
 games<jupyter_text>Wordclouds for each cluster<jupyter_code>from wordcloud import WordCloud
result=pd.DataFrame({'cluster':labels,'wiki':df['page_content']})
for k in range(0,3):
   s=result[result.cluster==k]
   text=s['wiki'].str.cat(sep=' ')
   text=text.lower()
   text=' '.join([word for word in text.split()])
   wordcloud = WordCloud(max_font_size=50, max_words=100, background_color="white").generate(text)
   print('Cluster: {}'.format(k))
   plt.figure()
   plt.imshow(wordcloud, interpolation="bilinear")
   plt.axis("off")
   plt.show()<jupyter_output>Cluster: 0
{'fraction_non_alphanumeric': 0.14663402692778457, 'fraction_numerical': 0.011015911872705019, 'mean_word_length': 4.426294820717131, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '604481', 'license': 'MIT', 'n_tokens_mistral': 1513, 'n_tokens_neox': 1447, 'n_words': 338, 'path': 'ANLY-501-INTRO/rq47_mod3_discussion/q1/q1.ipynb', 'repo_name': 'rexarski/ggtown-ds', 'size': '4085'}