<jupyter_start><jupyter_text><jupyter_code>import numpy as np
import pandas as pd
import nltk
from nltk.stem.snowball import SnowballStemmer
from bs4 import BeautifulSoup
import re
import os
import codecs
from sklearn import feature_extraction
import mpld3<jupyter_output><empty_output><jupyter_text>Read movie titles, 100 movies; the last one is empty string<jupyter_code>titles = open('data/title_list.txt').read().split('\n')
len(titles)
titles[:10]
titles = titles[:100]
# Just turning it around<jupyter_output><empty_output><jupyter_text>Read Genres information<jupyter_code>genres = open('data/genres_list.txt').read().split('\n')
genres = genres[:100]
genres[0]<jupyter_output><empty_output><jupyter_text>Read in the synopses from wiki<jupyter_code>synopses_wiki = open('data/synopses_list_wiki.txt').read().split('\n BREAKS HERE')
len(synopses_wiki)
synopses_wiki = synopses_wiki[:100]
#synopses_wiki[0]<jupyter_output><empty_output><jupyter_text>strips html formatting and converts to unicode<jupyter_code>synopses_clean_wiki = []
for text in synopses_wiki:
    text = BeautifulSoup(text, 'html.parser').getText()
    synopses_clean_wiki.append(text)
synopses_wiki = synopses_clean_wiki
#synopses_wiki[0]<jupyter_output><empty_output><jupyter_text>Read synopses information from imdb, which might be different from wiki. Also cleaned as above.<jupyter_code>synopses_imdb = open('data/synopses_list_imdb.txt').read().split('\n BREAKS HERE')
synopses_imdb = synopses_imdb[:100]

synopses_clean_imdb = []

for text in synopses_imdb:
    text = BeautifulSoup(text, 'html.parser').getText()
    synopses_clean_imdb.append(text)
synopses_imdb = synopses_clean_imdb
#synopses_imdb[0]<jupyter_output><empty_output><jupyter_text>Combine synopses from wiki and imdb<jupyter_code>synopses = []
for i in range(len(synopses_wiki)):
    item = synopses_wiki[i] + synopses_imdb[i]
    synopses.append(item)
#synopses[0]
print(str(len(titles)) + ' titles')
print(str(len(genres)) + ' genres')
print(str(len(synopses)) + ' synopses')
# generates index for each item in the corpora (in this case it's just rank) and I'll use this for scoring later
# the movies in the list are already ranked from 1 to 100
ranks = []
for i in range(1, len(titles)+1):
    ranks.append(i)<jupyter_output><empty_output><jupyter_text>Start Code Here<jupyter_code># load nltk's English stopwords as variable called 'stopwords'
# use nltk.download() to install the corpus first
# Stop Words are words which do not contain important significance to be used in Search Queries

##########   CODE BELOW    ##########

##########   CODE ABOVE    ##########

# load nltk's SnowballStemmer as variabled 'stemmer'

##########   CODE BELOW    ##########

##########   CODE ABOVE    ##########
len(stopwords)
#stopwords
# sents = [sent for sent in nltk.sent_tokenize("Today (May 19, 2016) is his only daughter's wedding. <NAME> is the Godfather. Vito's youngest son, Michael, in a Marine Corps uniform, introduces his girlfriend, <NAME>, to his family at the sprawling reception.")]
# Tokenize documents into sentences

#Create a variable 'sents' which is a list of strings obtained by tokenizing the string below:
'''
Today (May 19, 2016) is his only daughter's wedding. <NAME> is the Godfather. Vito's youngest son, Michael, in a Marine Corps uniform, introduces his girlfriend, <NAME>, to his family at the sprawling reception.
'''

##########   CODE BELOW    ##########

##########   CODE ABOVE    ##########
sents
# Tokenize sentences into words

#Create a variable 'words' which is a list of strings obtained by tokenizing the first string in sents 

words
# filter out any tokens not containing letters (e.g., numeric tokens, raw punctuation)
filtered_words = []
for word in words:
        if re.search('[a-zA-Z]', word):
            filtered_words.append(word)
filtered_words
# Stem words 

#Create a variable 'stems' which is a list of stemmed words obtained by stemming filtered_words

# see how "only" is stemmed to "onli" and "wedding" is stemmed to "wed"
stems<jupyter_output><empty_output><jupyter_text>Pause Code Here<jupyter_code># here I define a tokenizer and stemmer which returns the set of stems in the text that it is passed
# Punkt Sentence Tokenizer, sent means sentence 
def tokenize_and_stem(text):
    # first tokenize by sentence, then by word to ensure that punctuation is caught as it's own token
    tokens = [word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]
    filtered_tokens = []
    # filter out any tokens not containing letters (e.g., numeric tokens, raw punctuation)
    for token in tokens:
        if re.search('[a-zA-Z]', token):
            filtered_tokens.append(token)
    stems = [stemmer.stem(t) for t in filtered_tokens]
    return stems
def tokenize_only(text):
    # first tokenize by sentence, then by word to ensure that punctuation is caught as it's own token
    tokens = [word.lower() for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]
    filtered_tokens = []
    # filter out any tokens not containing letters (e.g., numeric tokens, raw punctuation)
    for token in tokens:
        if re.search('[a-zA-Z]', token):
            filtered_tokens.append(token)
    return filtered_tokens
words_stemmed = tokenize_and_stem("Today (May 19, 2016) is his only daughter's wedding.")
print(words_stemmed)
words_only = tokenize_only("Today (May 19, 2016) is his only daughter's wedding.")
words_only<jupyter_output><empty_output><jupyter_text>Below I use my stemming/tokenizing and tokenizing functions to iterate over the list of synopses to create two vocabularies: one stemmed and one only tokenized<jupyter_code>totalvocab_stemmed = []
totalvocab_tokenized = []
for i in synopses:
    allwords_stemmed = tokenize_and_stem(i) # for each item in 'synopses', tokenize/stem
    totalvocab_stemmed.extend(allwords_stemmed) # extend the 'totalvocab_stemmed' list
    
    allwords_tokenized = tokenize_only(i)
    totalvocab_tokenized.extend(allwords_tokenized)
print(len(totalvocab_stemmed))
print(len(totalvocab_tokenized))
vocab_frame = pd.DataFrame({'words': totalvocab_tokenized}, index = totalvocab_stemmed)
print('there are ' + str(vocab_frame.shape[0]) + ' items in vocab_frame')
print(vocab_frame.head())<jupyter_output><empty_output><jupyter_text>Resume Code Here Generate TF-IDF matrix (see http://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.TfidfVectorizer.html)<jupyter_code># Note that the result of this block takes a while to show

#Import TfidfVectorizer from sklearn.feature_extraction.text

##########   CODE BELOW    ##########

##########   CODE ABOVE    ##########



#define vectorizer parameters

# Create a TfidfVectorizer named tfidf_vectorizer with the below configuration

##########   CODE BELOW    ##########

__________ = __________(max_df=0.8, max_features=200000,
                                 min_df=0.2, stop_words='english',
                                 use_idf=True, tokenizer=_______, ngram_range=(_, _))

##########   CODE ABOVE    ##########


#fit transform synopses to tfidf_vectorizer and store as tfidf_matrix

##########   CODE BELOW    ##########

##########   CODE ABOVE    ##########

# (100, 563) means the matrix has 100 rows and 563 columns
print(tfidf_matrix.shape)
terms = tfidf_vectorizer.get_feature_names()
len(terms)<jupyter_output><empty_output><jupyter_text>Now onto the fun part. Using the tf-idf matrix, you can run a slew of clustering algorithms to better understand the hidden structure within the synopses. I first chose k-means. K-means initializes with a pre-determined number of clusters (I chose 5). Each observation is assigned to a cluster (cluster assignment) so as to minimize the within cluster sum of squares. Next, the mean of the clustered observations is calculated and used as the new cluster centroid. Then, observations are reassigned to clusters and centroids recalculated in an iterative process until the algorithm reaches convergence.I found it took several runs for the algorithm to converge a global optimum as k-means is susceptible to reaching local optima - how to decide that the algorithm converged???<jupyter_code>#import KMeans from sklearn.cluster
#create KMeans object km with n_clusters = 5

##########   CODE BELOW    ##########


##########   CODE ABOVE    ##########

#fit tfidf_matrix to km 

##########   CODE BELOW    ##########

##########   CODE ABOVE    ##########

clusters = km.labels_.tolist()<jupyter_output><empty_output><jupyter_text>Pause Code Here Here, I create a dictionary of titles, ranks, the synopsis, the cluster assignment, and the genre [rank and genre were scraped from IMDB].I convert this dictionary to a Pandas DataFrame for easy access. I'm a huge fan of Pandas and recommend taking a look at some of its awesome functionality which I'll use below, but not describe in a ton of detail.<jupyter_code>films = { 'title': titles, 'rank': ranks, 'synopsis': synopses, 'cluster': clusters, 'genre': genres }

frame = pd.DataFrame(films, index = [clusters] , columns = ['rank', 'title', 'cluster', 'genre'])

print(frame) # here the ranking is still 0 to 99

frame['cluster'].value_counts() #number of films per cluster (clusters from 0 to 4)<jupyter_output><empty_output><jupyter_text>Note that clusters 4 and 0 have the lowest rank, which indicates that they, on average, contain films that were ranked as "better" on the top 100 list.Here is some fancy indexing and sorting on each cluster to identify which are the top n (I chose n=6) words that are nearest to the cluster centroid. This gives a good sense of the main topic of the cluster.<jupyter_code>print("Top terms per cluster:")

num_clusters = 5

#sort cluster centers by proximity to centroid
order_centroids = km.cluster_centers_.argsort()[:, ::-1] 

for i in range(num_clusters):
    print("Cluster %d words:" % i, end='')
    
    for ind in order_centroids[i, :6]: #replace 6 with n words per cluster
        print(' %s' % vocab_frame.loc[terms[ind].split(' ')].values.tolist()[0][0].encode('utf-8', 'ignore'), end=',')
    print() #add whitespace
    print() #add whitespace
    
    print("Cluster %d titles:" % i, end='')
    for title in frame.loc[i]['title'].values.tolist():
        print(' %s,' % title, end='')
    print() #add whitespace
    print() #add whitespace<jupyter_output><empty_output><jupyter_text>Resume Code Here Cosine similarity is measured against the tf-idf matrix and can be used to generate a measure of similarity between each document and the other documents in the corpus (each synopsis among the synopses). cosine similarity 1 means the same document, 0 means totally different ones. dist is defined as 1 - the cosine similarity of each document.  Subtracting it from 1 provides cosine distance which I will use for plotting on a euclidean (2-dimensional) plane.Note that with dist it is possible to evaluate the similarity of any two or more synopses.<jupyter_code>#import cosine_similarity from sklearn.metrics.pairwise

#calculate cosine_similarity with argument tfidf_matrix and store in variable cs 

#calculate similarity_distance as 1 - cs

##########   CODE BELOW    ##########



##########   CODE ABOVE    ##########
#import matplotlib.pyplot as plt and configure it to output inline

##########   CODE BELOW    ##########


##########   CODE ABOVE    ##########
#import linkage and dendrogram from scipy.cluster.hierarchy

##########   CODE BELOW    ##########

##########   CODE ABOVE    ##########
#create mergings as an linkage object over similarity_distance.

##########   CODE BELOW    ##########



# Plot the dendrogram, using titles as labels
dendrogram(mergings,
           labels=_____,
           leaf_rotation=90,
           leaf_font_size=16,
)

##########   CODE ABOVE    ##########

fig = plt.gcf()
fig.set_size_inches(108, 21)

plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12147414372061786, 'fraction_numerical': 0.009402283411685695, 'mean_word_length': 4.839705882352941, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '662095', 'license': 'MIT', 'n_tokens_mistral': 3896, 'n_tokens_neox': 3633, 'n_words': 1240, 'path': 'unsupervised_nlp.ipynb', 'repo_name': 'GeekBoySupreme/Colab_Notebooks', 'size': '11932'}