<jupyter_start><jupyter_text>h/t https://github.com/davidsbatista/text-classification/blob/master/IMDB-movie-genre-classification.ipynb<jupyter_code>%matplotlib inline
import pandas as pd
from IPython.core.display import HTML

css = open("style-table.css").read() + open("style-notebook.css").read()
HTML("<style>{}</style>".format(css))
cast = pd.read_csv('data/cast.csv')
genres = pd.read_csv('movies_genres.csv', sep='\t')
def get_title(t):
    try:
        return t.split('"')[1]
    except:
        return None

def get_year(t):    
    try:
        return round(int(t.split('"')[2][2:].split(')')[0]))
    except:
        return None

genres['full_title'] = genres['title']
genres['title'] = genres['full_title'].apply(get_title)
genres['year'] = genres['full_title'].apply(get_year)
genres.sample(20)
genres.set_index(['title', 'year'], inplace=True)
cast.set_index(['title', 'year'], inplace=True)
cast_gen = cast.join(genres)
list(map(lambda l: l.dtype, cast.index.levels))
genres.head()
cast_gen = cast_gen.dropna()
genre_count = genres.drop(['plot', 'full_title'], axis=1)
genre_count.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
MultiIndex: 117352 entries, (#7DaysLater, 2013.0) to (Über Österreich - Juwele des Landes, 2016.0)
Data columns (total 28 columns):
Action         117352 non-null int64
Adult          117352 non-null int64
Adventure      117352 non-null int64
Animation      117352 non-null int64
Biography      117352 non-null int64
Comedy         117352 non-null int64
Crime          117352 non-null int64
Documentary    117352 non-null int64
Drama          117352 non-null int64
Family         117352 non-null int64
Fantasy        117352 non-null int64
Game-Show      117352 non-null int64
History        117352 non-null int64
Horror         117352 non-null int64
Lifestyle      117352 non-null int64
Music          117352 non-null int64
Musical        117352 non-null int64
Mystery        117352 non-null int64
News           117352 non-null int64
Reality-TV     117352 non-null int64
Romance        117352 non-null int64
Sci-Fi         117352 non-null int64
Short          1[...]<jupyter_text>Which genres are the most popular?<jupyter_code>genre_count.sum(axis=0).sort_values()<jupyter_output><empty_output><jupyter_text>Which words are typically used in plot description of drama movies?<jupyter_code>dramas = genres[genres.Drama == 1]
plots = dramas['plot'].values
txt = ' '.join(plots)

import wordcloud
import matplotlib.pyplot as plt

wc = wordcloud.WordCloud()
cloud = wc.generate(txt)
plt.imshow(cloud)<jupyter_output><empty_output><jupyter_text>Let's try again without almost meaningless words like 'Meanwhile', 'will', 'one'<jupyter_code>type(wordcloud.STOPWORDS)
wordcloud.STOPWORDS.update(['Meanwhile', 'will', 'one'])
cloud = wc.generate(txt)
plt.imshow(cloud)<jupyter_output><empty_output><jupyter_text>How about the Sci-Fi genre?<jupyter_code>scifis = genres[genres['Sci-Fi'] == 1]
plots = scifis['plot'].values
txt = ' '.join(plots)
cloud = wc.generate(txt)
plt.imshow(cloud)<jupyter_output><empty_output><jupyter_text>Hmm, there seem to be a lot of verbs like 'find' and 'take'. How about the nouns? Are there different nonus for drama and science fiction?<jupyter_code>import spacy
from spacy.lang.en import English

nlp = spacy.load("en_core_web_sm")
plots = scifis['plot'].values
txt = ' '.join(plots[:200])
doc = nlp(txt)
nouns = list(filter(lambda w: 'NOUN' == w.pos_, doc))
nouns_txt = ' '.join(map(lambda w: str(w), nouns))
cloud = wc.generate(nouns_txt)
plt.imshow(cloud)
nlp = spacy.load("en_core_web_sm")
plots = dramas['plot'].values
txt = ' '.join(plots[:200])
doc = nlp(txt)
nouns = list(filter(lambda w: 'NOUN' == w.pos_, doc))
nouns_txt = ' '.join(map(lambda w: str(w), nouns))
cloud = wc.generate(nouns_txt)
plt.imshow(cloud)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13883089770354906, 'fraction_numerical': 0.055323590814196244, 'mean_word_length': 4.571220930232558, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '638562', 'license': 'MIT', 'n_tokens_mistral': 1539, 'n_tokens_neox': 1367, 'n_words': 306, 'path': 'Exercises-7.ipynb', 'repo_name': 'dimidd/pycon-pandas-tutorial', 'size': '3832'}