<jupyter_start><jupyter_text>Starting analysis of horror/non-horror collectionThere are enough remaining issues with our collection that I'm going to add an extra week on this project. No programming homework this week!We will start to look at the 2D plots used in the Allison et al. reading.<jupyter_code>import csv, sys, os, re
from collections import Counter
import numpy

from matplotlib import pyplot

word_pattern = re.compile("\w[\w\-\']*\w|\w")
documents = []

with open("../data/Gutenberg-2019-10-21/metadata.csv", encoding="utf-8") as reader:
    csv_reader = csv.DictReader(reader)
    for document in csv_reader:
        try:
            with open("../data/Gutenberg-2019-10-21/{}".format(document["Filename"]), encoding="utf-8") as reader:
                print(document["Author"] + " / " + document["Title"])

                lines = []
                for line in reader:
                    lines.append(line.rstrip())

                text = " ".join(lines)
                document["Text"] = text
                document["Tokens"] = word_pattern.findall(text)
                
                documents.append(document)
        except Exception as e:
            print("! Problem with {}: {}".format(document["Filename"], e))
all_counts = Counter()

for document in documents:
    doc_counter = Counter(document["Tokens"])
    all_counts += doc_counter   
    document["TokenCounts"] = doc_counter
# Construct a fixed vocabulary

vocabulary = [w for w, c in all_counts.most_common()]
vocabulary_size = len(vocabulary)

reverse_vocab = { w: i for i, w in enumerate(vocabulary) }
def counter_to_vector(counter):
    vector = numpy.zeros(vocabulary_size)
    for word in counter.keys():
        ## look up the integer ID for the string
        word_id = reverse_vocab[word]
        vector[word_id] = counter[word]
    
    return vector
# Convert counters to vectors
doc_word_matrix = numpy.zeros( (len(documents), len(vocabulary)) )

for doc_id, document in enumerate(documents):
    doc_word_matrix[doc_id,:] = counter_to_vector(document["TokenCounts"])
doc_word_matrix[:5,:5]
doc_vectors,S,word_vectors = numpy.linalg.svd(doc_word_matrix, full_matrices=False)
pyplot.figure(figsize=(14, 8))
pyplot.yticks([])
pyplot.xticks(range(75), vocabulary[:75], rotation="vertical")
pyplot.imshow(word_vectors[:5,:75])
pyplot.show()
def show_2d(dimension1, dimension2):
    colors = ["red" if "Horror" in doc["Genre"] else "blue" for doc in documents]
    
    pyplot.figure(figsize=(10,10))
    pyplot.scatter(doc_vectors[:,dimension1], doc_vectors[:,dimension2], c=colors)
    for doc_id, document in enumerate(documents):
        pyplot.text(doc_vectors[doc_id,dimension1], 
                   doc_vectors[doc_id,dimension2],
                   document["Title"])
    pyplot.show()
show_2d(0,1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1255731922398589, 'fraction_numerical': 0.015873015873015872, 'mean_word_length': 3.4174454828660434, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '134435', 'license': 'Apache-2.0', 'n_tokens_mistral': 927, 'n_tokens_neox': 865, 'n_words': 212, 'path': 'week8/HorrorAnalysis.ipynb', 'repo_name': 'mimno/info-3350-fall-2019', 'size': '2835'}