<jupyter_start><jupyter_text>Monster Classifier<jupyter_code>from os import listdir
from os.path import isfile, join

spiderDir = 'C:\\Users\\peter\\Google Drive\\local\\sw\\ANewKindOfMonster\\experiments\\1-monster-space\\spider-pages-depth-1'
spider_files = [f for f in listdir(spiderDir) if isfile(join(spiderDir, f))]
print (len(spider_files), ' files read')<jupyter_output>7068  files read<jupyter_text>Vocabulary analysis<jupyter_code>def wordListToFreqDict(wordlist):
    wordfreq = [wordlist.count(p) for p in wordlist]
    return dict(list(zip(wordlist,wordfreq)))

def sortFreqDict(freqdict):
    aux = [(freqdict[key], key) for key in freqdict]
    aux.sort()
    aux.reverse()
    return aux<jupyter_output><empty_output><jupyter_text>All word frequencies (except stopwords)<jupyter_code>import nltk
from nltk import word_tokenize
from nltk.corpus import stopwords 

freqs = dict()
stop_words = set(stopwords.words('english')) 

for file in spider_files:
    filename = spiderDir + "\\" + file
    
    with open(filename, 'r', encoding='utf-8') as text:
        content = text.read()
    
    words = word_tokenize(content)
    
    words = [w.lower() for w in words]
    filtered_words = [w for w in words if not w in stop_words]
    
    for word in filtered_words:
        if word in freqs:
            freqs[word] += 1
        else:
            freqs[word] = 1

freqs = sortFreqDict(freqs)
freqs<jupyter_output><empty_output><jupyter_text>Frequencies of non-glove words<jupyter_code>import numpy as np

def loadGloveModel(File):
    print("Loading Glove Model")
    f = open(File,'r', encoding='utf-8')
    gloveModel = {}
    for line in f:
        splitLines = line.split()
        word = splitLines[0]
        wordEmbedding = np.array([float(value) for value in splitLines[1:]])
        gloveModel[word] = wordEmbedding
    print(len(gloveModel)," words loaded!")
    return gloveModel
glove = loadGloveModel('c:\pfd\glove.6B.100d.txt')
non_glove_freqs = dict()

for file in spider_files:
    filename = spiderDir + "\\" + file
    
    with open(filename, 'r', encoding='utf-8') as text:
        content = text.read()
    
    words = word_tokenize(content)
    
    words = [w.lower() for w in words]
    filtered_words = [w for w in words if not w in stop_words]
    
    for word in filtered_words:
        if not word in glove:
            if word in non_glove_freqs:
                non_glove_freqs[word] += 1
            else:
                non_glove_freqs[word] = 1

non_glove_freqs = sortFreqDict(non_glove_freqs)
non_glove_freqs<jupyter_output><empty_output><jupyter_text>Classifier  Frequent-word histogram features<jupyter_code>train_pos_dir='C:\\Users\\peter\\Google Drive\\local\\sw\\ANewKindOfMonster\\experiments\\1-monster-space\\labeled\\train\\positive'
train_neg_dir='C:\\Users\\peter\\Google Drive\\local\\sw\\ANewKindOfMonster\\experiments\\1-monster-space\\labeled\\train\\negative'
test_pos_dir='C:\\Users\\peter\\Google Drive\\local\\sw\\ANewKindOfMonster\\experiments\\1-monster-space\\labeled\\test\\positive'
test_neg_dir='C:\\Users\\peter\\Google Drive\\local\\sw\\ANewKindOfMonster\\experiments\\1-monster-space\\labeled\\test\\negative'

# Use this many words in the histogram
vector_size = 1000
pos_train_files = [f for f in listdir(train_pos_dir) if isfile(join(train_pos_dir, f))]
neg_train_files = [f for f in listdir(train_neg_dir) if isfile(join(train_neg_dir, f))]

train_freqs = dict()
stop_words = set(stopwords.words('english')) 

pos_train_files = [join(train_pos_dir, f) for f in pos_train_files ]
neg_train_files = [join(train_neg_dir, f) for f in neg_train_files ]

train_files = pos_train_files + neg_train_files;

for filename in train_files:
    
    with open(filename, 'r', encoding='utf-8') as text:
        content = text.read()
    
    words = word_tokenize(content)
    
    lc_words = [w.lower() for w in words]
    filtered_words = [w for w in lc_words if not w in stop_words]
    
    for word in filtered_words:
        if word in train_freqs:
            train_freqs[word] += 1
        else:
            train_freqs[word] = 1

train_freqs = sortFreqDict(train_freqs)
top_words = [w[1] for w in train_freqs][:vector_size]
top_words
def wordHistogram(text, feature_word_list):
    words = word_tokenize(text)
    lc_words = [w.lower() for w in words]
    
    histogram = []
    for fw in feature_word_list:
        histogram.append(lc_words.count(fw))
    return histogram        
train_data = []
labels = []

for filename in pos_train_files:
    with open(filename, 'r', encoding='utf-8') as text:
        content = text.read()
        
    histo=wordHistogram(content, top_words)
    train_data.append(histo)
    labels.append('monster')

for filename in neg_train_files:
    with open(filename, 'r', encoding='utf-8') as text:
        content = text.read()
        
    histo=wordHistogram(content, top_words)
    train_data.append(histo)
    labels.append('non-monster')

import matplotlib.pyplot as plt
from sklearn import datasets
from sklearn import svm
from sklearn.naive_bayes import GaussianNB
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import make_pipeline

##classifier = svm.SVC(gamma=0.001, C=100)
#classifier = svm.SVC(gamma=0.01, C=10)
#classifier.fit(train_data,labels)

#F1=.82
std_clf = make_pipeline(StandardScaler(), GaussianNB())

#std_clf = make_pipeline(StandardScaler(),svm.SVC(gamma=0.01, C=10))
std_clf.fit(train_data, labels)<jupyter_output><empty_output><jupyter_text>Performance Check<jupyter_code>pos_test_files = [f for f in listdir(test_pos_dir) if isfile(join(test_pos_dir, f))]
neg_test_files = [f for f in listdir(test_neg_dir) if isfile(join(test_neg_dir, f))]

pos_test_files = [(join(test_pos_dir, f)) for f in pos_test_files]
neg_test_files = [(join(test_neg_dir, f)) for f in neg_test_files]

tp = 0
fn = 0
for filename in pos_test_files:
    print(filename)
    with open(filename, 'r', encoding='utf-8') as text:
        content = text.read()
        
    histo=wordHistogram(content, top_words)
    features = []
    features.append(histo)
    
    #result = classifier.predict(features)
    result = std_clf.predict(features)
    if (result == 'monster'):
        tp += 1
    if (result == 'non-monster'):
        fn += 1
    print(result)

fp = 0
tn = 0
for filename in neg_test_files:
    print(filename)
    with open(filename, 'r', encoding='utf-8') as text:
        content = text.read()
        
    histo=wordHistogram(content, top_words)
    features = []
    features.append(histo)
    
    #result = classifier.predict(features)
    result = std_clf.predict(features)
    if (result == 'monster'):
        fp += 1
    if (result == 'non-monster'):
        tn += 1
    print(result)

    
print('tp:', tp)
print('tn:', tn)
print('fp:', fp)
print('fn:', fn)

f1 = tp / (tp + 0.5*(fp+fn))
print('f1: ', f1)
print('precision: ', tp/(tp+fp))
print('recall: ', tp/(tp+fn))<jupyter_output>C:\Users\peter\Google Drive\local\sw\ANewKindOfMonster\experiments\1-monster-space\labeled\test\positive\Alien_(creature_in_Alien_franchise).txt
['monster']
C:\Users\peter\Google Drive\local\sw\ANewKindOfMonster\experiments\1-monster-space\labeled\test\positive\Bigfoot.txt
['non-monster']
C:\Users\peter\Google Drive\local\sw\ANewKindOfMonster\experiments\1-monster-space\labeled\test\positive\Chimera_(mythology).txt
['monster']
C:\Users\peter\Google Drive\local\sw\ANewKindOfMonster\experiments\1-monster-space\labeled\test\positive\Cthulhu.txt
['monster']
C:\Users\peter\Google Drive\local\sw\ANewKindOfMonster\experiments\1-monster-space\labeled\test\positive\Doppelganger_(Dungeons_%26_Dragons).txt
['monster']
C:\Users\peter\Google Drive\local\sw\ANewKindOfMonster\experiments\1-monster-space\labeled\test\positive\Elemental_(Dungeons_%26_Dragons).txt
['monster']
C:\Users\peter\Google Drive\local\sw\ANewKindOfMonster\experiments\1-monster-space\labeled\test\positive\Ghost.txt
['monster']
C:[...]<jupyter_text>Performance check using sklearn's confusion matrix feature<jupyter_code>from sklearn.metrics import plot_confusion_matrix
import matplotlib.pyplot as plt

test_data = []
test_labels = []

for filename in pos_test_files:
    with open(filename, 'r', encoding='utf-8') as text:
        content = text.read()
        
    histo=wordHistogram(content, top_words)
    test_data.append(histo)
    test_labels.append('monster')

for filename in neg_test_files:
    with open(filename, 'r', encoding='utf-8') as text:
        content = text.read()
        
    histo=wordHistogram(content, top_words)
    test_data.append(histo)
    test_labels.append('non-monster')

class_names = ['monster', 'non-monster']
    
# Plot non-normalized confusion matrix
titles_options = [("Confusion matrix, without normalization", None),
                  ("Normalized confusion matrix", 'true')]
for title, normalize in titles_options:
    disp = plot_confusion_matrix(std_clf, test_data, test_labels,
                                 display_labels=class_names,
                                 cmap=plt.cm.Blues,
                                 normalize=normalize)
    disp.ax_.set_title(title)

    print(title)
    print(disp.confusion_matrix)

plt.show()<jupyter_output>Confusion matrix, without normalization
[[14  2]
 [ 4 13]]
Normalized confusion matrix
[[0.875      0.125     ]
 [0.23529412 0.76470588]]<jupyter_text>Use the Monster classifier to find new monster pages in the spider output<jupyter_code>abs_spider_files = [(join(spiderDir, f)) for f in spider_files]

monster_pages = []

for filename in abs_spider_files:
    with open(filename, 'r', encoding='utf-8') as text:
        content = text.read()
        
    histo=wordHistogram(content, top_words)
    features = []
    features.append(histo)
    
    result = std_clf.predict(features)
    if (result == 'monster'):
        monster_pages.append(filename)
        print(result, ' --- ', filename)

print('found ', len(monster_pages), ' monster pages')<jupyter_output>['monster']  ---  C:\Users\peter\Google Drive\local\sw\ANewKindOfMonster\experiments\1-monster-space\spider-pages-depth-1\-SLASH-Film.txt
['monster']  ---  C:\Users\peter\Google Drive\local\sw\ANewKindOfMonster\experiments\1-monster-space\spider-pages-depth-1\-zilla.txt
['monster']  ---  C:\Users\peter\Google Drive\local\sw\ANewKindOfMonster\experiments\1-monster-space\spider-pages-depth-1\10 Cloverfield Lane.txt
['monster']  ---  C:\Users\peter\Google Drive\local\sw\ANewKindOfMonster\experiments\1-monster-space\spider-pages-depth-1\1001 Movies You Must See Before You Die.txt
['monster']  ---  C:\Users\peter\Google Drive\local\sw\ANewKindOfMonster\experiments\1-monster-space\spider-pages-depth-1\101 Names of God.txt
['monster']  ---  C:\Users\peter\Google Drive\local\sw\ANewKindOfMonster\experiments\1-monster-space\spider-pages-depth-1\12 Monkeys.txt
['monster']  ---  C:\Users\peter\Google Drive\local\sw\ANewKindOfMonster\experiments\1-monster-space\spider-pages-depth-1\1782 Central At[...]
{'fraction_non_alphanumeric': 0.1294931294931295, 'fraction_numerical': 0.013013013013013013, 'mean_word_length': 3.7473002159827216, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '455501', 'license': 'MIT', 'n_tokens_mistral': 3965, 'n_tokens_neox': 3836, 'n_words': 736, 'path': 'experiments/1-monster-space/Monster-Classifier.ipynb', 'repo_name': 'peterfrostdavid/ANewKindOfMonster', 'size': '10989'}