<commit_before>import slate

''' Each wordlist contains tuples containing the words and the number of appearances '''
words_positive = {}
words_negative = {}

''' Reading the wordlists and initialize each counter with zero '''
file_positive = open("../wordlist_positive.txt")
for line in file_positive:
    words_positive[line.rstrip('\n')] = 0

file_negative = open("../wordlist_negative.txt")
for line in file_negative:
    words_negative[line.rstrip('\n')] = 0


with open('../text.pdf') as file:
    document = slate.PDF(file)


for page in document:
    words = page.split(" ")
    for word in words:
        for i in words_negative:
            if i.lower() in word.lower():
                words_negative[i] = words_negative[i] + 1

print words_negative<commit_msg>Counting completed<commit_after>import slate


def contains_word(word, word_list):
    for key in word_list:
            if key.lower() in word.lower():
                word_list[key] += 1

words_positive = {}
words_negative = {}

''' Reading the wordlists and initialize each counter with zero '''
file_positive = open("../wordlist_positive.txt")
for line in file_positive:
    words_positive[line.rstrip('\n')] = 0

file_negative = open("../wordlist_negative.txt")
for line in file_negative:
    words_negative[line.rstrip('\n')] = 0


with open('../text.pdf') as file:
    document = slate.PDF(file)


for page in document:
    words = page.split(" ")
    for word in words:
        contains_word(word, words_negative)
        contains_word(word, words_positive)


count_negative = 0
count_positive = 0

print "Positive:"
print "---------"
for key, value in sorted(words_positive.iteritems(), key=lambda (k, v): (v, k), reverse=True):
    count_positive += value
    print "%s: %s" % (key, value)


print "Negative:"
print "---------------"
for key, value in sorted(words_negative.iteritems(), key=lambda (k, v): (v, k), reverse=True):
    count_negative += value
    print "%s: %s" % (key, value)


print "Total: postive = %s, negative = %s" % (count_positive, count_negative)
{'fraction_non_alphanumeric': 0.1442495126705653, 'fraction_numerical': 0.003898635477582846, 'mean_word_length': 3.482532751091703, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4349331', 'n_tokens_mistral': 664, 'n_tokens_neox': 628, 'n_words': 186}