<jupyter_start><jupyter_text>1. Compute **bigram** counts in the corpora, ignoring bigrams which contain at least one token that is not a word   (it contains characters other than letters). The text has to be properly normalized before the counts are computed:   it should be downcased and all punctuation should be removed. Given the sentence: "The quick brown fox jumps over the   lazy dog", the bigram counts are as follows:   1. "the quick": 1   1. "quick brown": 1   1. "brown fox": 1   1. etc.1. Use [pointwise mutual information](https://en.wikipedia.org/wiki/Pointwise_mutual_information) to compute the measure    for all pairs of words. 1. Sort the word pairs according to that measure in the descending order and display 30 top results.1. Use [log likelihood ratio](http://tdunning.blogspot.com/2008/03/surprise-and-coincidence.html) (LLR) to compute the measure   for all pairs of words.1. Sort the word pairs according to that measure in the descending order and display 30 top results.<jupyter_code>import os
import requests
import regex
from collections import Counter
import math

def filesNames():
    path = '../ustawy'
    absolute_path = os.path.realpath(path) + "\\"
    return [(absolute_path + filename, filename) for filename in os.listdir(path)]

def getFileTextRaw(path):
    with open(path, 'r', encoding="utf8") as content_file:
        return content_file.read()
    
def cleanText(text):
    lower_text = text.replace('-\n', '').replace('\n', ' ').lower()
    unpunct_text = regex.sub(r"\p{P}", ' ' ,lower_text)
    without_empty_text = regex.sub(r"\s+", ' ' ,unpunct_text).strip()
    return without_empty_text

def splitWords(text):
    return text.split(' ')

def correctWord(word):
    return word.isalpha()

def countsBigrams(words, bigram_counter, left_counter, right_counter):
    bigram_size = 0
    for index, right_word in enumerate(words[1:]):
        left_word = words[index-1]
        if correctWord(right_word) and correctWord(left_word):
            bigram_counter[(left_word, right_word)] += 1
            left_counter[left_word] +=1
            right_counter[right_word] +=1
            bigram_size += 1
    return bigram_size

def shannon(values, N):
    return sum([k / N * math.log(k/N + (k == 0)) for k in values])

def pmi(bigram, val, left_counter, right_counter, bigram_size):
    return math.log((val*bigram_size)/(left_counter[bigram[0]]*right_counter[bigram[1]]))

def llr(bigram, val, left_counter, right_counter, bigram_size):
    k_11 = val
    k_12 = right_counter[bigram[1]] - val
    k_21 = left_counter[bigram[0]] - val
    k_22 = bigram_size - right_counter[bigram[1]] - left_counter[bigram[0]] + val
    return 2 * bigram_size * (shannon([k_11, k_12, k_21, k_22], bigram_size) - shannon([k_11 + k_12, k_21 + k_22], bigram_size) - shannon([k_11 + k_21, k_12 + k_22], bigram_size))

bigram_counter = Counter()
left_counter = Counter()
right_counter = Counter()
bigram_size = 0
for (path, filename) in filesNames():
    content = getFileTextRaw(path)
    clean_text = cleanText(content)
    words = splitWords(clean_text)
    bigram_size += countsBigrams(words, bigram_counter, left_counter, right_counter)
pmi_values = []
llr_values = []
for bigram, val in bigram_counter.items():
    pmi_values.append((bigram, pmi(bigram, val, left_counter, right_counter, bigram_size)))
    llr_values.append((bigram, llr(bigram, val, left_counter, right_counter, bigram_size)))
    
pmi_values.sort(key=lambda x: -x[1])
llr_values.sort(key=lambda x: -x[1])
pmi_values[:30]
llr_values[:30]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11852059400392266, 'fraction_numerical': 0.021574670776127767, 'mean_word_length': 4.288888888888889, 'pattern_counts': {'":': 3, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '594619', 'license': 'MIT', 'n_tokens_mistral': 1227, 'n_tokens_neox': 1170, 'n_words': 337, 'path': '4/4.ipynb', 'repo_name': 'Kotwic4/NLP', 'size': '3569'}