<jupyter_start><jupyter_text>[Частоты НКРЯ](http://www.ruscorpora.ru/corpora-freq.html) ---<jupyter_code>txt = open(FILE_TEXT, 'r').read()
txt
frequencies = {}

with open(FILE_UNIGRAMS, 'r') as f:
    for line in f:
        split = line.strip().split('\t')
        try:
            freq, word = split
        except:
            print(split)
        frequencies[word] = float(freq) / N_WORDS
len(frequencies)
frequencies
frequencies['и']
regex_puncts = re.compile('[%s]' % re.escape(string.punctuation))
regex_symbs = re.compile('[%s]' % re.escape(EXCLUDE_SYMBOLS_STR))
regex_digits = re.compile('[%s]' % re.escape(string.digits))
regex_spaces = re.compile('[%s]' % string.printable + string.whitespace)
txt_prep = regex_puncts.sub('', txt)
txt_prep = regex_symbs.sub('', txt_prep)
txt_prep = regex_digits.sub('', txt_prep)
txt_prep = regex_spaces.sub('', txt_prep)
txt_prep = txt_prep.strip().strip('\t').replace('\n', '').lower()
txt_prep
words = txt_prep.split()

print(len(words))
frequencies_empirical = {}

for word in log_progress(words):
    if frequencies_empirical.get(word, None) is None:
        frequencies_empirical[word] = 1
    else:
        frequencies_empirical[word] += 1
def perplexity(docs):
    docs = np.asarray(docs)
    out_of_dict = []
    
    tmp_sum_docs = 0.0
    N = 0.0
    for doc in log_progress(docs):
        tmp_sum_words = 0.0
        for word in log_progress(words):
            freq = frequencies.get(word, TOL)
            freq_empirical = frequencies_empirical.get(word, 0.0)
            N += freq_empirical
            if freq == TOL:
                out_of_dict.append(word)
            n_wd = words[words == word]
            tmp_sum_words += freq_empirical * np.log(freq)
        tmp_sum_docs += tmp_sum_words
        
    return np.exp(-tmp_sum_docs / N), out_of_dict
perp, ood = perplexity([words])

print(perp)
print(np.unique(ood))<jupyter_output>['быковский' 'григорьевых' 'двухтрех' 'докладывавшую' 'евгений' 'евгения'
 'евгеньич' 'игнатий' 'игнатия' 'натальи' 'наталья' 'петрович' 'петровича'
 'семеновна' 'семеновны' 'сергей' 'сережа' 'сереже' 'сережу']
{'fraction_non_alphanumeric': 0.14902705268153774, 'fraction_numerical': 0.004746084480303749, 'mean_word_length': 3.419287211740042, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '604162', 'license': 'Unlicense', 'n_tokens_mistral': 830, 'n_tokens_neox': 831, 'n_words': 143, 'path': 'class 3/p10. perplexity.ipynb', 'repo_name': 'arassadin/HSE_14AMI_NLP-2018', 'size': '2107'}