<jupyter_start><jupyter_text>Modelos de Lenguaje<jupyter_code>%matplotlib inline
import codecs

import numpy as np
import pandas as pd
with codecs.open('food.txt', 'r') as f:
    raw_corpus = []
    for line in f:
        raw_corpus.append('<s> {} </s>'.format(line.strip()))
raw_corpus = ' '.join(raw_corpus)
raw_corpus
corpus = np.array(raw_corpus.split(' '))
corpus<jupyter_output><empty_output><jupyter_text>Modelo de Unigrama<jupyter_code>unique, counts = np.unique(corpus, return_counts=True)
repetitions = dict(zip(unique, counts))
# P(A)
prob_A = repetitions['A'] / corpus.shape[0]
prob_A
# P(B)
prob_B = repetitions['B'] / corpus.shape[0]
prob_B
# P(C)
prob_C = repetitions['C'] / corpus.shape[0]
prob_C<jupyter_output><empty_output><jupyter_text>Modelo de Bigramas<jupyter_code>bigram_matrix = pd.DataFrame(index=unique, columns=unique)
bigram_matrix = bigram_matrix.fillna(0)
bigram_matrix = bigram_matrix.drop(['</s>'], axis=0)
bigram_matrix = bigram_matrix.drop(['<s>'], axis=1)
bigram_matrix
for i, w1 in enumerate(corpus):
    if w1 != '</s>':
        try:
            w2 = corpus[i + 1]
            if w2 != '<s>':
                bigram_matrix.at[w1, w2] += 1
        except IndexError:
            break
bigram_matrix
for word in unique:
    try:
        bigram_matrix.loc[[word]] = (bigram_matrix.loc[[word]] + 1) / (repetitions[word] + unique.shape[0])
    except KeyError:
        continue

bigram_matrix
bigram_matrix.T.plot()
# P(ACB) = P(A)*P(C|A)*P(B|C)
prob_ACB = prob_A * bigram_matrix.at['A', 'C'] * bigram_matrix.at['C', 'B']
prob_ACB
# P(CAB) = 
prob_CAB = prob_C * bigram_matrix.at['C', 'A'] * bigram_matrix.at['A', 'B']
prob_CAB<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16331360946745563, 'fraction_numerical': 0.009467455621301775, 'mean_word_length': 4.437299035369775, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '505262', 'license': 'MIT', 'n_tokens_mistral': 687, 'n_tokens_neox': 678, 'n_words': 122, 'path': 'Sesion3.ipynb', 'repo_name': 'alorozco53/Chatbots-Certificatic', 'size': '1690'}