<jupyter_start><jupyter_text>natural language processing from scratch  Make corpus  import setting<jupyter_code>from collections import Counter, defaultdict
import numpy as np<jupyter_output><empty_output><jupyter_text>split sentence into word<jupyter_code>text = "You say goodbye and I say hello."
# note that space should have to be between first `'` and `(`
text=text.lower()
text=text.replace('.',' .')
splitted_text=text.split(" ")
print(splitted_text)<jupyter_output>['you', 'say', 'goodbye', 'and', 'i', 'say', 'hello', '.']<jupyter_text>convert each word to id<jupyter_code>word_to_id={}
for w in splitted_text:
    if not w in word_to_id:
        new_id=len(word_to_id)
        word_to_id[w] = new_id

id_to_word = {i:w for i,w in word_to_id.items()}
id_to_word<jupyter_output><empty_output><jupyter_text>make corpus<jupyter_code>corpus = np.array([word_to_id[w] for w in splitted_text])
corpus<jupyter_output><empty_output><jupyter_text>create co-occurence matrix<jupyter_code>def create_co_matrix(corpus, vocab_size, window_size=1):
    corpus_size = len(corpus)
    co_matrix = np.zeros((vocab_size, vocab_size), dtype=np.int32)
    
    for c_idx, w_idx in enumerate(corpus):
        for i in range(1, window_size + 1):
            left_idx = c_idx-i
            right_idx = c_idx+i
            if not left_idx < 0:
                left_w_idx = corpus[left_idx]
                co_matrix[w_idx, left_w_idx] += 1
            if not right_idx >= corpus_size:
                right_w_idx = corpus[right_idx]
                co_matrix[w_idx, right_w_idx] += 1
    return co_matrix
co_matrix = create_co_matrix(corpus,len(word_to_id))<jupyter_output><empty_output><jupyter_text>cosine similarity<jupyter_code>def cosine_similarity(xs, ys, eps=1e-8):
    # to prevent zero division we add eps to denominator
    normed_xs = xs/(np.linalg.norm(xs)+eps)
    normed_ys = ys/(np.linalg.norm(ys)+eps)
    cos_sim = np.dot(normed_xs,normed_ys)
    return cos_sim
you_vector = co_matrix[word_to_id["you"]]
i_vector = co_matrix[word_to_id["i"]]
cosine_similarity(you_vector, i_vector)
cosine_similarity(np.ones(5),np.ones(5))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11745589600742803, 'fraction_numerical': 0.005571030640668524, 'mean_word_length': 4.554123711340206, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '129252', 'license': 'MIT', 'n_tokens_mistral': 840, 'n_tokens_neox': 815, 'n_words': 154, 'path': 'naturalLanguageProcessing/countbase_id2word.ipynb', 'repo_name': 'terasakisatoshi/pythonCodes', 'size': '2154'}