<jupyter_start><jupyter_text>데이터 읽어들이기<jupyter_code>train_data, test_data = [], []

for line in open("data/ratings_train_tokenized.txt", "r", encoding="utf-8").readlines():
    _, tokens, label = line.strip().split("\u241E")
    train_data.append([tokens.split(), label])

for line in open("data/ratings_test_tokenized.txt", "r", encoding="utf-8").readlines():
    _, tokens, label = line.strip().split("\u241E")
    test_data.append([tokens.split(), label])
import pandas as pd
pd.DataFrame(train_data[:10])<jupyter_output><empty_output><jupyter_text>임베딩 읽어들이기<jupyter_code>import numpy as np
from collections import defaultdict

def load_ft_embeddings(path):
    embeddings = defaultdict(list)
    for line in open(path, "r", encoding="utf-8").readlines()[1:]:
        splited_line = line.strip().split(" ")
        word = splited_line[0]
        vec = [float(el) for el in splited_line[1:]]
        embeddings[word] = vec / np.linalg.norm(vec)
    return embeddings
embeddings = load_ft_embeddings("embedding/fasttext.vec")<jupyter_output><empty_output><jupyter_text>word to sentence-level embedding<jupyter_code>def get_sentence_vector(tokens, dim=100):
    vector = np.zeros(dim)
    for token in tokens:
        if token in embeddings.keys():
            vector += embeddings[token]
    if len(tokens) > 1:
        vector /= len(tokens)
    vector_norm = np.linalg.norm(vector)
    if vector_norm != 0:
        unit_vector = vector / vector_norm
    else:
        unit_vector = np.zeros(dim)
    return unit_vector<jupyter_output><empty_output><jupyter_text>train<jupyter_code>train_table = np.zeros([len(train_data), 100], dtype=np.float16)
train_labels = []
for idx, train_instance in enumerate(train_data):
    tokens, label = train_instance
    sent_vec = get_sentence_vector(tokens)
    train_table[idx] = sent_vec
    train_labels.append(label)<jupyter_output><empty_output><jupyter_text>predict<jupyter_code>def predict(tokens):
    sent_vec = get_sentence_vector(tokens)
    scores = np.dot(train_table, sent_vec)
    pred = train_labels[np.argmax(scores)]
    return pred
print(test_data[0])
predict(test_data[0][0])
def predict_by_batch(sentences, labels):
    sentence_vectors, eval_score = [], 0
    for tokens in sentences:
        sentence_vectors.append(get_sentence_vector(tokens))
    scores = np.dot(train_table, np.array(sentence_vectors).T)
    preds = np.argmax(scores, axis=0)
    for pred, label in zip(preds, labels):
        if train_labels[pred] == label:
            eval_score += 1
    return preds, eval_score<jupyter_output><empty_output><jupyter_text>evaluation<jupyter_code>batch_size = 3000
data_size = len(test_data)
num_batches = int((data_size - 1) / batch_size) + 1
eval_score = 0
for batch_num in range(num_batches):
    batch_tokenized_sentences = []
    batch_labels = []
    start_index = batch_num * batch_size
    end_index = min((batch_num + 1) * batch_size, data_size)
    features = test_data[start_index:end_index]
    for feature in features:
        tokens, label = feature
        batch_tokenized_sentences.append(tokens)
        batch_labels.append(label)
    preds, curr_eval_score = predict_by_batch(batch_tokenized_sentences, batch_labels)
    eval_score += curr_eval_score
print("# of correct:", str(eval_score), ", total:", str(len(test_data)), ", score:", str(eval_score / len(test_data)))<jupyter_output># of correct: 35533 , total: 49997 , score: 0.7107026421585295
{'fraction_non_alphanumeric': 0.12478134110787172, 'fraction_numerical': 0.018950437317784258, 'mean_word_length': 4.6430921052631575, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '393701', 'license': 'MIT', 'n_tokens_mistral': 1230, 'n_tokens_neox': 1182, 'n_words': 217, 'path': 'python/nlp/3. Continuous Bag Of Words Model.ipynb', 'repo_name': 'love-adela/snippets', 'size': '3430'}