<jupyter_start><jupyter_text>Bag of words<jupyter_code>corpus = [
    '학교에 가서 수업을 들었다. 학교에 간건 오랜만이다.',
    '학교에 가서 친구 얘기를 들었다.',
    '내일 가서 뭐 먹지?'
]
from sklearn.feature_extraction.text import CountVectorizer

vect = CountVectorizer()
vect.fit(corpus)
vect.vocabulary_
vect.transform(corpus).toarray()
vect.transform(['수업을 들었다. 수업은 재미있다.']).toarray()<jupyter_output><empty_output><jupyter_text>TFIDF<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer

tfidv = TfidfVectorizer().fit(corpus)
tfidv.transform(corpus).toarray()<jupyter_output><empty_output><jupyter_text>Word2vec<jupyter_code># make datasets (.txt file)
!wget https://raw.githubusercontent.com/e9t/nsmc/master/ratings_train.txt

import pandas as pd

df = pd.read_csv('ratings_train.txt', sep='\t')
doc = list(df['document'])

with open('ratings_train_text_only.txt', 'w') as f:
  for text in doc:
    f.write(str(text) + '\n')
# read text file
with open('ratings_train_text_only.txt', 'r') as f:
  texts = [str(text).replace('\n', '') for text in doc if len(str(text)) >= 10]
# word2vec training

import os
from gensim.models import Word2Vec

def word2vec(texts):
    inputs = [tt.split(' ') for tt in texts]
    print('number of text = ', len(inputs))

    print('word2vec training...')
    model = Word2Vec(inputs, size=50, window=3, min_count=3, negative=5, workers=os.cpu_count(), iter=10, sg=1)
    model.init_sims

    model.save('word2vec')

word2vec(texts)
w2v = Word2Vec.load('word2vec')
# 단어 벡터
w2v.wv['감동']
# 유사 단어
w2v.wv.most_similar('이제까지')<jupyter_output><empty_output><jupyter_text>Fasttext<jupyter_code># fasttext training

import os
from gensim.models import FastText

def fasttext(texts):
    inputs = [tt.split(' ') for tt in texts]
    print('number of text = ', len(inputs))

    model = FastText(inputs, size=50, window=3, min_count=3, negative=5, workers=os.cpu_count(), iter=10, sg=1)
    model.init_sims()

    model.save('fasttext')
    print('fasttext is trained')

fasttext(texts)
w2v = Word2Vec.load('word2vec')
fasttext = FastText.load('fasttext')
wav.wv.most_similar('고능학교')
fasttext.wv.most_similar('고능학교')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1515291936978684, 'fraction_numerical': 0.016218721037998145, 'mean_word_length': 5.35, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '336794', 'license': 'Apache-2.0', 'n_tokens_mistral': 924, 'n_tokens_neox': 950, 'n_words': 164, 'path': 'notebooks/word_embedding.ipynb', 'repo_name': 'fpem123/LMkor', 'size': '2158'}