<jupyter_start><jupyter_text>word2vec<jupyter_code>import pandas as pd
data = pd.read_csv('../web_crawler_hyeom/KoreaNewsCrawler/OUTPUT/output/other_categories/Article_사회_202001_202003.csv', header=None, error_bad_lines=False)
data.columns = ['date', 'category', 'source', 'title', 'content', 'url']
print(data.shape)
sample = data.sample(n=10000)
sample.head()
!pip install konlpy
from konlpy.tag import Okt
from tqdm import tqdm
okt = Okt()
sample['content'][262504]
okt.pos(sample['content'][262504])
all_tokens = [
    okt.pos(text) for text in tqdm(sample['content'], desc='tokenize...')
]
['a','a']
new_all_tokens = []
for tokens in tqdm(all_tokens):
    tmp = []
    for t in tokens:
        tmp.append('/'.join(t))
    new_all_tokens.append(tmp)<jupyter_output>0%|          | 0/10000 [00:00<?, ?it/s][A
  6%|▋         | 645/10000 [00:00<00:01, 6447.90it/s][A
 13%|█▎        | 1292/10000 [00:00<00:01, 6090.95it/s][A
 20%|█▉        | 1985/10000 [00:00<00:01, 6102.03it/s][A
 27%|██▋       | 2678/10000 [00:00<00:01, 6166.81it/s][A
 34%|███▎      | 3371/10000 [00:01<00:01, 4984.32it/s][A
 38%|███▊      | 3800/10000 [00:01<00:02, 2388.25it/s][A
 41%|████▏     | 4133/10000 [00:02<00:04, 1392.32it/s][A
 44%|████▍     | 4387/10000 [00:02<00:06, 820.93it/s] [A
 48%|████▊     | 4782/10000 [00:02<00:04, 1043.79it/s][A
 50%|█████     | 5001/10000 [00:02<00:05, 934.25it/s] [A
 59%|█████▉    | 5889/10000 [00:02<00:03, 1277.05it/s][A
 66%|██████▋   | 6630/10000 [00:02<00:01, 1698.82it/s][A
 74%|███████▍  | 7388/10000 [00:02<00:01, 2214.15it/s][A
 80%|████████  | 8045/10000 [00:02<00:00, 2763.70it/s][A
 87%|████████▋ | 8669/10000 [00:03<00:00, 3318.12it/s][A
100%|██████████| 10000/10000 [00:03<00:00, 3077.20it/s][A<jupyter_text>```pythonimport torchtorch.save(new_all_tokens, 'save_dir/all_tokens.torch')```<jupyter_code>%%time
from gensim.models import Word2Vec

model = Word2Vec(
    sentences = new_all_tokens,
    size = 300,
    workers = 10
)<jupyter_output>CPU times: user 1min 21s, sys: 554 ms, total: 1min 22s
Wall time: 20.3 s<jupyter_text>```pythonmodel.save('save_dir/w2v.model')```<jupyter_code>## 학습이 끝나면, 필요없는 메모리 unload
model.init_sims(replace=True)
'코로나/Noun' in list(model.wv.vocab.keys())
for i in sample['content'][:5]:
    print(i, end='\n\n')
model.wv.most_similar('코로나/Noun')
model.wv.most_similar('마스크/Noun')
model.wv.get_vector('코로나/Noun')
class First(object):
    def __init__(self):
        super(First, self).__init__()
        print("first")

class Second(object):
    def __init__(self):
        super(Second, self).__init__()
        print("second")

class Third(First, Second):
    def __init__(self):
        super(Third, self).__init__()
        print("third")<jupyter_output><empty_output><jupyter_text>다시시작<jupyter_code>import torch
new_all_tokens = torch.load('save_dir/all_tokens.torch')

from gensim.models import Word2Vec
model = Word2Vec.load('save_dir/w2v.model')
len(vocabs)
vocabs = list(model.wv.vocab.keys())
word_vectors = [model.wv[v] for v in vocabs]
vocabs = [v for v in vocabs if v.split('/')[-1] == 'Noun']
from sklearn.decomposition import PCA
pca = PCA(n_components=2)
xys = pca.fit_transform(word_vectors)
xs = xys[:,0]
ys = xys[:,1]
import matplotlib.pyplot as plt

plt.rc('font', family='NanumGothic')
def plot_2d_graph(vocabs, xs, ys):
    plt.figure(figsize=(8,6))
    plt.scatter(xs, ys, marker='o')
    for i,v in enumerate(vocabs):
        plt.annotate(v, xy=(xs[i], ys[i]))
from numpy.random import choice
idxs = choice(range(len(vocabs)), size=50, replace=False)

plot_2d_graph(
    [v for i,v in enumerate(vocabs) if i in idxs], 
    [x for i,x in enumerate(xs) if i in idxs], 
    [y for i,y in enumerate(ys) if i in idxs]
)<jupyter_output>/opt/conda/envs/finbert/lib/python3.7/site-packages/matplotlib/backends/backend_agg.py:214: RuntimeWarning: Glyph 8722 missing from current font.
  font.set_text(s, 0.0, flags=flags)
/opt/conda/envs/finbert/lib/python3.7/site-packages/matplotlib/backends/backend_agg.py:183: RuntimeWarning: Glyph 8722 missing from current font.
  font.set_text(s, 0, flags=flags)
{'fraction_non_alphanumeric': 0.1974102125580259, 'fraction_numerical': 0.12069386757879307, 'mean_word_length': 4.5625, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '543414', 'license': 'Apache-2.0', 'n_tokens_mistral': 2197, 'n_tokens_neox': 1819, 'n_words': 308, 'path': 'word2vec_example.ipynb', 'repo_name': 'wonbeeny/Crawling-Word2Vec', 'size': '4093'}