<jupyter_start><jupyter_text>kor2vec train.py  settings<jupyter_code>import numpy as np
import tensorflow as tf
import collections
from konlpy.tag import Twitter
import re
import math
import random
import pandas as pd

DATA_PATH = "./data/"<jupyter_output><empty_output><jupyter_text>Pre-process Data test using Our data(poke wiki crawl)<jupyter_code># crawling한 데이터를 불러온다.
pk_data = pd.read_csv(DATA_PATH + 'pk_data_g1.csv')

# 단어 별로 자른다.
desc_list = []
words = []
for desc in pk_data['desc'][0].split('.'):
    desc_list.append(desc)
    sentence = re.sub(r"[^ㄱ-힣a-zA-Z0-9]+", ' ', desc).strip().split()
    if sentence:
        words.append(sentence)
        print(sentence)
# 단어의 빈도수를 세고 빈도수가 일정 이상인 단어만 저장한다.
word_counter = [['UNK', -1]] # 빈도수 문제로 word_dict에 없는 word를 처리하기 위함. unknown
word_counter.extend(collections.Counter([word for sentence in words for word in sentence]).most_common())
print(word_counter)
word_counter = [item for item in word_counter if item[1] >= 0 or item[0] == 'UNK']
print(word_counter)

# 저장한 단어의 index를 지정하고 저장.
word_dict = dict()
for word, count in word_counter:
    word_dict[word] = len(word_dict)
word_reverse_dict = dict(zip(word_dict.values(), word_dict.keys()))
print(len(word_dict))
print(word_dict)
print(word_reverse_dict)
# 단어들을 형태소 분석기로 분리한다.
word_to_pos_li = dict()
pos_list = list()
twitter = Twitter()
for w in word_dict:
    w_pos_li = list()
    for pos in twitter.pos(w, norm=True):
        w_pos_li.append(pos)
   
    # 단어별로 형태소 분리하고 원래의 index에 맞게 저장.
    word_to_pos_li[word_dict[w]] = w_pos_li
    pos_list += w_pos_li

# print(word_to_pos_li)
# print("pos_list: ", pos_list)

# 분리된 단어들 counting. (굳이 카운팅하는게 필요한지?)
pos_counter = collections.Counter(pos_list).most_common()
# print(pos_counter)

# 형태소 분석기에 의해 분리된 단어들을 각각 index를 지정하고 저장.
pos_dict = dict()
for pos, _ in pos_counter:
    pos_dict[pos] = len(pos_dict)

pos_reverse_dict = dict(zip(pos_dict.values(), pos_dict.keys()))

print(len(pos_dict))
print("pos_dict : ", pos_dict)
print("pos_reverse_dict : ", pos_reverse_dict)
# 분리된 단어들을 word vector의 index로 매핑하여 분리되기전 index로 저장.
word_to_pos_dict = dict()
print(word_to_pos_li)

for word_id, pos_li in word_to_pos_li.items():
    print(word_id, ' ', pos_li)
    pos_id_li = list()
    for pos in pos_li:
        pos_id_li.append(pos_dict[pos])
    word_to_pos_dict[word_id] = pos_id_li
print(word_to_pos_dict)
# Sub-sampling frequent words according to sampling_rate
# 빈번하게 나오는 단어를 희석한다. 전체 counter가 많아야 의미가 있는것 같음

def sub_sampling(data, word_counter, word_dict, sampling_rate):
    total_words = sum([len(sentence) for sentence in data])
    # print("total_words: {}".format(total_words))
    prob_dict = dict()
    for word, count in word_counter:
        f = count / total_words # 빈도수가 많을수록 f가 1에 가까워짐.
        p = max(0, 1 - math.sqrt(sampling_rate / f)) # sampling_rate가 0.0001이면 f가 클수록 prob이 커진다.
        prob_dict[word_dict[word]] = p
        # print("count : {}, f : {}, p : {}, prob_dict : {}".format(count, f, p, prob_dict))

    new_data = list()
    for sentence in data:
        s = list()
        for word in sentence:
            prob = prob_dict[word]
            if random.random() > prob: # prob이 작을수록 s에 저장되기 쉬움.
                s.append(word)
        new_data.append(s)

    return new_data
#
data = list()
unk_count = 0
for sentence in words:
    s = list()
    for word in sentence:
        print(word)
        if word in word_dict:
            index = word_dict[word]
        else:
            index = word_dict['UNK']
            unk_count += 1
        s.append(index)
        print(s)
    data.append(s)
print("before : ", word_counter)
word_counter[0][1] = max(1, unk_count)
print("after : ", word_counter)
print(data)

# data = sub_sampling(data, word_counter, word_dict, 0.0001)
# print(data)
print(word_reverse_dict[10])
print(word_to_pos_dict[10])
print(pos_reverse_dict[26])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12240184757505773, 'fraction_numerical': 0.008468052347959968, 'mean_word_length': 3.719128329297821, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '612147', 'license': 'MIT', 'n_tokens_mistral': 1652, 'n_tokens_neox': 1734, 'n_words': 356, 'path': '1_word_embedding/kor2vec_preprocess_data_khkim.ipynb', 'repo_name': 'RRoundTable/pocketmon_story', 'size': '3897'}