<gh_stars>0
# %%
"""
pip install konlpy
pip install customized_konlpy
"""

import numpy as np
import pandas as pd
import io
import re
import gensim
from konlpy.tag import Hannanum, Kkma, Komoran, Mecab, Okt
from gensim.models import Word2Vec
from ckonlpy.tag import Twitter
import os.path

story = pd.read_csv("story.csv")

# 토큰화 함수
hannanum = Hannanum() # 대상 text 길이 너무 길면 오류나는듯(대략 한글 12800자 정도)
kkma = Kkma()
komoran = Komoran()
twitter = Twitter()

twitter.add_dictionary('라칸', 'Noun')
twitter.add_dictionary('자야', 'Noun')
twitter.add_dictionary('릴리아', 'Noun')
twitter.add_dictionary('리븐', 'Noun')
twitter.add_dictionary('슈리마', 'Noun')
twitter.add_dictionary('아이오니아', 'Noun')

def tokenizer(row):
  pos_list = twitter.pos(row)
  r = []
  for word in pos_list:
    if not word[1] in ["Josa", "Punctuation", "Foreign", "Suffix", "Eomi", "Determiner", "PreEomi", "Verb"]:
      r.append(word[0])
  rl = (" ".join(r)).strip()
  return twitter.morphs(rl, stem=True)

story['rev_token'] = story['text'].apply(tokenizer)
print(story.head())

print("최대 토큰 개수 :", (max(len(i) for i in story['rev_token'])))
print("최소 토큰 개수 :", (min(len(i) for i in story['rev_token'])))
print("평균 토큰 개수 :", sum(len(i) for i in story['rev_token'])/len(story['rev_token']))

# load word2vec model
if not os.path.isfile('ko.model'):
  existing_model = gensim.models.Word2Vec.load('ko.bin')
  existing_model.save('ko.model')
else:
  existing_model = gensim.models.Word2Vec.load('ko.model')
if not os.path.isfile('ko.bin.gz'):
  existing_model.wv.save_word2vec_format("ko.bin.gz", binary=False)

# min_count = 0 필수
new_model = Word2Vec(size = 200, window = 5, min_count = 0, workers = 4, sg = 1)
new_model.build_vocab(story['rev_token'])
total_examples = new_model.corpus_count
new_model.build_vocab([list(existing_model.wv.vocab.keys())], update=True)
new_model.intersect_word2vec_format("ko.bin.gz", binary=False)
new_model.train(story['rev_token'], total_examples=total_examples, epochs=new_model.iter)

print("existing vocab count:", len(existing_model.wv.vocab.keys()))
print("new vocab count:", len(new_model.wv.vocab.keys()))
model_result = new_model.wv.most_similar('자야')
print(model_result)
new_model.save('new_model.model')
