<jupyter_start><jupyter_text>Word2vec 모델을 학습시키고 사용해보는 것이 목표* 나무위키 덤프 활용 - 정제 - 텍스트로 된 순수 문장 얻기* 형태소 분석 - 형태소 분류* 말뭉치(Corpus) 데이터 준비 * 말뭉치를 활용해서 Word2vec 모델 학습* Word2vec 모델 활용<jupyter_code>import ijson
input_filename = '/Users/swkim/Data/namuwiki180326/namuwiki_20180326.json'
output_filename = '/Users/swkim/Data/namuwiki180326/_namuwiki_20180326_mini.txt'
input_file = open(input_filename, 'r', encoding='utf-8')
texts = ijson.items(input_file, 'item.text')
for index, text in zip(range(10), texts):
    print(strip_wiki_literal(text))
import re

# We want only plain texts, so strip wiki grammer.
# Refer this link to know more about grammar. https://namu.wiki/w/%EB%82%98%EB%AC%B4%EC%9C%84%ED%82%A4:%EB%AC%B8%EB%B2%95%20%EB%8F%84%EC%9B%80%EB%A7%90

# Use regular expression to capture some pattern

# see http://stackoverflow.com/questions/2718196/find-all-chinese-text-in-a-string-using-python-and-regex
chinese = re.compile(u'[⺀-⺙⺛-⻳⼀-⿕々〇〡-〩〸-〺〻㐀-䶵一-鿃豈-鶴侮-頻並-龎]', re.UNICODE)
japanese = re.compile(u'[\u3000-\u303f\u3040-\u309f\u30a0-\u30ff\uff00-\uff9f\u4e00-\u9faf\u3400-\u4dbf]', re.UNICODE)

# hangul = re.compile('[^ ㄱ-ㅣ가-힣]+') # 한글과 띄어쓰기를 제외한 모든 글자
# hangul = re.compile('[^ \u3131-\u3163\uac00-\ud7a3]+')  # 위와 동일
# result = hangul.sub('', s) # 한글과 띄어쓰기를 제외한 모든 부분을 제거


def strip_wiki_literal(text):               
    text = re.sub(r"\{\{\{#\!html[^\}]*\}\}\}", '', text, flags=re.IGNORECASE|re.MULTILINE|re.DOTALL) # remove html
    text = re.sub(r"#redirect .*", '', text, flags=re.IGNORECASE) # remove redirect
    text = re.sub(r"\[\[분류:.*", '', text) # remove 분류
    text = re.sub(r"\[\[파일:.*", '', text) # remove 파일
    text = re.sub(r"\* 상위 문서 ?:.*", '', text) # remove 상위문서        
    text = re.sub(r"\[youtube\(\w+\)\]", '', text, flags=re.IGNORECASE) # remove youtube
    text = re.sub(r"\[include\(([^\]|]*)(\|[^]]*)?\]", r'\1', text, flags=re.IGNORECASE) # remove include
    text = re.sub(r"\[\[(?:[^\]|]*\|)?([^\]|]+)\]\]", r'\1', text) # remove link
    text = re.sub(r"\[\*([^\]]*)\]", '', text) # remove 각주
    text = re.sub(r"\{\{\{([^\ }|]*) ([^\}|]*)\}\}\}", r'\2', text) # remove text color/size
    text = re.sub(r"'''([^']*)'''", r'\1', text) # remove text bold
    text = re.sub(r"(~~|--)([^']*)(~~|--)", '', text) # remove strike-through
    text = re.sub(r"\|\|(.*)\|\|", '', text) # remove table
    text = re.sub(r"\n\s*\n*", '\n', text) # remove empty line                       
    text = chinese.sub('', text) # remove chinese
    text = japanese.sub('', text) # remove japanese
    return text

# output_filename = '/Users/swkim/Data/namuwiki180326/_namuwiki_20180326_mini.txt'

item_limit = 10000
minimum_length = 500

with open(output_filename, 'w', encoding='utf-8') as output_file:
    for index, text in zip(range(item_limit), texts):
        if (len(text) > minimum_length):
            try:
                a_line = strip_wiki_literal(text)
                output_file.write(a_line + '\n')
            except UnicodeEncodeError as e:
                print("UnicodeEncodeError ({0}) : {1}".format(e, text))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.22200647249190938, 'fraction_numerical': 0.04498381877022654, 'mean_word_length': 3.371994342291372, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 6, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '310696', 'license': 'MIT', 'n_tokens_mistral': 1494, 'n_tokens_neox': 1494, 'n_words': 238, 'path': 'nlp/1. Namuwiki json dump into plain text.ipynb', 'repo_name': 'zzsza/TIL', 'size': '3090'}