<jupyter_start><jupyter_text>This is the test file to the idea prove.Try to do the Json formatted corpus, but it is so hard, then I find the word2vec can avoid this hard work.<jupyter_code># -*- coding: utf-8 -*-

import os
import re
import time
import codecs
import argparse

TIME_FORMAT = '%Y-%m-%d %H:%M:%S'
BASE_FOLDER = os.getcwd()  # os.path.abspath(os.path.dirname(__file__))
DATA_FOLDER = os.path.join(BASE_FOLDER, 'data')
DEFAULT_FIN = os.path.join(DATA_FOLDER, '唐诗语料库.txt')
DEFAULT_FOUT = os.path.join(DATA_FOLDER, 'poem.txt')
reg_noisy = re.compile('[^\u3000-\uffee]')
reg_note = re.compile('(（.*）)') # Cannot deal with （） in seperate lines
# 中文及全角标点符号(字符)是\u3000-\u301e\ufe10-\ufe19\ufe30-\ufe44\ufe50-\ufe6b\uff01-\uffee<jupyter_output><empty_output><jupyter_text>读取数据，去掉不用的数据<jupyter_code>if __name__ == '__main__':
  #  parser = set_arguments()
  #  cmd_args = parser.parse_args()

    print('{} START'.format(time.strftime(TIME_FORMAT)))

    fd = codecs.open(DEFAULT_FIN, 'r', 'utf-8')
    fw = codecs.open( DEFAULT_FOUT, 'w', 'utf-8')
    reg = re.compile('〖(.*)〗')
    start_flag = False
    for line in fd:
        line = line.strip()
        if not line or '《全唐诗》' in line or '<http'  in line or '□' in line:
            continue
        elif '〖' in line and '〗' in line:
            if start_flag:
                fw.write('\n')
            start_flag = True
            g = reg.search(line)
            if g:
                fw.write(g.group(1))
                fw.write('\n')
            else:a
                # noisy data
                print(line)
        else:
            line = reg_noisy.sub('', line)
            line = reg_note.sub('', line)
            line = line.replace(' .', '')
            fw.write(line)

    fd.close()
    fw.close()

    print('{} STOP'.format(time.strftime(TIME_FORMAT)))<jupyter_output>2017-10-15 14:34:54 START
2017-10-15 14:34:56 STOP<jupyter_text>分词实验DEFAULT_FOUT = os.path.join(DATA_FOLDER, 'poem.txt')  thu1 = thulac.thulac(seg_only=True)   只进行分词，不进行词性标注text = thu1.cut("我爱北京天安门", text=True)  进行一句话分词print(text)thu1 = thulac.thulac(seg_only=True)  只进行分词，不进行词性标注thu1.cut_f(DEFAULT_FOUT, outp)  对input.txt文件内容进行分词，输出到output.txt<jupyter_code>print('{} START'.format(time.strftime(TIME_FORMAT)))

import thulac 
DEFAULT_Segment = os.path.join(DATA_FOLDER, 'wordsegment.txt')

fd = codecs.open(DEFAULT_FOUT, 'r', 'utf-8')
fw = codecs.open(DEFAULT_Segment, 'w', 'utf-8')

thu1 = thulac.thulac(seg_only=True)   #只进行分词，不进行词性标注


for line in fd:
    #print(line)
    fw.write(thu1.cut(line, text=True))
    fw.write('\n')
    
fd.close()
fw.close()

print('{} STOP'.format(time.strftime(TIME_FORMAT)))
    
print('{} START'.format(time.strftime(TIME_FORMAT)))
from gensim.models import word2vec


#DEFAULT_Segment = os.path.join(DATA_FOLDER, 'wordsegment.txt')
DEFAULT_Word2Vec = os.path.join(DATA_FOLDER, 'Word2Vec150.bin')

sentences = word2vec.Text8Corpus(DEFAULT_Segment)

model = word2vec.Word2Vec(sentences, size=150)

#DEFAULT_Segment = os.path.join(DATA_FOLDER, 'wordsegment.txt')
model.save(DEFAULT_Word2Vec)

print('{} STOP'.format(time.strftime(TIME_FORMAT)))
model[u'男']

DEFAULT_FIN = os.path.join(DATA_FOLDER, '唐诗语料库.txt')
DEFAULT_FOUT = os.path.join(DATA_FOLDER, 'poem.txt')
DEFAULT_Segment = os.path.join(DATA_FOLDER, 'wordsegment.txt')
def GetFirstNline(filePath, linesNumber):
    fd = codecs.open(filePath, 'r', 'utf-8')
    for i in range(1,linesNumber):
        print(fd.readline())
    fd.close()

GetFirstNline(DEFAULT_Segment, 3)
GetFirstNline(DEFAULT_FOUT, 3)<jupyter_output>饮马长城 窟行

塞外 悲风切 ， 交河 冰 已 结 。 瀚海 百重波 ， 阴山 千 里 雪 。 迥戍危 烽火 ， 层峦 引高节 。 悠悠 卷 旆旌 ， 饮马 出 长城 。 寒沙 连 骑迹 ， 朔吹断 边声 。 胡尘清玉塞 ， 羌 笛韵 金钲 。 绝漠 干戈戢 ， 车徒 振 原隰 。 都 尉反龙堆 ， 将 军旋 马邑 。 扬 麾氛 雾静 ， 纪石 功名 立 。 荒裔 一戎衣 ， 灵台 凯歌 入 。

饮马长城窟行

塞外悲风切，交河冰已结。瀚海百重波，阴山千里雪。迥戍危烽火，层峦引高节。悠悠卷旆旌，饮马出长城。寒沙连骑迹，朔吹断边声。胡尘清玉塞，羌笛韵金钲。绝漠干戈戢，车徒振原隰。都尉反龙堆，将军旋马邑。扬麾氛雾静，纪石功名立。荒裔一戎衣，灵台凯歌入。<jupyter_text>分词不是很成功，我们转向直接用汉字字符来代替分段，我们保留标点符号<jupyter_code>print('{} START'.format(time.strftime(TIME_FORMAT)))

DEFAULT_FOUT = os.path.join(DATA_FOLDER, 'poem.txt')
DEFAULT_charSegment = os.path.join(DATA_FOLDER, 'Charactersegment.txt')

fd = codecs.open(DEFAULT_FOUT, 'r', 'utf-8')
fw = codecs.open(DEFAULT_charSegment, 'w', 'utf-8')

start_flag = False
for line in fd:
    if len(line) > 0:
        for c in line:
            if c != '\n':
                fw.write(c)
                fw.write(' ')
    fw.write('\n')

fd.close()
fw.close()

print('{} STOP'.format(time.strftime(TIME_FORMAT)))

GetFirstNline(DEFAULT_charSegment, 3)
print('{} START'.format(time.strftime(TIME_FORMAT)))
from gensim.models import word2vec


#DEFAULT_Segment = os.path.join(DATA_FOLDER, 'wordsegment.txt')
DEFAULT_Char2Vec = os.path.join(DATA_FOLDER, 'Char2Vec100.bin')

fd = codecs.open(DEFAULT_charSegment, 'r', 'utf-8')

sentences = fd.readlines()

fd.close


model = word2vec.Word2Vec(sentences, size=100)

#DEFAULT_Segment = os.path.join(DATA_FOLDER, 'wordsegment.txt')
model.save(DEFAULT_Char2Vec)

print('{} STOP'.format(time.strftime(TIME_FORMAT)))
model[u'男']
print('{} START'.format(time.strftime(TIME_FORMAT)))
from gensim.models import word2vec

DEFAULT_charSegment = os.path.join(DATA_FOLDER, 'Charactersegment.txt')
DEFAULT_Char2Vec50 = os.path.join(DATA_FOLDER, 'Char2Vec50.bin')

fd = codecs.open(DEFAULT_charSegment, 'r', 'utf-8')

sentences = fd.readlines()

fd.close

model = word2vec.Word2Vec(sentences, size=50)

#DEFAULT_Segment = os.path.join(DATA_FOLDER, 'wordsegment.txt')
model.save(DEFAULT_Char2Vec50)

print('{} STOP'.format(time.strftime(TIME_FORMAT)))
model.wv.most_similar([u'好'])<jupyter_output><empty_output><jupyter_text>把汉字转成拼音<jupyter_code>from pypinyin import pinyin<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15460122699386503, 'fraction_numerical': 0.020157756354075372, 'mean_word_length': 4.126684636118599, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '109713', 'license': 'MIT', 'n_tokens_mistral': 2754, 'n_tokens_neox': 2741, 'n_words': 411, 'path': 'PrepareData.ipynb', 'repo_name': 'seth2000/chinesepoem', 'size': '5705'}