<jupyter_start><jupyter_text>自动写作  文本预处理<jupyter_code># 载入所需工具包
import numpy as np
import pandas as pd

from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import LSTM
from keras.callbacks import ModelCheckpoint
from keras.utils import np_utils
df = pd.read_csv(r"D:\资料\教程\NLP基础-网易云课堂\TMData190320\PythonData\金庸-射雕英雄传txt精校版.txt", names=["txt"], sep='aaa', encoding="gbk")

# 构建新列
print(len(df))
print(df.shape)
def m_head(tmpstr):
    return tmpstr[:1]

def m_mid(tmpstr):
    return tmpstr.find("回 ")

df["head"] = df.txt.apply(m_mid)
df["mid"] = df.txt.apply(m_head)
df["len"] = df.txt.apply(len)

# 标明章序号
chapnum = 0
for i in range(len(df)):
    if df["mid"][i] == "第" and df["head"][i] > 0 and df["len"][i] < 30:
        chapnum += 1
    if chapnum >= 40 and df.txt[i] == "附录一：成吉思汗家族":
        chapnum = 0
    df.loc[i, "chap"] = chapnum

dfgrp = df.groupby("chap")
chapter = dfgrp.apply(sum)
chapter = chapter[chapter.index != 0]
chapter.head()
rawtxt = pd.read_csv(r"D:\资料\教程\NLP基础-网易云课堂\TMData190320\PythonData\r&j.txt",
                     sep = 'aaaaa', names = ['txt'], engine = 'python', encoding="utf-8")
rawtxt.head()
rawtxt.txt[1]
# 处理大小写，末尾增加空格
def m_perproc(tmpstr):
    return (tmpstr + " ").lower()

rawtxt.txt = rawtxt.txt.apply(m_perproc)
rawtxt.txt[1]
raw_txt = rawtxt.txt.apply("sum")
raw_txt
# 将字符转换为数值代码以便处理
chars = sorted(list(set(raw_txt))) # 生成字符list
char_to_int = dict((c, i) for i, c in enumerate(chars)) # 字符-数值对应字典
int_to_char = dict((i, c) for i, c in enumerate(chars)) # 数值-字符对应字典
chars<jupyter_output><empty_output><jupyter_text>构造训练测试集<jupyter_code>seq_length = 100
x = []; y = []
for i in range(0, len(raw_txt) - seq_length):
    given = raw_txt[i:i + seq_length] # 将前seq_length个字符作为预测用变量
    predict = raw_txt[i + seq_length] # 将当前字符作为因变量
    x.append([char_to_int[char] for char in given])
    y.append(char_to_int[predict])
x[:3]
y[:3]

n_patterns = len(x)
n_vocab = len(chars)

# 把x变成LSTM需要的格式，reshape最后的1表示每个数值均为单独一个向量（代表一个字母输入）
x = np.reshape(x, (n_patterns, seq_length, 1)) 
x = x / float(n_vocab) # 转换为0-1之间的数值以方便计算
x[0]
# 将因变量的类型正确指定为类别
y = np_utils.to_categorical(y) 
y[0]<jupyter_output><empty_output><jupyter_text>建立LSTM模型<jupyter_code>model = Sequential() 
# LSTM层指定为128个神经元
model.add(LSTM(128, input_shape = (x.shape[1], x.shape[2]))) 
model.add(Dropout(0.2)) # 抛弃20%的结果，防止过拟合
model.add(Dense(y.shape[1], activation = 'softmax')) # 使用标准的NN作为内核
# 指定损失函数
model.compile(loss = 'categorical_crossentropy', optimizer = 'adam') 
# batch_size为分批量将数据用于训练，以减小计算资源的需求
# epochs次数越多，模型训练效果越好，但所需时间也线性增加
model.fit(x, y, epochs = 2, batch_size = 64)<jupyter_output>WARNING:tensorflow:From d:\python_install\venv_set\py353\dataanalysiscase\lib\site-packages\tensorflow\python\ops\math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.
Instructions for updating:
Use tf.cast instead.
Epoch 1/2
149672/149672 [==============================] - 197s 1ms/step - loss: 2.9230
Epoch 2/2
149672/149672 [==============================] - 178s 1ms/step - loss: 2.8044<jupyter_text>进行文本预测<jupyter_code>def predict_next(input_array): # 进行下一个字符的预测
    x = np.reshape([0 for i in range(seq_length - len(input_array))]
                   + input_array, (1, seq_length, 1)) # 生成预测用的x序列
    x = x / float(n_vocab)
    y = model.predict(x)
    return y

def string_to_index(raw_input): # 将输入的字符转换为索引值
    res = []
    for c in raw_input[(len(raw_input) - seq_length):]:
        res.append(char_to_int[c])
    return res

def y_to_char(y): # 将预测结果由索引值转换回字符
    largest_index = y.argmax() # 取最大数值对应的索引值
    c = int_to_char[largest_index]
    return c
def generate_article(init, rounds = 50): # 按照指定的字符长度进行预测
    in_string = init.lower()
    for i in range(rounds):
        n = y_to_char(predict_next(string_to_index(in_string)))
        in_string += n # 将预测到的新字符合并，用于下一步预测
    return in_string
# 进行字母预测
init = 'We produce about two million dollars for each hour we work. The fifty hours is one conservative estimate for how long'
article = generate_article(init)
article<jupyter_output><empty_output><jupyter_text>将LSTM与word2vec结合实现中文自动写作<jupyter_code># 载入所需工具包
import jieba
from gensim.models.word2vec import Word2Vec

import numpy as np
import pandas as pd

from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout
from keras.layers import LSTM
from keras.callbacks import ModelCheckpoint
from keras.utils import np_utils
jinyong_words_path = r'D:\资料\教程\NLP基础-网易云课堂\TMData190320\PythonData\金庸小说词库.txt'
jieba.load_userdict(jinyong_words_path) # dict为自定义词典的路径

# 以整句或者整段为基本单位进行分析显然更为合适
corpus = [jieba.lcut(item) for item in df.txt]
corpus[:3]
# 此处完全可以使用外部语料库进行更全面的训练
w2v_model = Word2Vec(corpus, size = 100, window = 5, 
                     min_count = 5, workers = 4)
w2v_model.wv['郭啸天']
# 将数据还原为一个长list
raw_input = [item for sublist in corpus for item in sublist]
print(len(raw_input))
raw_input[:10]
# 列出模型中纳入的词条
vocab = w2v_model.wv.vocab
vocab
# min_count = 5参数会过滤掉低频词，因此需要在文本中同步清除这些低频词
text_stream = []

for word in raw_input:
    if word in vocab:
        text_stream.append(word)

print(len(text_stream))
text_stream[:10]<jupyter_output>558799<jupyter_text>构造训练集和测试集<jupyter_code>seq_length = 10 # 取前面10个单词用于预测
x = []; y = []
for i in range(0, len(text_stream) - seq_length):
    given = text_stream[i : i + seq_length]
    predict = text_stream[i + seq_length]
    x.append(np.array([w2v_model.wv[word] for word in given]))
    y.append(w2v_model.wv[predict])
len(x)
x[0][0]
y[0]
x = np.reshape(x, (-1, seq_length, 100)) # 每一个词条，对应一个word2vec向量
y = np.reshape(y, (-1, 100))<jupyter_output><empty_output><jupyter_text>建立LSTM模型<jupyter_code>model = Sequential()
model.add(LSTM(128, input_shape = (seq_length, 100)))
model.add(Dropout(0.2))
model.add(Dense(100, activation = 'sigmoid'))
model.compile(loss = 'mse', optimizer = 'adam')
model.fit(x, y, epochs = 5, batch_size = 64)
model.summary()
model.save_weights('LSTM.h5') # 文件类型是HDF5
model.load_weights('LSTM.h5')
model.fit(x, y, epochs = 10) # 按照指定的数据和参数继续训练模型
from keras.callbacks import ModelCheckpoint

checkpointer = ModelCheckpoint(filepath = 'LSTM_best.hdf5', 
                               monitor = 'val_loss', 
                               save_best_only = True, 
                               verbose = 1)
model.compile(loss = 'mse', optimizer = 'adam')
model.fit(x, y, epochs = 50, 
          validation_data = (x, y), 
          callbacks = [checkpointer])<jupyter_output>Train on 558789 samples, validate on 558789 samples
Epoch 1/50
558789/558789 [==============================] - 157s 281us/step - loss: 0.2804 - val_loss: 0.2789

Epoch 00001: val_loss improved from inf to 0.27894, saving model to LSTM_best.hdf5
Epoch 2/50
558789/558789 [==============================] - 152s 273us/step - loss: 0.2802 - val_loss: 0.2789

Epoch 00002: val_loss improved from 0.27894 to 0.27889, saving model to LSTM_best.hdf5
Epoch 3/50
558789/558789 [==============================] - 153s 273us/step - loss: 0.2802 - val_loss: 0.2786

Epoch 00003: val_loss improved from 0.27889 to 0.27858, saving model to LSTM_best.hdf5
Epoch 4/50
558789/558789 [==============================] - 152s 272us/step - loss: 0.2800 - val_loss: 0.2786

Epoch 00004: val_loss improved from 0.27858 to 0.27856, saving model to LSTM_best.hdf5
Epoch 5/50
558789/558789 [==============================] - 153s 273us/step - loss: 0.2799 - val_loss: 0.2784

Epoch 00005: val_loss improved from 0.27856 to 0.[...]<jupyter_text>文本预测<jupyter_code>def predict_next(input_array):
    x = np.reshape(input_array, (-1, seq_length, 100))
    y = model.predict(x)
    return y

def string_to_index(raw_input):
    input_stream = []
    for word in jieba.lcut(raw_input):
        if word in vocab:
            input_stream.append(word)
    res = []
    for word in input_stream[(len(input_stream) - seq_length):]:
        res.append(w2v_model.wv[word])
    return res

def y_to_word(y):
    word = w2v_model.wv.most_similar(positive = y, topn = 1)
    return word
def generate_article(init, rounds = 10):
    in_string = init.lower()
    for i in range(rounds):
        n = y_to_word(predict_next(string_to_index(in_string)))
        in_string += n[0][0]
    return in_string
init = '杨过道：“靖康年间徽钦二帝被金兵掳去这件大耻，郭啸天、杨铁心越听越怒。'
article = generate_article(init)
print(article)<jupyter_output>杨过道：“靖康年间徽钦二帝被金兵掳去这件大耻，郭啸天、杨铁心越听越怒。哼快逃别生气黄蓉喜活不成赌气小朋友先赌气活不成
{'fraction_non_alphanumeric': 0.14977557287975432, 'fraction_numerical': 0.05220883534136546, 'mean_word_length': 4.424087123638693, 'pattern_counts': {'":': 1, '<': 30, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '175069', 'license': 'MIT', 'n_tokens_mistral': 4027, 'n_tokens_neox': 3897, 'n_words': 683, 'path': 'KnowledgeMapping/NLP_code/1-code/nlp3.ipynb', 'repo_name': 'nickliqian/ralph_doc_to_chinese', 'size': '8466'}