<jupyter_start><jupyter_text>数据读取<jupyter_code>data_path = Path('/media/bnu/data/nlp-practice/text-generation/Winston_Churchil.txt')
with open(data_path) as f:
    raw_text = f.read()
raw_text = raw_text.lower()

sentences = nltk.sent_tokenize(raw_text)
print('Sentence Sample:')
print(sentences[20])
corpus = [nltk.word_tokenize(s) for s in sentences]
print('Corpus Sample:')
print(corpus[20])
print('Corpus Length:', len(corpus))<jupyter_output>Corpus Sample:
['it', 'was', 'after', 'the', 'surrender', 'of', 'the', 'confederate', 'army', ',', 'that', ',', 'with', 'other', 'southern', 'officers', ',', 'he', 'served', 'under', 'maximilian', 'in', 'mexico', ';', 'in', 'egypt', ',', 'and', 'in', 'france', '.']
Corpus Length: 1792<jupyter_text>生成Word2Vec词向量<jupyter_code>w2v = Word2Vec(corpus, 
               size=128,     # 词向量的维度
               window=5,     # 上下文的最大距离
               min_count=2,  # 最小词频, 过滤低频词
               workers=4     # 并行数
)
print('Word2Vec Sample:')
print(w2v.wv['france'][:5])<jupyter_output>Word2Vec Sample:
[-0.0228541  -0.00161459 -0.01522535  0.01911901  0.04889536]<jupyter_text>构造训练集<jupyter_code>raw_data = [w for s in corpus for w in s]
print('Raw Data Length:', len(raw_data))

# 过滤掉不在word2vec中的低频词
filter_data = []
for w in raw_data:
    if w in w2v.wv.vocab:
        filter_data.append(w)
print('Filter Data Length:', len(filter_data))
sequence_length = 10

x_train, y_train = [], []
for i in range(len(filter_data) - sequence_length):
    x_temp = filter_data[i: i+sequence_length]
    y_temp = filter_data[i+sequence_length]
    x_train.append([w2v.wv[w] for w in x_temp])
    y_train.append(w2v.wv[y_temp])
    
x_train = np.reshape(x_train, (-1, sequence_length, 128))
y_train = np.reshape(y_train, (-1, 128))
print('X Train Shape:', x_train.shape)
print('Y Train Shape:', y_train.shape)<jupyter_output>X Train Shape: (51866, 10, 128)
Y Train Shape: (51866, 128)<jupyter_text>构建和训练模型<jupyter_code>model = Sequential()
model.add(LSTM(256, dropout=0.2,      # x与h之间的dropout
               recurrent_dropout=0.2, # h与h之间的dropout
               input_shape=(sequence_length, 128)))
model.add(Dropout(0.2))
model.add(Dense(128, activation='sigmoid'))
model.compile(loss='mse', optimizer='adam')
model.fit(x_train, y_train, epochs=50, batch_size=4096, validation_split=0.1)<jupyter_output>Train on 46679 samples, validate on 5187 samples
Epoch 1/50
46679/46679 [==============================] - 1s 18us/step - loss: 0.2804 - val_loss: 0.1297
Epoch 2/50
46679/46679 [==============================] - 0s 8us/step - loss: 0.1311 - val_loss: 0.1102
Epoch 3/50
46679/46679 [==============================] - 0s 8us/step - loss: 0.1228 - val_loss: 0.1081
Epoch 4/50
46679/46679 [==============================] - 0s 8us/step - loss: 0.1213 - val_loss: 0.1073
Epoch 5/50
46679/46679 [==============================] - 0s 8us/step - loss: 0.1205 - val_loss: 0.1069
Epoch 6/50
46679/46679 [==============================] - 0s 8us/step - loss: 0.1200 - val_loss: 0.1067
Epoch 7/50
46679/46679 [==============================] - 0s 8us/step - loss: 0.1195 - val_loss: 0.1064
Epoch 8/50
46679/46679 [==============================] - 0s 8us/step - loss: 0.1191 - val_loss: 0.1063
Epoch 9/50
46679/46679 [==============================] - 0s 8us/step - loss: 0.1187 - val_loss: 0.1059
Epoch 10/50
46[...]<jupyter_text>模型预测<jupyter_code>def predict_next(input_array):
    """根据输入的特征预测下一个单词"""
    test_data = np.reshape(input_array, (1, sequence_length, 128))
    return model.predict(test_data)

def string_to_vector(input_string):
    """将文本中末尾的单词转换为word2vec向量"""
    input_string = input_string.lower()
    input_string = nltk.word_tokenize(input_string)
    return [w2v.wv[w] for w in input_string[-sequence_length:]]

def pred_to_word(pred):
    """根据预测值获取最相似的单词"""
    return w2v.wv.most_similar(positive=pred, topn=5)[0][0]

def generate_text(init_string, steps=30):
    """根据初始字符串生成文本"""
    result = init_string.lower()
    for i in range(steps):
        w = pred_to_word(predict_next(string_to_vector(result)))
        result += ' ' + w
    return result
init_string = 'His object in coming to New York was to engage officers for that service.'
result = generate_text(init_string)
print(result)<jupyter_output>his object in coming to new york was to engage officers for that service. beginning beginning beginning tuesday smile smile smile smile smile smile tuesday tuesday tuesday tuesday tuesday tuesday tuesday tuesday tuesday tuesday tuesday tuesday tuesday tuesday tuesday tuesday tuesday tuesday tuesday tuesday
{'fraction_non_alphanumeric': 0.19640114110160192, 'fraction_numerical': 0.08185209567698047, 'mean_word_length': 4.6975, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '638677', 'license': 'MIT', 'n_tokens_mistral': 2015, 'n_tokens_neox': 1804, 'n_words': 364, 'path': 'notebooks/exercises/Keras_WordRNN_TextGeneration.ipynb', 'repo_name': 'RegiusQuant/nlp-practice', 'size': '4557'}