<jupyter_start><jupyter_text>训练Word2Vec模型<jupyter_code># 加载通用词 stopwords
import jieba
def getStopwords(path):
    stopwords = []
    with open(path, "r", encoding='utf8') as f:
        lines = f.readlines()
        for line in lines:
            stopwords.append(line.strip())
    return stopwords
stop_words_path = "./data/baidu_stopwords.txt"
stopwords = getStopwords(stop_words_path)
# 分词 去停用词
def segment_line(data_path, data_segment_path):
    with open(data_path, 'r', encoding='utf-8') as fo:
        document = fo.read()
        document_cut = jieba.cut(document)
        segment_data = []
        for word in document_cut:
            if word not in stopwords:
                segment_data.append(word)
        result = ' '.join(segment_data)
        with open(data_segment_path, 'w', encoding='utf-8') as fw:
            fw.write(result)
        print("success")
data_segment_path = './data/toutiao_news_dataset_segment.txt'
datas_path = './data/toutiao_news_dataset.txt'
segment_line(datas_path, data_segment_path)
from gensim.models import word2vec
import multiprocessing

def save_wordVectors(w2vModel,word2vec_path):
    w2vModel.save(word2vec_path)
 
def load_wordVectors(word2vec_path):
    w2vModel = word2vec.Word2Vec.load(word2vec_path)
    return w2vModel
 
def trainvec():
    segment_dir= './data/toutiao_news_dataset_segment.txt'
    sentences = word2vec.LineSentence(segment_dir)
 
    # 简单的训练
    model = word2vec.Word2Vec(sentences, hs=1,min_count=1,window=3,size=100)
    
    word2vec_path='./model/word2Vec.model'
    save_wordVectors(model,word2vec_path)
#     model=load_wordVectors(word2vec_path)
    return model

# model = trainvec()
word2vec_path='./model/word2Vec.model'
model=load_wordVectors(word2vec_path)
model['文化'].shape<jupyter_output>/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:1: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).
  """Entry point for launching an IPython kernel.<jupyter_text>embedding weights<jupyter_code>import jieba
from tqdm import tqdm

# 读取原始数据集分词预处理 并保存词典
def read_toutiao_dataset(data_path, save_vocab_path):
    with open(data_path, "r", encoding="utf8") as fo:
        all_lines = fo.readlines()
    datas, labels = [], []
    word_vocabs = {}
    for line in tqdm(all_lines):
        content_words = []
        category, content = line.strip().split("_!_")
        for term in jieba.lcut(content):
            if term not in stopwords and term not in word_vocabs and term != ' ' and term != '':
                word_vocabs[term] = len(word_vocabs)+1
            content_words.append(term)
        datas.append(content_words)
        labels.append(category)
    with open(save_vocab_path, "w", encoding="utf8") as fw:
        for word, index in word_vocabs.items():
            fw.write(word+"\n")
    return datas, labels

# 读取词典 生成词-索引对应关系, 其中special_words = ['<PAD>', '<UNK>']
def read_word_vocabs(save_vocab_path, special_words):
    with open(save_vocab_path, "r", encoding="utf8") as fo:
        word_vocabs = [word.strip() for word in fo]
    word_vocabs = special_words + word_vocabs
    idx2vocab = {idx: char for idx, char in enumerate(word_vocabs)}
    vocab2idx = {char: idx for idx, char in idx2vocab.items()}
    return idx2vocab, vocab2idx

# 把预处理过的数据索引化 即变成词编号序列
def process_dataset(datas, labels, category2idx, vocab2idx):
    new_datas, new_labels = [], []
    for data, label in zip(datas, labels):
        index_data = [vocab2idx[word] if word in vocab2idx else vocab2idx['<UNK>'] for word in data]
        index_label = category2idx[label]
        new_datas.append(index_data)
        new_labels.append(index_label)
    return new_datas, new_labels
data_path = './data/toutiao_news_dataset.txt'

save_vocab_path = './data/w2v_word_vocabs.txt'
special_words = ['<PAD>', '<UNK>']
category_lists = ["民生故事","文化","娱乐","体育","财经","房产","汽车","教育","科技","军事",
                "旅游","国际","证券股票","农业","电竞游戏"]

category2idx = {cate: idx for idx, cate in enumerate(category_lists)}
idx2category = {idx: cate for idx, cate in enumerate(category_lists)}

datas, labels = read_toutiao_dataset(data_path, save_vocab_path)
idx2vocab, vocab2idx = read_word_vocabs(save_vocab_path, special_words)
all_datas, all_labels = process_dataset(datas, labels, category2idx, vocab2idx)
vocab2idx.pop('<PAD>')
vocab2idx.pop('<UNK>')
if '' in vocab2idx:
    vocab2idx.pop('')
if '---' in vocab2idx:
    vocab2idx.pop('---')
if '----' in vocab2idx:
    vocab2idx.pop('----')
if '-----' in vocab2idx:
    vocab2idx.pop('-----')
if '------' in vocab2idx:
    vocab2idx.pop('------')
import numpy as np

def vocab2vec(model, vocab2idx):
#     with open(save_vocab_path, "r", encoding="utf8") as fo:
#         word_vocabs = [word.strip() for word in fo]
    special_words = ['<PAD>', '<UNK>']
    
    w2vec = {word: model[word] for word in vocab2idx.keys()}

    n_vocabs = len(vocab2idx) + 10
    embedding_weights = np.zeros((n_vocabs, 100))
    for w, index in vocab2idx.items():  # 从索引为1的词语开始，用词向量填充矩阵
        embedding_weights[index, :] = w2vec[w]
    return embedding_weights
embedding_weights = vocab2vec(model, vocab2idx)


import numpy
import keras
from keras import backend as K
from keras import activations
from keras.engine.topology import Layer
from keras.preprocessing import sequence
from keras.models import Sequential
from keras.models import Model
from keras.layers import Input, Dense, Embedding, LSTM, Bidirectional,Dropout
from keras.models import load_model
import os
K.clear_session()

class AttentionLayer(Layer):
    def __init__(self, attention_size=None, **kwargs):
        self.attention_size = attention_size
        super(AttentionLayer, self).__init__(**kwargs)
        
    def get_config(self):
        config = super().get_config()
        config['attention_size'] = self.attention_size
        return config
        
    def build(self, input_shape):
        assert len(input_shape) == 3
        
        self.time_steps = input_shape[1]
        hidden_size = input_shape[2]
        if self.attention_size is None:
            self.attention_size = hidden_size
            
        self.W = self.add_weight(name='att_weight', shape=(hidden_size, self.attention_size),
                                initializer='uniform', trainable=True)
        self.b = self.add_weight(name='att_bias', shape=(self.attention_size,),
                                initializer='uniform', trainable=True)
        self.V = self.add_weight(name='att_var', shape=(self.attention_size,),
                                initializer='uniform', trainable=True)
        super(AttentionLayer, self).build(input_shape)
    
    def call(self, inputs):
        self.V = K.reshape(self.V, (-1, 1))
        H = K.tanh(K.dot(inputs, self.W) + self.b)
        score = K.softmax(K.dot(H, self.V), axis=1)
        outputs = K.sum(score * inputs, axis=1)
        return outputs
    
    def compute_output_shape(self, input_shape):
        return input_shape[0], input_shape[2]
    
    
def create_classify_model(max_len, vocab_size, embedding_size, hidden_size, attention_size, class_nums, embedding_weights):
    inputs = Input(shape=(max_len,), dtype='int32')
    x = Embedding(vocab_size, embedding_size, weights=[embedding_weights])(inputs)
    x = Bidirectional(LSTM(hidden_size, dropout=0.2, return_sequences=True))(x)
    x = Dropout(0.5)(x)
    x = AttentionLayer(attention_size=attention_size)(x)
    outputs = Dense(class_nums, activation='softmax')(x)
    model = Model(inputs=inputs, outputs=outputs)
    model.summary() # 输出模型结构和参数数量
    return model

MAX_LEN = 30
VOCAB_SIZE = len(vocab2idx)+10
EMBEDDING_SIZE = 100
HIDDEN_SIZE = 64
ATT_SIZE = 50
CLASS_NUMS = len(category2idx)
BATCH_SIZE = 64
EPOCHS = 20

all_datas_mem = all_datas
all_labels_mem = all_labels

# all_datas = all_datas_mem
# all_labels = all_labels_mem

count = len(all_labels)
rate1, rate2 = 0.8, 0.9 # train-0.8, test-0.1, dev-0.1
# padding the data
new_datas = sequence.pad_sequences(all_datas, maxlen=MAX_LEN)
new_labels = keras.utils.to_categorical(all_labels, CLASS_NUMS)
# split all data to train, test and dev
x_train, y_train = new_datas[:int(count*rate1)], new_labels[:int(count*rate1)]
x_test, y_test = new_datas[int(count*rate1):int(count*rate2)], new_labels[int(count*rate1):int(count*rate2)]
x_dev, y_dev = new_datas[int(count*rate2):], new_labels[int(count*rate2):]

print(len(all_datas))
print(new_datas[0])
print(x_train[30])
print(new_labels[0])
print(new_labels[20])

import numpy as np
from sklearn.metrics import f1_score, recall_score, precision_score
from sklearn.model_selection import train_test_split
class Metrics(tf.keras.callbacks.Callback):
    def __init__(self, valid_data):
        super(Metrics, self).__init__()
        self.validation_data = valid_data

    def on_epoch_end(self, epoch, logs=None):
        logs = logs or {}
        val_predict = np.argmax(self.model.predict(self.validation_data[0]), -1)
        val_targ = self.validation_data[1]
        if len(val_targ.shape) == 2 and val_targ.shape[1] != 1:
            val_targ = np.argmax(val_targ, -1)

        _val_f1 = f1_score(val_targ, val_predict, average='macro')
        _val_recall = recall_score(val_targ, val_predict, average='macro')
        _val_precision = precision_score(val_targ, val_predict, average='macro')

        logs['val_f1'] = _val_f1
        logs['val_recall'] = _val_recall
        logs['val_precision'] = _val_precision
        print(" — val_f1: %f — val_precision: %f — val_recall: %f" % (_val_f1, _val_precision, _val_recall))
        return
# create model
from keras.callbacks import TensorBoard
tbCallBack = TensorBoard(log_dir='./model/logs',  # log 目录
                 histogram_freq=0,  # 按照何等频率（epoch）来计算直方图，0为不计算
#                  batch_size=32,     # 用多大量的数据计算直方图
                 write_graph=True,  # 是否存储网络结构图
                 write_grads=True, # 是否可视化梯度直方图
                 write_images=True,# 是否可视化参数
                 embeddings_freq=0, 
                 embeddings_layer_names=None, 
                 embeddings_metadata=None)
model_path = "./model/w2v_news_classify_model.h5"
# ATT_SIZE = 50
if os.path.exists('./model/w2v_news_classify_model.h5'):
    print('加载模型...')
    model = load_model(model_path, custom_objects={'AttentionLayer': AttentionLayer(ATT_SIZE)}, compile=False)
else :
    model = create_classify_model(MAX_LEN, VOCAB_SIZE, EMBEDDING_SIZE, HIDDEN_SIZE, ATT_SIZE, CLASS_NUMS,embedding_weights)
# model = create_classify_model(MAX_LEN, VOCAB_SIZE, EMBEDDING_SIZE, HIDDEN_SIZE, ATT_SIZE, CLASS_NUMS,embedding_weights)
# loss and optimizer
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
# train model
model.fit(x_train, y_train, batch_size=BATCH_SIZE, epochs=10, validation_data=(x_test, y_test),callbacks=[tbCallBack,Metrics(valid_data=(x_test, y_test))])
# test model 
model.save("./model/w2v_news_classify_model.h5")
score, acc = model.evaluate(x_dev, y_dev, batch_size=2)
# model.optimizer.lr
print('Test score:', score)
print('Test accuracy:', acc)
import tensorflow.keras.backend as K
print(K.get_value(model.optimizer.lr))<jupyter_output><empty_output><jupyter_text>模型预测加载模型，预测，输出结果<jupyter_code>import jieba
from tqdm import tqdm

# 读取原始数据集分词预处理 并保存词典
def read_toutiao_dataset(data_path, save_vocab_path):
    with open(data_path, "r", encoding="utf8") as fo:
        all_lines = fo.readlines()
    datas, labels = [], []
    word_vocabs = {}
    for line in tqdm(all_lines):
        content_words = []
        category, content = line.strip().split("_!_")
        for term in jieba.lcut(content):
            if term not in stopwords and term not in word_vocabs and term != ' ' and term != '':
                word_vocabs[term] = len(word_vocabs)+1
            content_words.append(term)
        datas.append(content_words)
        labels.append(category)
    with open(save_vocab_path, "w", encoding="utf8") as fw:
        for word, index in word_vocabs.items():
            fw.write(word+"\n")
    return datas, labels

# 读取词典 生成词-索引对应关系, 其中special_words = ['<PAD>', '<UNK>']
def read_word_vocabs(save_vocab_path, special_words):
    with open(save_vocab_path, "r", encoding="utf8") as fo:
        word_vocabs = [word.strip() for word in fo]
    word_vocabs = special_words + word_vocabs
    idx2vocab = {idx: char for idx, char in enumerate(word_vocabs)}
    vocab2idx = {char: idx for idx, char in idx2vocab.items()}
    return idx2vocab, vocab2idx

# 把预处理过的数据索引化 即变成词编号序列
def process_dataset(datas, labels, category2idx, vocab2idx):
    new_datas, new_labels = [], []
    for data, label in zip(datas, labels):
        index_data = [vocab2idx[word] if word in vocab2idx else vocab2idx['<UNK>'] for word in data]
        index_label = category2idx[label]
        new_datas.append(index_data)
        new_labels.append(index_label)
    return new_datas, new_labels
!pip list
import numpy
import keras
from keras import backend as K
from keras import activations
from keras.engine.topology import Layer
from keras.preprocessing import sequence
from keras.models import Sequential
from keras.models import Model
from keras.layers import Input, Dense, Embedding, LSTM, Bidirectional,Dropout
from keras.models import load_model
import os
K.clear_session()

class AttentionLayer(Layer):
    def __init__(self, attention_size=None, **kwargs):
        self.attention_size = attention_size
        super(AttentionLayer, self).__init__(**kwargs)
        
    def get_config(self):
        config = super().get_config()
        config['attention_size'] = self.attention_size
        return config
        
    def build(self, input_shape):
        assert len(input_shape) == 3
        
        self.time_steps = input_shape[1]
        hidden_size = input_shape[2]
        if self.attention_size is None:
            self.attention_size = hidden_size
            
        self.W = self.add_weight(name='att_weight', shape=(hidden_size, self.attention_size),
                                initializer='uniform', trainable=True)
        self.b = self.add_weight(name='att_bias', shape=(self.attention_size,),
                                initializer='uniform', trainable=True)
        self.V = self.add_weight(name='att_var', shape=(self.attention_size,),
                                initializer='uniform', trainable=True)
        super(AttentionLayer, self).build(input_shape)
    
    def call(self, inputs):
        self.V = K.reshape(self.V, (-1, 1))
        H = K.tanh(K.dot(inputs, self.W) + self.b)
        score = K.softmax(K.dot(H, self.V), axis=1)
        outputs = K.sum(score * inputs, axis=1)
        return outputs
    
    def compute_output_shape(self, input_shape):
        return input_shape[0], input_shape[2]
    
    
def create_classify_model(max_len, vocab_size, embedding_size, hidden_size, attention_size, class_nums, embedding_weights):
    inputs = Input(shape=(max_len,), dtype='int32')
    x = Embedding(vocab_size, embedding_size, weights=[embedding_weights])(inputs)
    x = Bidirectional(LSTM(hidden_size, dropout=0.2, return_sequences=True))(x)
    x = Dropout(0.5)(x)
    x = AttentionLayer(attention_size=attention_size)(x)
    outputs = Dense(class_nums, activation='softmax')(x)
    model = Model(inputs=inputs, outputs=outputs)
    model.summary() # 输出模型结构和参数数量
    return model


from keras.models import load_model
import numpy as np
import jieba
np.set_printoptions(suppress=True)

save_vocab_path = "./data/word_vocabs.txt"
model_path = "./model/w2v_news_classify_model.h5"
special_words = ['<PAD>', '<UNK>']
category_lists = ["民生故事","文化","娱乐","体育","财经","房产","汽车","教育","科技","军事",
                "旅游","国际","证券股票","农业","电竞游戏"]
maxlen = 30
ATT_SIZE = 50

category2idx = {cate: idx for idx, cate in enumerate(category_lists)}
idx2category = {idx: cate for idx, cate in enumerate(category_lists)}
idx2vocab, vocab2idx = read_word_vocabs(save_vocab_path, special_words)

content = "陕西省制定贫困退出指标"
content_words = [term for term in jieba.lcut(content)]
sent2id = [vocab2idx[word] if word in vocab2idx else 1 for word in content_words]
sent2id_new = np.array([sent2id[:maxlen] + [0] * (maxlen-len(sent2id))])

model = load_model(model_path, custom_objects={'AttentionLayer': AttentionLayer(ATT_SIZE)}, compile=False)
y_pred = model.predict(sent2id_new)
print(y_pred)
result = {}
for idx, pred in enumerate(y_pred[0]):
    result[idx2category[idx]] = pred
result_sorted = sorted(result.items(), key=lambda item: item[1], reverse=True)
print(result_sorted)
# y_label = np.argmax(y_pred[0])
# print(y_label, idx2category[y_label])

# category_lists = ["民生故事","文化","娱乐","体育","财经","房产","汽车","教育","科技","军事",
#                 "旅游","国际","证券股票","农业","电竞游戏"]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12419393360401242, 'fraction_numerical': 0.013673274420826368, 'mean_word_length': 3.7799657534246576, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '309078', 'license': 'MIT', 'n_tokens_mistral': 6191, 'n_tokens_neox': 6190, 'n_words': 1165, 'path': 'code/Word2Vec+bilstm.ipynb', 'repo_name': 'li-donglei/TextClassification', 'size': '16748'}