<jupyter_start><jupyter_text>导库<jupyter_code>import torch
import torch.optim as optim
import torch.nn as nn
import torch.nn.functional as F

import numpy as np
import pandas as pd
import json
import random

from utils import extend_maps, prepocess_data_for_lstmcrf, get_device, seed_everything
from models.util import tensorized, sort_by_lengths, cal_loss, cal_lstm_crf_loss
from models.bilstm_crf import BiLSTM_CRF

from ModelHandler import ModelHandler
device_all, n_gpu=get_device()

# 使用第一块 gpu
device = device_all
print(device, n_gpu)

device = "cpu"
# 设置随机种子
seed_everything()
random.randint(1,10)<jupyter_output><empty_output><jupyter_text>数据预处理<jupyter_code>df_train_data = pd.read_csv("./ResumeNER/df_train_data.csv", index_col=0)
df_dev_data = pd.read_csv("./ResumeNER/df_dev_data.csv", index_col=0)
df_test_data = pd.read_csv("./ResumeNER/df_test_data.csv", index_col=0)

print(df_train_data.shape, df_dev_data.shape, df_dev_data.shape)
df_train_data.head()
train_word_lists = [eval(i) for i in df_train_data['words'].tolist()]
train_tag_lists = [eval(i) for i in df_train_data['tags'].tolist()]

dev_word_lists = [eval(i) for i in df_dev_data['words'].tolist()]
dev_tag_lists = [eval(i) for i in df_dev_data['tags'].tolist()]

test_word_lists = [eval(i) for i in df_test_data['words'].tolist()]
test_tag_lists = [eval(i) for i in df_test_data['tags'].tolist()]

with open('./ResumeNER/word2id.json', 'r') as f:
    word2id = json.load(f)
with open('./ResumeNER/tag2id.json', 'r') as f:
    tag2id = json.load(f)
print(len(word2id), len(tag2id))
    
# 增加 unk，pad，<start>，<end>等字段
word2id, tag2id = extend_maps(word2id, tag2id, for_crf=True)

# 给每个句子添加 <end> 字段，测试数据不用加
train_word_lists, train_tag_lists = prepocess_data_for_lstmcrf(train_word_lists, train_tag_lists)
dev_word_lists, dev_tag_lists = prepocess_data_for_lstmcrf(dev_word_lists, dev_tag_lists)
test_word_lists, test_tag_lists = prepocess_data_for_lstmcrf(test_word_lists, test_tag_lists, test=True)

# 给句子进行排序，按照句子长度从小到大排序
train_word_lists, train_tag_lists, _ = sort_by_lengths(train_word_lists, train_tag_lists)
dev_word_lists, dev_tag_lists, _ = sort_by_lengths(dev_word_lists, dev_tag_lists)
test_word_lists, test_tag_lists, indices = sort_by_lengths(test_word_lists, test_tag_lists)<jupyter_output>1902 28<jupyter_text>模型单独测试<jupyter_code>print(len(train_word_lists))
batch_sents = train_word_lists[:1]
batch_tags = train_tag_lists[:1]

emb_size, hidden_size = 128, 128
vocab_size = len(word2id)
out_size = len(tag2id)
print("out_size", out_size)
model = BiLSTM_CRF(vocab_size, emb_size, hidden_size, out_size).to(device)

model.train()
# 准备数据
tensorized_sents, lengths = tensorized(batch_sents, word2id)
print(tensorized_sents.shape)
tensorized_sents = tensorized_sents.to(device)

targets, lengths = tensorized(batch_tags, tag2id)
print(targets.shape)
targets = targets.to(device)

# forward
scores = model(tensorized_sents, lengths).to(device)
print(scores.shape)
# 计算损失 更新参数
# self.optimizer.zero_grad()
loss = cal_lstm_crf_loss(scores, targets, tag2id).to(device)
print(loss.item())
# loss.backward()
# self.optimizer.step()<jupyter_output>3821
out_size 32
torch.Size([1, 179])
torch.Size([1, 179])
torch.Size([1, 179, 32, 32])
609.7691650390625<jupyter_text>bilstm_crf 模型<jupyter_code>emb_size, hidden_size = 128, 128
vocab_size = len(word2id)
out_size = len(tag2id)
print("out_size", out_size)

model = BiLSTM_CRF(vocab_size, emb_size, hidden_size, out_size)<jupyter_output>out_size 32<jupyter_text>模型训练<jupyter_code># %%time
# seed_everything()

batch_size = 2**5

modelHandlerParams = {}
modelHandlerParams['epoch_nums'] = 1000000
modelHandlerParams['batch_size'] = batch_size
modelHandlerParams['device'] = device
modelHandlerParams['model'] = model
modelHandlerParams['word2id'] = word2id
modelHandlerParams['tag2id'] = tag2id
modelHandler = ModelHandler(modelHandlerParams)

model.to(device)

model_path = "./models/blstm_crf.pickle"

# crf 损失函数
loss_fn = cal_lstm_crf_loss
# 调参地方，分别调整为0.1,0.01,0.001，最优为0.01
optimizer = optim.Adam(model.parameters(), lr=0.01,
                       weight_decay=0.0001) # lr sets the learning rate of the optimizer

modelHandler.fit(X_train=train_word_lists, y_train=train_tag_lists, loss_fn=loss_fn, model_path = model_path,
                 optimizer=optimizer, eval_set = (dev_word_lists, dev_tag_lists), early_stopping_rounds=10, verbose=2)<jupyter_output>epoch: 0
train loss: 23.516282858968783
*****Checking accuracy on validation set*****
batchNum: 14
val_loss: 5.601794106619699
epoch: 1
train loss: 3.8028446165453484
*****Checking accuracy on validation set*****
batchNum: 14
val_loss: 3.5273020608084544
epoch: 2
train loss: 1.9461650367544479
*****Checking accuracy on validation set*****
batchNum: 14
val_loss: 3.2519258771623885
epoch: 3
train loss: 1.21913233524611
*****Checking accuracy on validation set*****
batchNum: 14
val_loss: 2.889012472970145
epoch: 4
train loss: 0.799991511497177
*****Checking accuracy on validation set*****
batchNum: 14
val_loss: 2.972699301583426
epoch: 5
train loss: 0.5698378747250853
*****Checking accuracy on validation set*****
batchNum: 14
val_loss: 2.9491909572056363
epoch: 6
train loss: 0.4239474705287388
*****Checking accuracy on validation set*****
batchNum: 14
val_loss: 2.8080313546316966
epoch: 7
train loss: 0.2543248729545529
*****Checking accuracy on validation set*****
batchNum: 14
val_loss: 3[...]<jupyter_text>模型测试<jupyter_code>pred_tag_lists, test_tag_lists = modelHandler.test(test_word_lists, test_tag_lists, indices)

from Metrics import Metrics
metrics = Metrics(test_tag_lists, pred_tag_lists)
metrics.report_scores()
metrics.report_confusion_matrix()<jupyter_output>precision    recall  f1-score   support
   B-RACE     1.0000    0.9286    0.9630        14
  B-TITLE     0.9429    0.9404    0.9416       772
    B-ORG     0.9521    0.9711    0.9615       553
    E-EDU     0.9911    0.9911    0.9911       112
    M-ORG     0.9444    0.9660    0.9551      4325
    B-LOC     1.0000    0.8333    0.9091         6
    M-PRO     0.8472    0.8971    0.8714        68
        O     0.9727    0.9697    0.9712      5190
   M-CONT     1.0000    1.0000    1.0000        53
   B-CONT     1.0000    1.0000    1.0000        28
   E-NAME     1.0000    1.0000    1.0000       112
    B-PRO     0.8788    0.8788    0.8788        33
    M-LOC     1.0000    0.8095    0.8947        21
   M-NAME     1.0000    0.9878    0.9939        82
   E-CONT     1.0000    1.0000    1.0000        28
  M-TITLE     0.9563    0.9100    0.9326      1922
    E-ORG     0.9075    0.9222    0.9148       553
    E-PRO     0.9412    0.9697    0.9552        33
    E-LOC     1.0000    0.8333 [...]
{'fraction_non_alphanumeric': 0.11362954067847493, 'fraction_numerical': 0.10807565295706995, 'mean_word_length': 4.506611570247934, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '547128', 'license': 'Apache-2.0', 'n_tokens_mistral': 3076, 'n_tokens_neox': 2711, 'n_words': 510, 'path': 'train.ipynb', 'repo_name': 'ddz-mark/pytorch_ner', 'size': '6662'}