<jupyter_start><jupyter_text>本地测试集评估效果分析----<jupyter_code>import sys, os
from pprint import pprint
sys.path.insert(0, "/home/team55/notespace/zengbin")
from jddc.config import BaseConfig
from jddc.utils import bleu, read_file
conf = BaseConfig()
path_answers = "/home/team55/notespace/zengbin/answers"

import jddc.datasets as d
def one_match_ten(file_a):
    a = read_file(file_a)
    ref = [x for x in read_file(conf.file_test_a) if x != ""]
    for i, a_i in enumerate(a):
        print(i,"", "-"* 68)
        print("输出答案：", a_i)
        print("参考答案：")
        pprint(ref[i*10:(i+1)*10])
answers = d.read_test_answers(conf.file_test_a)
for i in range(int(len(answers)/10)):
    print(i, "-ref","="*88)
    pprint(answers[i*10:(i+1)*10])<jupyter_output><empty_output><jupyter_text>n-grams + BM25检索--- 模型简介* 输入：questions、answers一一对应的问答对* 参数：n，top* 过程：1）提取n-gram特征；2）创建bm25模型* 输出：bm25模型 测试结论1. 预测是根据n-gram特征长度进行预筛选不是一个好的想法，它使得得分降低2. top5得到的预测结果评分高于top103. 数据量的增大并不能有效提升得分4. tri-grams特征效果(本地测试集得分0.19)明显好于jieba分词（0.13）<jupyter_code>file_bm25_001 = os.path.join(path_answers, "bm25_answers001.txt")
file_bm25_003 = os.path.join(path_answers, "bm25_answers003.txt")
file_bm25_004 = os.path.join(path_answers, "bm25_answers004.txt")

# <BM25Config | model:/home/team55/notespace/data/BM25/bm25.model.pkl; n:2; top:5; bucket:None; random_num:1234>
file_bm25_005 = os.path.join(path_answers, "bm25_answers005.txt")

# <BM25Config | model:/home/team55/notespace/data/BM25/bm25_3.model.pkl; n:3; top:5; bucket:None; random_num:1234>
file_bm25_006 = os.path.join(path_answers, "bm25_answers006.txt")

# <BM25Config | model:/home/team55/notespace/data/BM25/bm25_3.model.small.pkl; n:3; top:5; random_num:1234>
file_bm25_007 = os.path.join(path_answers, "bm25_answers007.txt")

# <BM25Config | model:/home/team55/notespace/data/BM25/bm25_3.model.middle.pkl; n:3; top:5; random_num:1234>
file_bm25_008 = os.path.join(path_answers, "bm25_answers008.txt")

bleu(file_bm25_008, conf.file_test_a)
one_match_ten(file_bm25_008)<jupyter_output><empty_output><jupyter_text>Tfidf + Cosine--- 测试结论1. 单纯的增加QA对的数量，无法使得tfidf效果更好2. top5选最长的效果比top10选最长更好一些3. QQ + A 模式的数据集效果更好<jupyter_code># 数据集：1000个session中抽取的约3000个QA对；分词：3-gram；top10选最长
file_tfidf_001 =  os.path.join(path_answers, "tfidf_answers001.txt")

# 数据集：create_dataset03；分词：3-gram；top10选最长
file_tfidf_002 =  os.path.join(path_answers, "tfidf_answers002.txt")

# 数据集：create_dataset03；分词：3-gram；top5选最长
file_tfidf_003 =  os.path.join(path_answers, "tfidf_answers003.txt")

# <TfidfConfig | model:/home/team55/notespace/data/tfidf/tfidf_ir.model.middle.pkl; n:3; top:5
file_tfidf_004 =  os.path.join(path_answers, "tfidf_answers004.txt")

# 数据集：create_datasets05，n=30000
# <TfidfConfig | model:/home/team55/notespace/data/tfidf/tfidf_ir.model.middle.pkl; n:3; top:5
file_tfidf_010 =  os.path.join(path_answers, "tfidf_answers010.txt")

# 数据集：create_datasets05，n=30000
# <TfidfConfig | model:/home/team55/notespace/data/tfidf/tfidf_ir.model.middle.pkl; n:3; top:10
file_tfidf_011 =  os.path.join(path_answers, "tfidf_answers011.txt")

bleu(file_tfidf_011, conf.file_test_a)
one_match_ten(file_tfidf_002)<jupyter_output><empty_output><jupyter_text>DQN + TFIDF---- 测试结论* DQN在<jupyter_code>file_dt_001 = os.path.join(path_answers, "dqn_tfidf_answers001.txt")

bleu(file_dt_001, conf.file_test_a)<jupyter_output><empty_output><jupyter_text>Seq2seq---测试结论：1. beam size设置为6，删除前两个候选，得分更高<jupyter_code>file_ss_001 = os.path.join(path_answers, "seq2seq_answers001.txt")

file_ss_003 = os.path.join(path_answers, "seq2seq_answers003.txt")
file_ss_004 = os.path.join(path_answers, "seq2seq_answers004.txt")
bleu(file_ss_004, conf.file_test_a)
bleu("/submitwork/answers03.txt", conf.file_test_a)<jupyter_output><empty_output><jupyter_text>Transformer---测试结论：1. 不断的训练只会导致过拟合，当eval loss开始持续走高，停止训练2. batch_size大小对最终结果没有影响 预测参数优化---* beam_size: 6<jupyter_code>file_tf_001 = os.path.join(path_answers, "transformer_answers001.txt")

# step: 221000
file_tf_002 = os.path.join(path_answers, "transformer_answers002.txt")

# step: 238000; beam_size: 4 - score:0.500588
file_tf_003 = os.path.join(path_answers, "transformer_answers003.txt")

# step: 238000; beam_size: 6 - score:0.513192
file_tf_004 = os.path.join(path_answers, "transformer_answers004.txt")

# step: 238000; beam_size: 10 - score:0.457139
file_tf_005 = os.path.join(path_answers, "transformer_answers005.txt")

# step: 250000; beam_size: 6 - score:0.435266
file_tf_006 = os.path.join(path_answers, "transformer_answers006.txt")

# step: 249000; beam_size: 6 - score:0.478948
file_tf_007 = os.path.join(path_answers, "transformer_answers007.txt")

# step: 248000; beam_size: 6 - score:0.490112
file_tf_008 = os.path.join(path_answers, "transformer_answers008.txt")

# step: 247000; beam_size: 6 - score:0.47816
file_tf_009 = os.path.join(path_answers, "transformer_answers009.txt")

# step: 246000; beam_size: 6 - score:0.482311
file_tf_010 = os.path.join(path_answers, "transformer_answers010.txt")

# step: 240000; beam_size: 6 - score:0.490637
file_tf_011 = os.path.join(path_answers, "transformer_answers011.txt")

# step: 231000; beam_size: 6 - score:0.475969
file_tf_012 = os.path.join(path_answers, "transformer_answers012.txt")

# step: 235000; beam_size: 6 - score:0.494737
file_tf_013 = os.path.join(path_answers, "transformer_answers013.txt")

# step: 238000; beam_size: 6; batch_size:16 - score:0.513192
file_tf_014 = os.path.join(path_answers, "transformer_answers014.txt")

# step: 238000; beam_size: 6; batch_size:8 - score:0.513192
file_tf_015 = os.path.join(path_answers, "transformer_answers015.txt")


bleu(file_tf_015, conf.file_test_a)
one_match_ten(file_tf_001)<jupyter_output><empty_output><jupyter_text>Transformer-Big---<jupyter_code># step: 19000; beam_size: 6 - score:0.552725
file_tfb_001 = os.path.join(path_answers, "transformer_big_answers001.txt")

# step: 59000; beam_size: 6 - score:0.510235
file_tfb_002 = os.path.join(path_answers, "transformer_big_answers002.txt")

# step: 50000; beam_size: 6 - score:0.50707
file_tfb_003 = os.path.join(path_answers, "transformer_big_answers003.txt")

# step: 59000; beam_size: 4 - score:0.530586
file_tfb_004 = os.path.join(path_answers, "transformer_big_answers004.txt")

# step: 93000; beam_size: 4 - score:0.532848
file_tfb_005 = os.path.join(path_answers, "transformer_big_answers005.txt")

# step: 93000; beam_size: 6 - score:0.507243
file_tfb_006 = os.path.join(path_answers, "transformer_big_answers006.txt")

# step: avg（20）-93000; beam_size: 4 - score:0.52554
file_tfb_007 = os.path.join(path_answers, "transformer_big_answers007.txt")

# step: avg（5）-93000; beam_size: 4 - score:0.575925
file_tfb_008 = os.path.join(path_answers, "transformer_big_answers008_1.txt")

# step: avg（5）-93000; beam_size: 6 - score:0.532789
file_tfb_009 = os.path.join(path_answers, "transformer_big_answers009.txt")

# step: avg（5）-137000; beam_size: 4 - score:0.525987
file_tfb_010 = os.path.join(path_answers, "transformer_big_answers010.txt")

# step: avg（5）-137000; beam_size: 6 - score:0.521963
file_tfb_011 = os.path.join(path_answers, "transformer_big_answers011.txt")

# step:137000; beam_size: 4 - score:0.550732
file_tfb_012 = os.path.join(path_answers, "transformer_big_answers012.txt")

# step:avg（20）-137000; beam_size: 4 - score:0.55631
file_tfb_013 = os.path.join(path_answers, "transformer_big_answers013.txt")

# step:avg（20）-137000; beam_size: 4; alpha=0.8 - score:0.532453
file_tfb_014 = os.path.join(path_answers, "transformer_big_answers014.txt")

# step:avg（20）-142000; beam_size: 4; alpha=0.6 - score:0.537915
file_tfb_015 = os.path.join(path_answers, "transformer_big_answers015.txt")

# step:avg（5）-142000; beam_size: 4; alpha=0.6 - score:0.505074
file_tfb_016 = os.path.join(path_answers, "transformer_big_answers016.txt")

# step:142000; beam_size: 4; alpha=0.6 - score:0.537309
file_tfb_017 = os.path.join(path_answers, "transformer_big_answers017.txt")

# step:123000; beam_size: 4; alpha=0.6 - score:0.560559
file_tfb_018 = os.path.join(path_answers, "transformer_big_answers018.txt")

# step:124000; beam_size: 4; alpha=0.6 - score:0.56745
file_tfb_019 = os.path.join(path_answers, "transformer_big_answers019.txt")

# step:125000; beam_size: 4; alpha=0.6 - score:0.494822
file_tfb_020 = os.path.join(path_answers, "transformer_big_answers020.txt")

# step:126000; beam_size: 4; alpha=0.6 - score:0.537438
file_tfb_021 = os.path.join(path_answers, "transformer_big_answers021.txt")

# step:127000; beam_size: 4; alpha=0.6 - score:0.543757
file_tfb_022 = os.path.join(path_answers, "transformer_big_answers022.txt")

# step:128000; beam_size: 4; alpha=0.6 - score:0.535475
file_tfb_023 = os.path.join(path_answers, "transformer_big_answers023.txt")

# step:avg（5）-127000; beam_size: 4; alpha=0.6 - score:0.51102
file_tfb_024 = os.path.join(path_answers, "transformer_big_answers024.txt")

# step:avg（5）-128000; beam_size: 4; alpha=0.6 - score:0.545317
file_tfb_025 = os.path.join(path_answers, "transformer_big_answers025.txt")

# step:avg（5）-129000; beam_size: 4; alpha=0.6 - score:0.538883
file_tfb_026 = os.path.join(path_answers, "transformer_big_answers026.txt")

# step:139000; beam_size: 4; alpha=0.6 - score:0.527343
file_tfb_027 = os.path.join(path_answers, "transformer_big_answers027.txt")

# step:avg（2）-124000; beam_size: 4; alpha=0.6 - score:0.533959
file_tfb_028 = os.path.join(path_answers, "transformer_big_answers028.txt")

bleu(file_tfb_008, conf.file_test_a)
bleu("/home/team55/notespace/data/answers_temp_0918.txt", conf.file_test_a)<jupyter_output><empty_output><jupyter_text>Transformer-Big-Rq---<jupyter_code># step:46000; beam_size:4; - score:0.497161
file_tfbr_001 = os.path.join(path_answers, "transformer_big_rq_answers001.txt")

# step:45000; beam_size:4; - score:0.541437
file_tfbr_002 = os.path.join(path_answers, "transformer_big_rq_answers002.txt")

# step:44000; beam_size:4; - score:0.50136
file_tfbr_003 = os.path.join(path_answers, "transformer_big_rq_answers003.txt")

# step:43000; beam_size:4; - score:0.526366
file_tfbr_004 = os.path.join(path_answers, "transformer_big_rq_answers004.txt")

# step:42000; beam_size:4; - score:0.577759
file_tfbr_005 = os.path.join(path_answers, "transformer_big_rq_answers005.txt")

# step:41000; beam_size:4; - score:0.547324
file_tfbr_006 = os.path.join(path_answers, "transformer_big_rq_answers006.txt")

# step:40000; beam_size:4; - score:0.514353
file_tfbr_007 = os.path.join(path_answers, "transformer_big_rq_answers007.txt")

# step:39000; beam_size:4; - score:0.560943
file_tfbr_008 = os.path.join(path_answers, "transformer_big_rq_answers008.txt")

# step:38000; beam_size:4; - score:0.511699
file_tfbr_009 = os.path.join(path_answers, "transformer_big_rq_answers009.txt")

# step:37000; beam_size:4; - score:0.542784
file_tfbr_010 = os.path.join(path_answers, "transformer_big_rq_answers010.txt")

# step:36000; beam_size:4; - score:0.521156
file_tfbr_011 = os.path.join(path_answers, "transformer_big_rq_answers011.txt")

# step:35000; beam_size:4; - score:0.47994
file_tfbr_012 = os.path.join(path_answers, "transformer_big_rq_answers012.txt")

# step:34000; beam_size:4; - score:0.548106
file_tfbr_013 = os.path.join(path_answers, "transformer_big_rq_answers013.txt")

# step:33000; beam_size:4; - score:0.459477
file_tfbr_014 = os.path.join(path_answers, "transformer_big_rq_answers014.txt")

# step:32000; beam_size:4; - score:0.495406
file_tfbr_015 = os.path.join(path_answers, "transformer_big_rq_answers015.txt")

# step:31000; beam_size:4; - score:0.477652
file_tfbr_016 = os.path.join(path_answers, "transformer_big_rq_answers016.txt")

# step:30000; beam_size:4; - score:0.452962
file_tfbr_017 = os.path.join(path_answers, "transformer_big_rq_answers017.txt")

# step:29000; beam_size:4; - score:0.586272
file_tfbr_018 = os.path.join(path_answers, "transformer_big_rq_answers018.txt")

# step:28000; beam_size:4; - score:0.465243
file_tfbr_019 = os.path.join(path_answers, "transformer_big_rq_answers019.txt")

# step:27000; beam_size:4; - score:0.501558
file_tfbr_020 = os.path.join(path_answers, "transformer_big_rq_answers020.txt")

# step:29000; beam_size:6; - score:0.559624
file_tfbr_018_1 = os.path.join(path_answers, "transformer_big_rq_answers018_1.txt")

bleu(file_tfbr_018_1, conf.file_test_a)
# step:avg（20）-46000; beam_size:4; - score:0.497524
file_tfbr_021 = os.path.join(path_answers, "transformer_big_rq_answers021.txt")

# step:avg（20）-46000; beam_size:6; - score:0.513055
file_tfbr_022 = os.path.join(path_answers, "transformer_big_rq_answers022.txt")

# step:50000; beam_size:4 - score:0.465903
file_tfbr_023 = os.path.join(path_answers, "transformer_big_rq_answers023.txt")

# step:49000; beam_size:4; - score:0.489081
file_tfbr_024 = os.path.join(path_answers, "transformer_big_rq_answers024.txt")

# step:48000; beam_size:4; - score:0.482493
file_tfbr_025 = os.path.join(path_answers, "transformer_big_rq_answers025.txt")

# step:47000; beam_size:4; - score:0.486796
file_tfbr_026 = os.path.join(path_answers, "transformer_big_rq_answers026.txt")

# step:48000; beam_size:6; - score:0.415668
# answer选择方案： top5选最长
file_tfbr_027 = os.path.join(path_answers, "transformer_big_rq_answers027.txt")

# step:avg（11）-45000; beam_size:4; - score:0.515042
file_tfbr_028 = os.path.join(path_answers, "transformer_big_rq_answers028.txt")

# step:avg（5）-39-45000; beam_size:4; - score:0.579406
file_tfbr_029 = os.path.join(path_answers, "transformer_big_rq_answers029.txt")

# step:avg（5）-41-45000; beam_size:4; - score:0.552037
file_tfbr_030 = os.path.join(path_answers, "transformer_big_rq_answers030.txt")

# step:avg（7）-29-45000; beam_size:4; - score:0.544211
file_tfbr_031 = os.path.join(path_answers, "transformer_big_rq_answers031.txt")

# step:avg（5）-39-45000; beam_size:4; - score:0.43291
# answer选择方案： top5选最长
file_tfbr_032 = os.path.join(path_answers, "transformer_big_rq_answers032.txt")

# step:avg（5）-39-45000; beam_size:4; - score:0.468545
# answer选择方案： 得分大于-3的answer中选最长
file_tfbr_033 = os.path.join(path_answers, "transformer_big_rq_answers033.txt")

# step:avg（5）-39-45000; beam_size:4; - score:0.530711
# answer选择方案： 得分大于-3的answer中选最长
file_tfbr_034 = os.path.join(path_answers, "transformer_big_rq_answers034.txt")

# step:avg（5）-39-45000; beam_size:4; - score:0.634846
# replace #E-s\[数字x\] with ""
file_tfbr_035 = os.path.join(path_answers, "transformer_big_rq_answers035.txt")

# step:avg（5）-39-45000; beam_size:4; - score:0.634846
# replace #E-s\[数字x\] with "请问还有其他还可以帮到您的吗?"
file_tfbr_036 = os.path.join(path_answers, "transformer_big_rq_answers036.txt")

bleu(file_tfbr_036, conf.file_test_a)
one_match_ten(file_tfbr_033)
import re
x = "#E-s[数字x]#E-s[数字x]#E-s[数字x]请问还有什么可以帮到您的?怠慢之处请多多谅解哦#E-s[数字x]#E-b[数字x]亲爱的商家，以后您的物流问题可以通过咚咚—小耳机图标—京东物流商家支持或青龙系统、沧海系统、物流官网找到我们哦，感谢您对京东的支持，祝您生活愉快!"
re.sub("(#E-s\[数字x\])", "", x)
re.sub?<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14438682739578423, 'fraction_numerical': 0.1315240083507307, 'mean_word_length': 6.882165605095541, 'pattern_counts': {'":': 0, '<': 40, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '415772', 'license': 'MIT', 'n_tokens_mistral': 8160, 'n_tokens_neox': 6940, 'n_words': 777, 'path': 'final/ipynb/answers_analyze.ipynb', 'repo_name': 'caoxu915683474/jddc_solution_4th', 'size': '14849'}