<jupyter_start><jupyter_text>熟悉数据---<jupyter_code>import os
import re
from tqdm import tqdm
import traceback
import sys

sys.path.insert(0, "/home/team55/notespace/zengbin")

from jddc.config import PreConfig
from jddc.utils import write_file, read_file, save_to_pkl, read_from_pkl, create_logger

conf = PreConfig()
logger = create_logger(name='pre', log_file=conf.log_file, cmd=conf.cmd_log)<jupyter_output><empty_output><jupyter_text>测试数据---单轮测试50条，每条对应10个参考答案。多轮会话5个，每个都有若干个问题，每个问题对应有10个参考答案。<jupyter_code># 读入测试数据
test_q = read_file(conf.file_test_q)
test_a = read_file(conf.file_test_a) 
# 拆分50条单轮测试 & 5个多轮会话测试
test_q50 = [x for x in test_q if "<s>" in x]
test_mq5 = [x for x in test_q if "<s>" not in x]
q50_qs = [" ".join((x.split("<s>")[0], x.split("<s>")[2], x.split("<s>")[4]))  for x in test_q50]
q50_qs_l = [len(x) for x in q50_qs]
q50_qs_l
test_a[-10:]
test_q50[0]
re.sub('(<s>)|(，)', ' ', test_q50[0]+"<s")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.167192429022082, 'fraction_numerical': 0.03890641430073607, 'mean_word_length': 6.867768595041323, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '151874', 'license': 'MIT', 'n_tokens_mistral': 485, 'n_tokens_neox': 496, 'n_words': 79, 'path': 'final/ipynb/understand_data.ipynb', 'repo_name': 'wangle1218/jddc_solution_4th', 'size': '951'}