<jupyter_start><jupyter_text>读取数据<jupyter_code>import pandas as pd

train_labeled_cn = pd.read_csv('../data/raw/cn_train.csv', encoding='utf-8')
dev_cn = pd.read_csv('../data/raw/cn_dev.csv', encoding='utf-8')
train_labeled_cn.shape
train_labeled_cn.columns
train_labeled_cn.head(5)<jupyter_output><empty_output><jupyter_text>统计重复的句子<jupyter_code>dup_sentence = train_labeled_cn[train_labeled_cn['Sentence'].duplicated(keep=False)]
len(dup_sentence)
dup_sentence
dup_sentence['Label'].value_counts()<jupyter_output><empty_output><jupyter_text>Dialogue长度<jupyter_code>dialogue_info = {}
greater = 0
for index, line in train_labeled_cn.iterrows():
    dialogue_id = int(line[1])
    dialogue_info[dialogue_id] = dialogue_info.get(dialogue_id, 0) + 1

for key, value in dialogue_info.items():
    if value > 64:
        greater += 1

print(len(dialogue_info))
print(greater)<jupyter_output>348
47<jupyter_text>标签分布<jupyter_code>train_labeled_cn['Label'].value_counts()  # 1:2.4769
%matplotlib inline
train_labeled_cn['Label'].value_counts(normalize=True).plot(kind='bar');<jupyter_output><empty_output><jupyter_text>人物数量及标签分布<jupyter_code>speaker_labels_mean = dict(train_labeled_cn.groupby("Speaker")['Label'].mean())
# sorted(speaker_labels_mean.items(), key=lambda item: item[1], reverse=True)
speaker_labels_sum = dict(train_labeled_cn.groupby("Speaker")['Label'].sum())
# print(speaker_labels_sum)
# sorted(speaker_labels_sum.items(), key=lambda item: item[1], reverse=True)
from collections import Counter

train_cn_counter = Counter(train_labeled_cn['Speaker'])
dev_cn_counter = Counter(dev_cn['Speaker'])
# print(train_cn_counter)
# print("**********************************************")
print(dev_cn_counter)
speaker_analysis = {}

for (key, value) in speaker_labels_sum.items():
    speaker_analysis[key] = [round(train_cn_counter[key], 3), value, round(speaker_labels_mean[key], 3)]

# print(speaker_analysis)
sorted(speaker_analysis.items(), key=lambda item: item[1][2], reverse=True)
train_cn_set = set(train_labeled_cn['Speaker'])
dev_cn_set = set(dev_cn['Speaker'])
print("训练集人物个数：", len(train_cn_set))
print("开发集人物个数：", len(dev_cn_set))
print(dev_cn_set - train_cn_set)<jupyter_output>训练集人物个数： 76
开发集人物个数： 24
{'股东男', '股东女'}<jupyter_text>把训练集中的笑点数为0的speaker全丢到一类：other，开发集中有在训练集中未出现过的speaker就不管了  文本长度<jupyter_code>train_labeled_cn['Sentence'].str.len().describe()
dev_cn['Sentence'].str.len().describe()<jupyter_output><empty_output><jupyter_text>数据划分<jupyter_code>from sklearn.model_selection import train_test_split

dataset = train_labeled[['Dialogue_id', 'Speaker', 'Sentence', 'Label']]
train, valid = train_test_split(dataset, test_size=0.2, stratify=train_labeled['Label'], random_state=1000)
train.to_csv('data/preprocess/cn_train.tsv', index=False, header=False, sep='\t',encoding='utf-8')
valid.to_csv('data/preprocess/cn_dev.tsv', index=False, header=False, sep='\t', encoding='utf-8')
dataset.to_csv('data/preprocess/cn_total.tsv', index=False, header=False, sep='\t', encoding='utf-8')<jupyter_output><empty_output><jupyter_text>英文<jupyter_code>import pandas as pd

train_labeled_en = pd.read_csv('../data/raw/en_train.csv', encoding='utf-8')
dev_en = pd.read_csv('../data/raw/en_dev.csv', encoding='utf-8')
train_labeled_en.shape
train_labeled_en.columns
train_labeled_en.head(5)
train_labeled_en['Sentence'].head(10)
pd.set_option('max_columns',1000)
pd.set_option('max_row',1000)
pd.set_option('display.float_format', lambda x: '%.5f' % x)
dup_sentence_en = train_labeled_en[train_labeled_en['Sentence'].duplicated(keep=False)]
len(dup_sentence_en)
dup_sentence_en
dup_sentence_en['Label'].value_counts()<jupyter_output><empty_output><jupyter_text>Dialogue长度<jupyter_code>dialogue_info = {}
greater = 0
for index, line in train_labeled_en.iterrows():
    dialogue_id = int(line[1])
    dialogue_info[dialogue_id] = dialogue_info.get(dialogue_id, 0) + 1

for key, value in dialogue_info.items():
    if value > 16:
        greater += 1

print(len(dialogue_info))
print(greater)<jupyter_output>551
138<jupyter_text>标签分布<jupyter_code>train_labeled_en['Label'].value_counts()   # 1:3.214
%matplotlib inline
train_labeled_en['Label'].value_counts(normalize=True).plot(kind='bar');<jupyter_output><empty_output><jupyter_text>人物数量及标签分布<jupyter_code>speaker_en_labels_mean = dict(train_labeled_en.groupby("Speaker")['Label'].mean())
# sorted(speaker_labels_mean.items(), key=lambda item: item[1], reverse=True)
speaker_en_labels_sum = dict(train_labeled_en.groupby("Speaker")['Label'].sum())
# print(speaker_labels_sum)
# sorted(speaker_labels_sum.items(), key=lambda item: item[1], reverse=True)
from collections import Counter

train_en_counter = Counter(train_labeled_en['Speaker'])
dev_en_counter = Counter(dev_en['Speaker'])
# print(train_cn_counter)
# print("**********************************************")
print(dev_en_counter)
speaker_analysis_en = {}

for (key, value) in speaker_en_labels_sum.items():
    speaker_analysis_en[key] = [round(train_en_counter[key], 3), value, round(speaker_en_labels_mean[key], 3)]

# print(speaker_analysis)
sorted(speaker_analysis_en.items(), key=lambda item: item[1][2], reverse=True)
train_en_set = set(train_labeled_en['Speaker'])
dev_en_set = set(dev_en['Speaker'])
print("训练集人物个数：", len(train_en_set))
print("开发集人物个数：", len(dev_en_set))
print(dev_en_set - train_en_set)<jupyter_output>训练集人物个数： 215
开发集人物个数： 36
{'TV Announcer', 'Doctor', '<NAME>', 'Cookie', 'Stage Manager', 'Lauren', 'The Hot Girl', 'Jen', 'Estelle', 'Whitney', 'Kori', '<NAME>', 'Dr. Harad', 'Max'}<jupyter_text>文本长度<jupyter_code>train_labeled_en['Sentence'].apply(lambda x : len(str(x).split(" "))).describe()
dev_en['Sentence'].apply(lambda x : len(str(x).split(" "))).describe()<jupyter_output><empty_output><jupyter_text>数据划分<jupyter_code>from sklearn.model_selection import train_test_split

dataset = train_labeled_en[['Dialogue_id', 'Speaker', 'Sentence', 'Label']]
train, valid = train_test_split(dataset, test_size=0.2, stratify=train_labeled['Label'], random_state=1000)
train.to_csv('data/preprocess/en_train.tsv', index=False, header=False, sep='\t',encoding='utf-8')
valid.to_csv('data/preprocess/en_dev.tsv', index=False, header=False, sep='\t', encoding='utf-8')
dataset.to_csv('data/preprocess/en_total.tsv', index=False, header=False, sep='\t', encoding='utf-8')
import pandas as pd
train_labeled_en = pd.read_csv('data/raw/cn_dev.csv', encoding='utf-8')
train_labeled_en.shape
train_labeled_en.columns
train_labeled_en.head(5)
train_labeled_en['Sentence'].apply(lambda x : len(str(x).split(" "))).describe()
dataset = train_labeled_en[['Dialogue_id', 'Speaker', 'Sentence']]
dataset.to_csv('data/preprocess/cn_test.tsv', index=False, header=False, sep='\t', encoding='utf-8')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1743566992014197, 'fraction_numerical': 0.01419698314108252, 'mean_word_length': 10.178512396694215, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '28645', 'license': 'MIT', 'n_tokens_mistral': 2711, 'n_tokens_neox': 2631, 'n_words': 323, 'path': 'utils/EDA.ipynb', 'repo_name': 'CCChenhao997/CCL2020-Humor-Computation', 'size': '6782'}