<jupyter_start><jupyter_text>Daily<jupyter_code>import matplotlib.pyplot as plt
import numpy as np
import re
def load(data_path):
    with open(data_path) as f:
        lines = f.readlines()
    data = [line.strip().split('__eou__')[:-1] for line in lines]
    convs = [[s.strip().lower().split() for s in d] for d in data]
    return convs
train_path = 'data/daily/train/dialogues_train.txt'
valid_path = 'data/daily/validation/dialogues_validation.txt'
test_path = 'data/daily/test/dialogues_test.txt'

train_convs = load(train_path)
valid_convs = load(valid_path)
test_convs = load(test_path)
with open(train_path) as f:
    lines = f.readlines()
data = [line.strip().split('__eou__')[:-1] for line in lines]
re.search(r' - ', 'a  -  b')[0]
def load(data_path):
    with open(data_path) as f:
        lines = f.readlines()
    convs = [line.strip().split('__eou__')[:-1] for line in lines]
    return convs

convs = load(train_path)

def clean(convs):
    def _clean(s):
        s = s.strip().lower()
        s = re.sub(r'[()\[\]#$@&*%:"“”~‘=+/]', ' ', s)
        s = re.sub(r' [-—] ', ' ', s)
        s = re.sub(r'(\w)\.(\w)', r'\1 . \2', s)
        s = s.replace('p . m.', ' pm ')
        s = s.replace('a . m.', ' am ')
        s = s.replace('。', '.')
        s = s.replace(';', ',')
        s = re.sub(r'(\w+)\s+’\s+(\w+)', r"\1'\2", s)
        s = re.sub(r'\d+(\.|:,)?\d*', r' <number> ', s)
        s = s.split()
        return s

    convs_clean = []
    for conv in convs:
        convs_clean.append(list(map(_clean, conv)))
    return convs_clean

convs = clean(convs)
def filter(convs):
    cnt = 0
    for i in range(len(convs)):
        for j in range(len(convs[i])):
            for k in range(len(convs[i][j])):
                word = convs[i][j][k]
                if word == '<number>':
                    continue
                f = False
                for c in word:
                    if not c.isalpha() and c not in ",.?!'":
                        f = True
                if f:
                    print(f'[{i},{j},{k}] {word}')
                    cnt += 1
        if cnt >= 800:
            break
                    
filter(convs)
def sent_lens(convs):
    lens = []
    for conv in convs:
        for sent in conv:
            lens.append(len(sent))
            
    short = [l for l in lens if l <= 25]
    print('%.3f%%' % (len(short) / len(lens) * 100))
            
    stat(lens)
    plt.hist(lens, bins=30)
    plt.show()
sent_lens(test_convs)
for conv in test_convs:
    for sent in conv:
        if len(sent) <= 1:
            print(sent)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1796934865900383, 'fraction_numerical': 0.00842911877394636, 'mean_word_length': 1.9337078651685393, 'pattern_counts': {'":': 1, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '154546', 'license': 'MIT', 'n_tokens_mistral': 991, 'n_tokens_neox': 951, 'n_words': 225, 'path': 'test.ipynb', 'repo_name': 'DaDaMrX/seq2seq', 'size': '2610'}