<jupyter_start><jupyter_text>Формирование транзитивного корпуса из статьи https://ai.facebook.com/research/publications/the-flores-101-evaluation-benchmark-for-low-resource-and-multilingual-machine-translationhttps://scontent-hel3-1.xx.fbcdn.net/v/t39.8562-6/196203317_1861942553982349_5142503689226033347_n.pdf?_nc_cat=110&ccb=1-5&_nc_sid=ae5e01&_nc_ohc=naDhnLsjFFAAX-feR3_&_nc_ht=scontent-hel3-1.xx&oh=00_AT-zFtEpWD57R0Rv16P91dEoy_x8YKvCdJfhvMEpoRXwbw&oe=623830CD<jupyter_code>import os
os.listdir('flores101_dataset')
meta_1 = 'flores101_dataset/metedata_dev.tsv'
meta_2 = 'flores101_dataset/metedata_devtest.tsv'
os.listdir('flores101_dataset/dev/')
with open('flores101_dataset/dev/rus.dev', encoding='utf-8') as f:
    file = f.readlines()
len(file)
os.listdir('flores101_dataset/devtest/')
with open('flores101_dataset/devtest/rus.devtest', encoding='utf-8') as f:
    file = f.readlines()
file[123]
997 + 1012
def get_list(path_file):
    with open(path_file, encoding='utf-8') as f:
        return f.readlines()


paths = ['flores101_dataset/dev', 'flores101_dataset/devtest']
dev_names = set([i.split('.')[0] for i in os.listdir(paths[0])])
devtest_names = set([i.split('.')[0] for i in os.listdir(paths[1])])
names = dev_names | devtest_names
result = dict.fromkeys(names, [])

for path in paths:
    for file in os.listdir(path):
        path_file = f'{path}/{file}'
        name = file.split('.')[0]
        tmp = result[name][:]
        tmp.extend(get_list(path_file))
        result[name] = tmp[:]
len(result['rus'])
import pandas as pd
corpus = pd.DataFrame.from_dict(result)
corpus.shape
corpus[['rus','eng']]
meta_1 = 'flores101_dataset/metedata_dev.tsv'
meta_2 = 'flores101_dataset/metedata_devtest.tsv'
meta_data_1 = pd.read_csv(meta_1, sep='\t')
meta_data_2 = pd.read_csv(meta_2, sep='\t')
meta_data = pd.concat([meta_data_1, meta_data_2])
meta_data_1.head()
meta_data_2.head()
meta = meta_data_1.append(meta_data_2)
meta = meta[['topic']].reset_index()
del meta['index']
meta
len(meta), len(corpus)
meta['id'] = meta.index
meta
data = corpus
data['id'] = corpus.index
meta.shape, data.shape
meta.nunique()
uni = data.nunique()
tmp = pd.DataFrame(uni, columns=['num'])
tmp[tmp['num'] < 2009]
total = pd.merge(meta, data, how ='left', left_on=('id'), right_index=True)
del total['id_x']
del total['id_y']
total
total.to_excel('101_languages.xlsx') # самый минимальный размер создает
corpus.to_excel('languages.xlsx')
total.to_csv('101_languages.csv')<jupyter_output><empty_output><jupyter_text>Одинаковые предложения; на примере kam<jupyter_code>total.nunique().values
kam = total['kam']
kam = total[kam.isin(kam[kam.duplicated()])]
kam.to_excel('kam.xlsx')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16364979682305134, 'fraction_numerical': 0.058367196158108606, 'mean_word_length': 6.37874659400545, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '408028', 'license': 'MIT', 'n_tokens_mistral': 1225, 'n_tokens_neox': 1135, 'n_words': 156, 'path': 'test_dataset/flores101_dataset/parser.ipynb', 'repo_name': 'eleldar/Translator', 'size': '2707'}