<jupyter_start><jupyter_text>Preprocessing europarl data<jupyter_code>euro_data_dir = "../europarl/aligned/"
output_data_dir = "./data/original/"<jupyter_output><empty_output><jupyter_text>Join Files<jupyter_code>for l_pair in listdir(euro_data_dir):
    pair_path = join_path(euro_data_dir, l_pair)
    for l_dir in listdir(pair_path):
        out_file = open(output_data_dir + l_pair + "_" + l_dir + ".txt", "w")
        l_path = join_path(pair_path, l_dir)
        for file_name in sorted(listdir(l_path)):
            # Consume content and split lines
            content = open(join_path(l_path, file_name), "r").read() 

            # Write content in output file
            out_file.write(content)<jupyter_output><empty_output><jupyter_text>Load corpus in memory<jupyter_code>def load_corpus(data_dir, as_np_array=False):
    corpus = {}
    for file_name in listdir(data_dir):
        path = join_path(data_dir, file_name)
        if os.path.isfile(path):
            file = open(path, "r+")
            l_pair = file_name[:5]
            if l_pair not in corpus:
                corpus[l_pair] = {}
            if as_np_array:
                corpus[l_pair][file_name] = np.array(file.read().split('\n'), dtype=object)
            else:
                corpus[l_pair][file_name[:3]] = file.read().split('\n')
            file.close()
            remove(path)
    return corpus
corpus = load_corpus(output_data_dir, True)<jupyter_output><empty_output><jupyter_text>Delete short lines and lines with xml tags<jupyter_code>for pair, l_data in corpus.items():

    # Find special lines
    empty_lines = np.array([], dtype=int)
    for _, value in l_data.items():
        is_empty = np.vectorize(lambda x: len(x) <= 7 or x[0] == "<" or x[0] == "(")
        indices = np.nonzero(is_empty(value))[0]
        empty_lines = np.concatenate((empty_lines, indices))
    empty_lines = np.unique(empty_lines)

    # Remove lines with an xml tag or an empty char
    for key, value in l_data.items():
        new_corpus = np.delete(value, empty_lines)
        corpus[pair][key] = new_corpus.tolist()
        print(len(new_corpus))<jupyter_output>2000491
2000491
1910858
1910858
1958593
1958593<jupyter_text>Write new file(s)<jupyter_code>for lang_pair, pair_dict in corpus.items():
    for file_name, data in pair_dict.items():
        file = open(join_path(output_data_dir, file_name), 'w')
        file.write("\n".join(data))<jupyter_output><empty_output><jupyter_text>The following steps must be run after executing the multialign script and the BPE tokenization script  Join language files and add translation token<jupyter_code>tokenized_dir = './data/tokenized/'
output_dir = './data/'

for sub_set in listdir(tokenized_dir):
    sub_set_path = join_path(tokenized_dir, sub_set)

    corpus = {}
    src_data = []
    tgt_data = []

    for file_name in listdir(sub_set_path):
        file_path = join_path(sub_set_path, file_name)
        file = open(file_path, 'r')
        corpus[file_name[:-4]] = file.read().split('\n')

    # An extra line will be created when loading the data to memory,
    # that's why the -1 is there
    for i in range(0, len(corpus['en']) - 1):
        for src in corpus:
            targets = filter(lambda x: x != src, corpus.keys())
            for tgt in targets:
                src_sentence = '_src_{}_tgt_{} '.format(src, tgt) + corpus[src][i]
                src_data.append(src_sentence)
                tgt_data.append(corpus[tgt][i])

    out_src = open(join_path(output_dir, 'europarl_{}_src.txt'.format(sub_set)), 'w')
    out_src.write("\n".join(src_data))
    out_src = open(join_path(output_dir, 'europarl_{}_tgt.txt'.format(sub_set)), 'w')
    out_src.write("\n".join(tgt_data))<jupyter_output><empty_output><jupyter_text>Preprocessing classifier data<jupyter_code>location = './data/translated/'
out_location = './data/'
lang_pool = ['en', 'es', 'fr', 'de']
back_translation_num = 2<jupyter_output><empty_output><jupyter_text>Generate Back-Translations<jupyter_code>ignore = []

def translate_text(text, src, tgt, file_name, dir_path):
    start = time.time()
    response = requests.post('http://localhost:8080/translate', data = {'text': text, 'src': src, 'tgt': tgt})
    end = time.time()
    print('Elapsed time to generate {} for {} to {}: {}s'.format(file_name, src, tgt, end - start))
    res_file = open(join_path(dir_path, '{}.txt'.format(file_name)), 'w')
    res_file.write(response.text)
    return response.text

for file_name in listdir(location):
    path = join_path(location, file_name)
    dir_path = path[:-4]
    if os.path.isfile(path) and file_name[:-4] not in ignore:
        print("Starting loops for:", file_name)
        original_lang = file_name[:2]
        translated_lang = file_name[-6:-4]

        if not os.path.isdir(dir_path):
            mkdir(dir_path)

        content = open(path, 'r').read()
        
        for tgt in filter(lambda x: x != translated_lang, lang_pool):
            tgt_dir = join_path(dir_path, tgt)
            if not os.path.isdir(tgt_dir):
                mkdir(tgt_dir)

            last_back = content
            for i in range(0, back_translation_num):
                translation = translate_text(last_back, translated_lang, tgt, 'T{}'.format(i), tgt_dir)
                last_back = translate_text(translation, tgt, translated_lang, 'Back{}'.format(i), tgt_dir)
            translate_text(last_back, translated_lang, tgt, 'T{}'.format(back_translation_num), tgt_dir)<jupyter_output>Starting loops for: fr_es.txt
Elapsed time to generate T0 for es to en: 207.33868741989136s
Elapsed time to generate Back0 for en to es: 205.3305778503418s
Elapsed time to generate T1 for es to en: 198.34174418449402s
Elapsed time to generate Back1 for en to es: 196.4911506175995s
Elapsed time to generate T2 for es to en: 189.56953811645508s
Elapsed time to generate T0 for es to fr: 233.66205096244812s
Elapsed time to generate Back0 for fr to es: 213.88674139976501s
Elapsed time to generate T1 for es to fr: 216.71543908119202s
Elapsed time to generate Back1 for fr to es: 203.10585403442383s
Elapsed time to generate T2 for es to fr: 211.56110501289368s
Elapsed time to generate T0 for es to de: 207.34401178359985s
Elapsed time to generate Back0 for de to es: 203.3491015434265s
Elapsed time to generate T1 for es to de: 191.28350114822388s
Elapsed time to generate Back1 for de to es: 191.35160493850708s
Elapsed time to generate T2 for es to de: 183.46519470214844s
Starting loops for: fr_de[...]<jupyter_text>Generate BLEU-Score Table<jupyter_code>columns = ['T{}-T{}_{}'.format(i, i+1, lang) for lang in lang_pool for i in range(
    0, back_translation_num)] + ['src', 'origin', 'len']
data = np.zeros((4*back_translation_num+3))
tokenizer = Tokenizer('conservative')
smoothing = SmoothingFunction()
score_func = lambda ref, hyp: sentence_bleu([ref], hyp, smoothing_function=smoothing.method4, weights=(0.50, 0.35, 0.10, 0.05))
#score_func = lambda ref, hyp: meteor_score([ref], hyp)

for directory in [d for d in os.listdir(location) if os.path.isdir(join_path(location, d))]:
    path = join_path(location, directory)

    # Get the original file_size
    src_file = open(join_path(location, "{}.txt".format(directory)), 'r')
    src_sentences = src_file.read().split('\n')
    file_size = len(src_sentences)

    bleu_data = np.zeros((file_size, 4*back_translation_num))

    # Calculate BLEU Score for each sentence and its back-translations
    for lang_idx, lang in enumerate(lang_pool):
        lang_path = join_path(path, lang)
        if not os.path.isdir(lang_path):
            pass
        else:
            #sentences = [src_sentences]
            sentences = []
            for i in range(0, back_translation_num + 1):
                file_name = 'T{}.txt'.format(i)
                file_data = open(join_path(lang_path, file_name),
                                 'r').read().split('\n')
                sentences.append(file_data)

            for i in range(0, file_size):
                for j in range(0, back_translation_num):
                    ref_sentence, _ = tokenizer.tokenize(sentences[j][i])
                    hypothesis, _ = tokenizer.tokenize(sentences[j+1][i])
                    #ref_sentence = sentences[j][i]
                    #hypothesis = sentences[j+1][i]
                    bleu_data[i, lang_idx * back_translation_num +
                        j] = score_func(ref_sentence, hypothesis) if len(hypothesis) > 3 else 0

    # Append src and origin data
    bleu_data = np.hstack(
        (bleu_data, np.full((file_size, 1), lang_pool.index(directory[-2:]))))
    bleu_data = np.hstack(
        (bleu_data, np.full((file_size, 1), lang_pool.index(directory[:2]))))

    # Append sentence length data fron the src file
    src_sentences = open(join_path(location, '{}.txt'.format(directory)), 'r').read().split('\n')
    tok_sentences = [tokenizer.tokenize(s)[0] for s in src_sentences]
    lengths = [len(ts) for ts in tok_sentences]
    bleu_data = np.hstack((bleu_data, np.array([lengths]).T))
    
    data = np.vstack((data, bleu_data))

data.shape
final_data = np.delete(data, (0), axis=0)
dataFrame = pd.DataFrame(final_data, index=range(0, final_data.shape[0]), columns=columns)
dataFrame['src'] = dataFrame['src'].apply(lambda x: lang_pool[int(x)])
dataFrame['origin'] = dataFrame['origin'].apply(lambda x: lang_pool[int(x)])

dataFrame.head()
dataFrame.to_csv(join_path(out_location, 'bleu_table_iter_weighted.csv'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12473706352545225, 'fraction_numerical': 0.038388725283971394, 'mean_word_length': 3.373965041398344, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '408030', 'license': 'MIT', 'n_tokens_mistral': 3494, 'n_tokens_neox': 3220, 'n_words': 749, 'path': 'preprocess.ipynb', 'repo_name': 'pablogadhi/LanguageDetection', 'size': '9508'}