<commit_before>#!/bin/bash

#for fetch_corpus
corpus=europarl
corpus_type=parallel
corpus_test=news-test
corpus_dev=news-dev 

#for translation
data_dir=data/europarl-parallel-fr-en-UNK
train_dir=model/europarl-parallel-fr-en-UNK
size_layer=1024
num_layers=1
gpu_id=1


echo ">>>>>>>>> calling fetch_corpus"

./scripts/fetch-corpus.py ${corpus} ${corpus_type} fr ${data_dir}
./scripts/fetch-corpus.py ${corpus_test} mono fr en ${data_dir}
./scripts/fetch-corpus.py ${corpus_dev}  mono fr en ${data_dir}

echo ">>>>>>>>> calling prepare_data"

./scripts/prepare-data.py ${data_dir}/${corpus} fr en ${data_dir} --mode all \
                                                                  --verbose \
                                                                  --normalize-digits \
                                                                  --normalize-punk \
                                                                  --unk-align \
                                                                  --dev-corpus  ${data_dir}/${corpus_dev} \
                                                                  --test-corpus ${data_dir}/${corpus_test}

echo ">>>>>>>>> calling translation model"

python -m translate ${data_dir} ${train_dir} \
--size ${size_layer} \
--num-layers ${num_layers} \
--src-ext fr \
--trg-ext en \
--verbose \
--log-file ${train_dir}/log.txt \
--gpu-id ${gpu_id}
--replace-unk<commit_msg>Update europarl-parallel-fr-en-UNK.sh<commit_after>#!/bin/bash

#for fetch_corpus
corpus=europarl
corpus_type=parallel
corpus_test=news-test
corpus_dev=news-dev 

#for translation
data_dir=data/europarl-parallel-fr-en-UNK
train_dir=model/europarl-parallel-fr-en-UNK
size_layer=1024
num_layers=1
gpu_id=1


echo ">>>>>>>>> calling fetch_corpus"

./scripts/fetch-corpus.py ${corpus} ${corpus_type} fr ${data_dir}
./scripts/fetch-corpus.py ${corpus_test} mono fr en ${data_dir}
./scripts/fetch-corpus.py ${corpus_dev}  mono fr en ${data_dir}

echo ">>>>>>>>> calling prepare_data"

./scripts/prepare-data.py ${data_dir}/${corpus} fr en ${data_dir} --mode all \
                                                                  --verbose \
                                                                  --normalize-digits \
                                                                  --normalize-punk \
                                                                  --unk-align \
                                                                  --dev-corpus  ${data_dir}/${corpus_dev} \
                                                                  --test-corpus ${data_dir}/${corpus_test}

echo ">>>>>>>>> calling translation model"

python -m translate ${data_dir} ${train_dir} \
--size ${size_layer} \
--num-layers ${num_layers} \
--src-ext fr \
--trg-ext en \
--verbose \
--log-file ${train_dir}/log.txt \
--gpu-id ${gpu_id} \
--replace-unk
{'fraction_non_alphanumeric': 0.14265927977839335, 'fraction_numerical': 0.004155124653739612, 'mean_word_length': 1.42162615255658, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2223162', 'n_tokens_mistral': 848, 'n_tokens_neox': 827, 'n_words': 150}