<jupyter_start><jupyter_text>**Epoch 200**<jupyter_code># Train the baseline transformer model
# For independent vocabulary, update the training command as, "fairseq-train data-bin/en_si_unigram --source-lang en --target-lang si"
import os
os.chdir("your directory")
!CUDA_VISIBLE_DEVICES=0 fairseq-train \
    data-bin/en_si_unigram5000/ \
    --source-lang en --target-lang si \
    --arch transformer --share-all-embeddings \
    --encoder-layers 5 --decoder-layers 5 \
    --encoder-embed-dim 512 --decoder-embed-dim 512 \
    --encoder-ffn-embed-dim 2048 --decoder-ffn-embed-dim 2048 \
    --encoder-attention-heads 2 --decoder-attention-heads 2 \
    --encoder-normalize-before --decoder-normalize-before \
    --dropout 0.3 --attention-dropout 0.2 --relu-dropout 0.2 \
    --weight-decay 0.0001 \
    --label-smoothing 0.2 --criterion label_smoothed_cross_entropy \
    --optimizer adam --adam-betas '(0.9, 0.98)' --clip-norm 0 \
    --lr-scheduler inverse_sqrt --warmup-updates 4000 --warmup-init-lr 1e-7 \
    --lr 1e-3 --min-lr 1e-9 \
    --batch-size 32 \
    --update-freq 4 \
    --max-epoch 200 --save-interval 10 \
    --tensorboard-logdir logs/
#200 epoch - validation set
# Obtain the BLEU score for validation dataset
# For independent vocabulary, update the scoring command as, "fairseq-generate data-bin/en_si_unigram --source-lang en --target-lang si"
os.chdir("your directory")
!fairseq-generate \
    data-bin/en_si_unigram5000/ \
    --source-lang en --target-lang si \
    --path checkpoints/checkpoint_best.pt \
    --beam 5 --lenpen 1.2 \
    --gen-subset valid \
    --remove-bpe=sentencepiece
#200 epoch - test set
# Obtain the BLEU score for test dataset
# For independent vocabulary, update the scoring command as, "fairseq-generate data-bin/en_si_unigram --source-lang en --target-lang si"
os.chdir("your directory")
!fairseq-generate \
    data-bin/en_si_unigram5000/ \
    --source-lang en --target-lang si \
    --path checkpoints/checkpoint_best.pt \
    --beam 5 --lenpen 1.2 \
    --gen-subset test \
    --remove-bpe=sentencepiece
# Obtain the checkpoint ensembled BLEU score for validation dataset
# For independent vocabulary, update the scoring command as, "fairseq-generate data-bin/en_si_unigram --source-lang en --target-lang si"
os.chdir("your directory")
!fairseq-generate \
    data-bin/en_si_unigram5000/ \
    --source-lang en --target-lang si \
    --path checkpoints/checkpoint_last.pt:checkpoints/checkpoint190.pt \
    --beam 5 --lenpen 1.2 \
    --gen-subset valid \
    --remove-bpe=sentencepiece
# Obtain the checkpoint ensembled BLEU score for test dataset
# For independent vocabulary, update the scoring command as, "fairseq-generate data-bin/en_si_unigram --source-lang en --target-lang si"
os.chdir("your directory")
!fairseq-generate \
    data-bin/en_si_unigram5000/ \
    --source-lang en --target-lang si \
    --path checkpoints/checkpoint_last.pt:checkpoints/checkpoint190.pt \
    --beam 5 --lenpen 1.2 \
    --gen-subset test \
    --remove-bpe=sentencepiece
# Load the TensorBoard notebook extension
%load_ext tensorboard
# Visualize using tensorboard
%tensorboard --logdir logs/<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12861635220125786, 'fraction_numerical': 0.03238993710691824, 'mean_word_length': 2.907862407862408, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '129754', 'license': 'Apache-2.0', 'n_tokens_mistral': 1164, 'n_tokens_neox': 1052, 'n_words': 269, 'path': 'src/Subword-segmentation/Transformer-subword-regularization/notebooks/train.ipynb', 'repo_name': 'aaivu/aaivu-machine-trans-eng-sin', 'size': '3180'}