<jupyter_start><jupyter_text>Sample notebook for https://github.com/m3yrin/nar-latent-alignmentAuther: @m3yrin* Tested on Google Colaboratory.* Check gpu is enabled.  Preparation<jupyter_code># check gpu is enabled.
!nvidia-smi
# download codes
!git clone https://github.com/m3yrin/nar-latent-alignment.git
%cd nar-latent-alignment
!ls
!pip install -r requirements.txt
# download dataset
%cd datasets
!git clone https://github.com/odashi/small_parallel_enja.git
!ls small_parallel_enja
%cd ../
# check common config.
!cat configs/common.jsonnet
%%writefile configs/common.jsonnet
# if you don't use cuda, cuda_device=-1
local cuda_device = 0;

# https://arxiv.org/pdf/2004.07437.pdf
# Our models consists of 12 self-attention
# layers, with 512 hidden size, 2048 filter size, and
# 8 attention heads per layer. We use 0.1 dropout
# for regularization. 

{
    "direction" : "ja-en",
    "train_data_path": "datasets/small_parallel_enja/train",
    "validation_data_path": "datasets/small_parallel_enja/dev",

    "embedding_dim" : 128,
    "feedforward_hidden_dim" : 2048,
    "num_layers" : 12,

    "batch_size" : 128,
    "min_count" : 2,

    "num_epochs" : 150,
    "patience": 10,
    "cuda_device" : cuda_device,
}<jupyter_output><empty_output><jupyter_text>CTC model<jupyter_code>!cat configs/ctc.jsonnet
# %%writefile configs/ctc.jsonnet
!allennlp train -f --include-package src -s tmp_ctc configs/ctc.jsonnet
%%time
!allennlp evaluate --output-file tmp_ctc/output_test.json --cuda-device 0 --include-package src tmp_ctc/model.tar.gz datasets/small_parallel_enja/test
# building a json file for prediction
!python datasets/make_json.py -I datasets/small_parallel_enja/test.ja -O datasets/test.ja.json
!allennlp predict --output-file tmp_ctc/output_pred.json --cuda-device 0 --include-package src --predictor small_parallel_enja_predictor  tmp_ctc/model.tar.gz datasets/test.ja.json --silent
!cat tmp_ctc/output_pred.json<jupyter_output><empty_output><jupyter_text>Transformer Autoregressive model (for comparison)<jupyter_code>!cat configs/transformer_ar.jsonnet
# %%writefile configs/transformer_ar.jsonnet
!allennlp train -f --include-package src -s tmp_ar configs/transformer_ar.jsonnet
%%time
!allennlp evaluate --output-file tmp_ar/output_test.json --cuda-device 0 --include-package src tmp_ar/model.tar.gz datasets/small_parallel_enja/test
# !python datasets/make_json.py -I datasets/small_parallel_enja/test.ja -O datasets/test.ja.json
!allennlp predict --output-file tmp_ar/output_pred.json --cuda-device 0 --include-package src --predictor small_parallel_enja_predictor  tmp_ar/model.tar.gz datasets/test.ja.json --silent
!cat tmp_ar/output_pred.json<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11712379540400296, 'fraction_numerical': 0.017790956263899184, 'mean_word_length': 4.931868131868132, 'pattern_counts': {'":': 3, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '289604', 'license': 'MIT', 'n_tokens_mistral': 1009, 'n_tokens_neox': 949, 'n_words': 204, 'path': 'colab/experiment.ipynb', 'repo_name': 'm3yrin/nar-latent-alignment', 'size': '2698'}