<jupyter_start><jupyter_text>Demo code of m3yrin/aligned-cross-entropyauther: @m3yrin<jupyter_code>!nvidia-smi
!git clone https://github.com/m3yrin/aligned-cross-entropy.git
%cd aligned-cross-entropy
!ls<jupyter_output><empty_output><jupyter_text>AXE<jupyter_code>!cat axe/axe.py<jupyter_output><empty_output><jupyter_text>DemoThis notebook is a example code for testing axe onLatent alignment model. (CMLM model is still under construction..)  https://arxiv.org/abs/2004.07437* Base implementatios is https://github.com/m3yrin/nar-latent-alignment. Built on AllenNLP 0.9.0<jupyter_code>!pip install allennlp==0.9.0<jupyter_output><empty_output><jupyter_text>Dataset is an En-Ja parallel provided by @odashi.  See detail at https://github.com/odashi/small_parallel_enja<jupyter_code>%cd demos/datasets
!git clone https://github.com/odashi/small_parallel_enja.git
!ls small_parallel_enja
%cd ../../<jupyter_output><empty_output><jupyter_text>Config<jupyter_code># check common config.
!cat demos/configs/common.jsonnet
%%writefile demos/configs/common.jsonnet
# if you don't use cuda, cuda_device=-1
local cuda_device = 0;

# https://arxiv.org/pdf/2004.07437.pdf
# Our models consists of 12 self-attention
# layers, with 512 hidden size, 2048 filter size, and
# 8 attention heads per layer. We use 0.1 dropout
# for regularization. 

{
    "direction" : "ja-en",
    "train_data_path": "demos/datasets/small_parallel_enja/train",
    "validation_data_path": "demos/datasets/small_parallel_enja/dev",

    "embedding_dim" : 128,
    "feedforward_hidden_dim" : 2048,
    "num_layers" : 12,

    "batch_size" : 256,
    "min_count" : 2,

    "num_epochs" : 150,
    "patience": 10,
    "cuda_device" : cuda_device,
}
# check model config.
!cat demos/configs/axe.jsonnet
%%writefile demos/configs/axe.jsonnet
# common settings
local COMMON = import 'common.jsonnet';

local direction = COMMON['direction'];
local train_data_path = COMMON['train_data_path'];
local validation_data_path = COMMON['validation_data_path'];

local embedding_dim = COMMON['embedding_dim'];
local feedforward_hidden_dim = COMMON['feedforward_hidden_dim'];
local num_layers = COMMON['num_layers'];
local num_epochs = COMMON['num_epochs'];
local batch_size = COMMON['batch_size'];
local patience = COMMON['patience'];
local min_count = COMMON['min_count'];
local cuda_device = COMMON['cuda_device'];

local learning_rate_ctc = 0.001;
local SPECIAL_BLANK_TOKEN = <PASSWORD>@@";

{
    "dataset_reader": {
      "type": "small_parallel_enja_reader",
      "direction" : direction,
      "add_start_end_tokens" : false, # the model doesn't use BOS/EOS for now.
    },
    "train_data_path": train_data_path,
    "validation_data_path": validation_data_path,
    "model": {
        "type": "latent_alignment_ctc",
        "source_embedder": {
            "type": "basic",
            "token_embedders": {
                "tokens": {
                    "type": "embedding",
                    "embedding_dim": embedding_dim
                }
            },
        },
        "net": {
            "type": "bidirectional_language_model_transformer",
            "input_dim": embedding_dim,
            "hidden_dim": feedforward_hidden_dim,
            "num_layers": num_layers,
        },
        "loss_type" : "axe",
        "label_smoothing" : 0.1,
    },
    "iterator": {
        "type": "bucket",
        "sorting_keys": [
            [
                "source_tokens",
                "num_tokens"
            ]
        ],
        "batch_size": batch_size
    },
    "trainer": {
        "optimizer": {
            "type": "adam",
            "lr": learning_rate_ctc
        },
        "patience": patience,
        "validation_metric": "+BLEU",
        "num_epochs": num_epochs,
        "cuda_device": cuda_device,
        "learning_rate_scheduler": {
            "type": "exponential",
            "gamma": 0.98
        },
    },
    "vocabulary": {
        "min_count": {
            "source_tokens": min_count,
            "target_tokens": min_count
        },
        "tokens_to_add": {
            "target_tokens": [
                SPECIAL_BLANK_TOKEN
            ]
        }
    }
}<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>!allennlp train -f --include-package demos -s demos/tmp demos/configs/axe.jsonnet<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code>!allennlp evaluate --output-file demos/tmp/output_test.json --include-package demos demos/tmp/model.tar.gz demos/datasets/small_parallel_enja/test<jupyter_output><empty_output><jupyter_text>Prediction<jupyter_code>!python demos/datasets/make_json.py -I demos/datasets/small_parallel_enja/test.ja -O demos/datasets/test.ja.json
!allennlp predict --output-file demos/tmp/output_pred.json --include-package demos --predictor small_parallel_enja_predictor  demos/tmp/model.tar.gz demos/datasets/test.ja.json  --silent
!cat demos/tmp/output_pred.json<jupyter_output><empty_output><jupyter_text>For comparisonUsing CTC as loss function as https://arxiv.org/abs/2004.07437<jupyter_code>!cat demos/configs/ctc.jsonnet
# %%writefile demos/configs/ctc.jsonnet
!allennlp train -f --include-package demos -s demos/tmp_ctc demos/configs/ctc.jsonnet
!allennlp evaluate --output-file demos/tmp_ctc/output_test.json --cuda-device 0 --include-package demos demos/tmp_ctc/model.tar.gz demos/datasets/small_parallel_enja/test
#!python datasets/make_json.py -I datasets/small_parallel_enja/test.ja -O datasets/test.ja.json
!allennlp predict --output-file demos/tmp_ctc/output_pred.json --cuda-device 0 --include-package demos --predictor small_parallel_enja_predictor  demos/tmp_ctc/model.tar.gz demos/datasets/test.ja.json --silent
!cat demos/tmp_ctc/output_pred.json<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12582551268682657, 'fraction_numerical': 0.013903371567605145, 'mean_word_length': 3.5066562255285825, 'pattern_counts': {'":': 41, '<': 38, '<?xml version=': 0, '>': 38, 'https://': 7, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '527880', 'license': 'MIT', 'n_tokens_mistral': 2080, 'n_tokens_neox': 1992, 'n_words': 337, 'path': 'demo.ipynb', 'repo_name': 'm3yrin/aligned-cross-entropy', 'size': '5752'}