<jupyter_start><jupyter_text>Preparing OpenNMT and other dependencies<jupyter_code>%cd ./drive/My Drive
!pip install OpenNMT-py
!git clone https://github.com/cdli-gh/Unsupervised-NMT-for-Sumerian-English.git
!git clone https://github.com/OpenNMT/OpenNMT-py.git
%cd OpenNMT-py/
!mkdir ./data/sum-en
%cp ../Unsupervised-NMT-for-Sumerian-English/dataset/dataToUse/* ./data/sum-en/<jupyter_output><empty_output><jupyter_text>Preparing Data<jupyter_code>!onmt_preprocess -train_src data/sum-en/sum_train.txt -train_tgt data/sum-en/eng_train.txt -valid_src data/sum-en/sum_dev.txt -valid_tgt data/sum-en/eng_dev.txt -save_data data/sum-en/demo<jupyter_output>Traceback (most recent call last):
  File "/usr/local/bin/onmt_preprocess", line 8, in <module>
    sys.exit(main())
  File "/usr/local/lib/python3.6/dist-packages/onmt/bin/preprocess.py", line 298, in main
    preprocess(opt)
  File "/usr/local/lib/python3.6/dist-packages/onmt/bin/preprocess.py", line 246, in preprocess
    ArgumentParser.validate_preprocess_args(opt)
  File "/usr/local/lib/python3.6/dist-packages/onmt/utils/parse.py", line 144, in validate_preprocess_args
    assert os.path.isfile(file), "Please check path of %s" % file
AssertionError: Please check path of data/sum-en/sum_train.txt<jupyter_text>Training<jupyter_code>!python  train.py -data ./data/sum-en/demo -save_model ./data/sum-en/ \
        -layers 6 -rnn_size 512 -word_vec_size 512 -transformer_ff 2048 -heads 8  \
        -encoder_type transformer -decoder_type transformer -position_encoding \
        -train_steps 100000  -max_generator_batches 2 -dropout 0.1 \
        -batch_size 4096 -batch_type tokens -normalization tokens  -accum_count 2 \
        -train_from ./data/sum-en/_step_5000.pt \
        -optim adam -adam_beta2 0.998 -decay_method noam -warmup_steps 8000 -learning_rate 2 \
        -max_grad_norm 0 -param_init 0  -param_init_glorot \
        -label_smoothing 0.1 -valid_steps 1000 -save_checkpoint_steps 1000 \
        -world_size 1 -gpu_ranks 0
        # -early_stopping 5 \<jupyter_output>[2020-06-08 17:01:03,939 INFO] Loading checkpoint from ./data/sum-en/_step_5000.pt
[2020-06-08 17:01:05,596 INFO] Loading vocab from checkpoint at ./data/sum-en/_step_5000.pt.
[2020-06-08 17:01:05,628 INFO]  * src vocab size = 29242
[2020-06-08 17:01:05,628 INFO]  * tgt vocab size = 15062
[2020-06-08 17:01:05,628 INFO] Building model...
[2020-06-08 17:01:15,102 INFO] NMTModel(
  (encoder): TransformerEncoder(
    (embeddings): Embeddings(
      (make_embedding): Sequential(
        (emb_luts): Elementwise(
          (0): Embedding(29242, 512, padding_idx=1)
        )
        (pe): PositionalEncoding(
          (dropout): Dropout(p=0.1, inplace=False)
        )
      )
    )
    (transformer): ModuleList(
      (0): TransformerEncoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_keys): Linear(in_features=512, out_features=512, bias=True)
          (linear_values): Linear(in_features=512, out_features=512, bias=True)
          (linear_query): Linear(in_features=512,[...]<jupyter_text>Translating test data and saving<jupyter_code>!onmt_translate -model ./data/sum-en/_step_21000.pt -src ./data/sum-en/sum_test.txt -output ./data/sum-en/pred_engg1.txt -replace_unk -verbose
with open('./data/sum-en/sum_test.txt', 'r') as f:
  inp = f.readlines()
with open('./data/sum-en/eng_test.txt', 'r') as f:
  actual = f.readlines()
with open('./data/sum-en/pred_engg1.txt', 'r') as f:
  preds = f.readlines()
len(actual), len(preds), len(inp)
import pandas as pd
to_df = {'input':inp[:len(preds)], 'actual':actual[:len(preds)], 'predicted':preds}

pd.DataFrame(to_df).to_csv('../Unsupervised-NMT-for-Sumerian-English/Results/Tranformer1Preds2.csv')<jupyter_output><empty_output><jupyter_text>Evaluating Translations  BLEU<jupyter_code>df = pd.read_csv('../Unsupervised-NMT-for-Sumerian-English/Results/Tranformer1Preds2.csv')
actual = list(df['actual'])
preds = list(df['predicted'])
import nltk

#there may be several references

blues = []
for i in range(len(actual)):
  if(len(preds[i].split(' ')) > 4):
    blues.append(nltk.translate.bleu_score.sentence_bleu([actual[i]], preds[i]))
  else:
    blues.append(nltk.translate.bleu_score.sentence_bleu([actual[i]], preds[i], weights=(0.5, 0.5)))
print(len(blues))
import numpy as np
np.mean(blues)
df1 = pd.read_csv('../../../Transform_model_results.csv')
df1.head()
actual = list(df1['English - Exact'])
preds = list(df1['English - Predicted'])

blues = []
for i in range(len(actual)):
  try:
    if(len(preds[i].split(' ')) > 4):
      blues.append(nltk.translate.bleu_score.sentence_bleu([actual[i]], preds[i]))
    else:
      blues.append(nltk.translate.bleu_score.sentence_bleu([actual[i]], preds[i], weights=(0.5, 0.5)))

  except:
    continue
np.mean(blues)<jupyter_output><empty_output><jupyter_text>ROUGE<jupyter_code>!pip install py-rouge
import rouge
import nltk
nltk.download('punkt')
actual = list(df['actual'])
preds = list(df['predicted'])
evaluator = rouge.Rouge(metrics=['rouge-n', 'rouge-l', 'rouge-w'],
                        max_n=4,
                        limit_length=True,
                        length_limit=100,
                        length_limit_type='words',
                        apply_avg=1,
                        apply_best=0,
                        alpha=0.5, # Default F1_score
                        weight_factor=1.2,
                        stemming=False)

scores = evaluator.get_scores(preds, actual[:len(preds)])
scores
actual = list(df['actual'])
preds = list(df['predicted'])
scores = evaluator.get_scores(preds[:900], actual[:900])
scores<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14839737913936604, 'fraction_numerical': 0.04621923145032761, 'mean_word_length': 3.599348534201954, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 26, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '409593', 'license': 'MIT', 'n_tokens_mistral': 2270, 'n_tokens_neox': 2056, 'n_words': 358, 'path': 'Notebooks/TransformerBase.ipynb', 'repo_name': 'cdli-gh/Unsupervised-NMT-for-Sumerian-English-', 'size': '5647'}