<commit_before>#!/usr/bin/env python

import time
import numpy as np
import tensorflow as tf
import regex as re

from sentence_encoder import SentenceEncoder
from decoder import Decoder
from vocabulary import Vocabulary
from learning_utils import log, training_loop, print_header, tokenize_char_seq, load_tokenized, feed_dropout_and_train
from language_utils import GermanPreprocessor, GermanPostprocessor
from cross_entropy_trainer import CrossEntropyTrainer
from copy_net_trainer import CopyNetTrainer
import cli_options
from language_utils import untruecase, bleu_1, bleu_4_dedup, bleu_4
from decoding_function import Attention, CoverageAttention

def shape(string):
    res_shape = [int(s) for s in string.split("x")]
    return res_shape

if __name__ == "__main__":
    parser = cli_options.get_postediting_parser()
    args = parser.parse_args()

    print_header("TRANSLATION + POSTEDITING", args)

    postedit = untruecase
    preprocess = None
    if args.target_german:
        postedit = GermanPostprocessor()
        preprocess = GermanPreprocessor()

    train_tgt_sentences = load_tokenized(args.train_target_sentences, preprocess=preprocess)
    tokenized_train_tgt_sentences = train_tgt_sentences
    log("Loaded {} training tgt_sentences.".format(len(train_tgt_sentences)))
    val_tgt_sentences = load_tokenized(args.val_target_sentences, preprocess=preprocess)
    log("Loaded {} validation tgt_sentences.".format(len(val_tgt_sentences)))

    train_src_sentences = load_tokenized(args.train_source_sentences)
    log("Loaded {} training src_sentences.".format(len(train_src_sentences)))
    val_src_sentences = load_tokenized(args.val_source_sentences)
    log("Loaded {} validation src_sentences.".format(len(val_src_sentences)))

    train_trans_sentences = load_tokenized(args.train_translated_sentences, preprocess)
    log("Loaded {} training translated sentences.".format(len(train_trans_sentences)))
    val_trans_sentences = load_tokenized(args.val_translated_sentences, preprocess)
    log("Loaded {} validation translated sentences.".format(len(val_trans_sentences)))

    if args.test_output_file:
        if not args.test_source_sentences or not args.test_translated_sentences:
            raise Exception("must supply src and trans sentences when want to translate test set")

        test_src_sentences = load_tokenized(args.test_source_sentences)
        log("Loaded {} test src_sentences.".format(len(test_src_sentences)))
        test_trans_sentences = load_tokenized(args.test_translated_sentences)
        log("Loaded {} test trans_sentences.".format(len(test_trans_sentences)))


    tgt_vocabulary = \
        Vocabulary(tokenized_text=[w for s in train_tgt_sentences for w in s])
    tgt_vocabulary.add_tokenized_text([w for s in train_trans_sentences for w in s])
    src_vocabulary = \
        Vocabulary(tokenized_text=[w for s in train_src_sentences for w in s])

    log("Training tgt_vocabulary has {} words".format(len(tgt_vocabulary)))

    log("Buiding the TensorFlow computation graph.")
    dropout_placeholder = tf.placeholder(tf.float32, name="dropout_keep_prob")
    training_placeholder = tf.placeholder(tf.bool, name="is_training")
    encoder_src = SentenceEncoder(args.maximum_output, src_vocabulary,
                                  args.embeddings_size, args.encoder_rnn_size, dropout_placeholder,
                                  training_placeholder, use_noisy_activations=args.use_noisy_activations,
                                  eval(args.use_attention), name="source_encoder")
    encoder_trans = SentenceEncoder(args.maximum_output, tgt_vocabulary,
                                    args.embeddings_size, args.encoder_rnn_size, dropout_placeholder,
                                    training_placeholder, use_noisy_activations=args.use_noisy_activations,
                                    eval(args.use_attention), name="trans_encoder")

    copy_net = None
    if args.use_copy_net:
        copy_net = (encoder_trans.inputs, encoder_trans.attention_tensor)

    if args.shared_embeddings:
        reused_word_embeddings = encoder_trans.word_embeddings
    else:
        reused_word_embeddings = None

    decoder = Decoder([encoder_src, encoder_trans], tgt_vocabulary, args.decoder_rnn_size,
                      training_placeholder,
                      embedding_size=args.embeddings_size, use_attention=args.use_attention,
                      max_out_len=args.maximum_output, use_peepholes=True,
                      scheduled_sampling=args.scheduled_sampling, dropout_placeholder=dropout_placeholder,
                      copy_net=copy_net, reused_word_embeddings=reused_word_embeddings,
                      use_noisy_activations=args.use_noisy_activations)

    if args.use_copy_net:
        trainer = CopyNetTrainer(decoder, args.l2_regularization)
    else:
        trainer = CrossEntropyTrainer(decoder, args.l2_regularization)

    if args.mixer:
        xent_calls, moving_calls = args.mixer
        trainer = Mixer(decoder, trainer, xent_calls, moving_calls)
<commit_msg>attention arg fix<commit_after>#!/usr/bin/env python

import time
import numpy as np
import tensorflow as tf
import regex as re

from sentence_encoder import SentenceEncoder
from decoder import Decoder
from vocabulary import Vocabulary
from learning_utils import log, training_loop, print_header, tokenize_char_seq, load_tokenized, feed_dropout_and_train
from language_utils import GermanPreprocessor, GermanPostprocessor
from cross_entropy_trainer import CrossEntropyTrainer
from copy_net_trainer import CopyNetTrainer
import cli_options
from language_utils import untruecase, bleu_1, bleu_4_dedup, bleu_4
from decoding_function import Attention, CoverageAttention

def shape(string):
    res_shape = [int(s) for s in string.split("x")]
    return res_shape

if __name__ == "__main__":
    parser = cli_options.get_postediting_parser()
    args = parser.parse_args()

    print_header("TRANSLATION + POSTEDITING", args)

    postedit = untruecase
    preprocess = None
    if args.target_german:
        postedit = GermanPostprocessor()
        preprocess = GermanPreprocessor()

    train_tgt_sentences = load_tokenized(args.train_target_sentences, preprocess=preprocess)
    tokenized_train_tgt_sentences = train_tgt_sentences
    log("Loaded {} training tgt_sentences.".format(len(train_tgt_sentences)))
    val_tgt_sentences = load_tokenized(args.val_target_sentences, preprocess=preprocess)
    log("Loaded {} validation tgt_sentences.".format(len(val_tgt_sentences)))

    train_src_sentences = load_tokenized(args.train_source_sentences)
    log("Loaded {} training src_sentences.".format(len(train_src_sentences)))
    val_src_sentences = load_tokenized(args.val_source_sentences)
    log("Loaded {} validation src_sentences.".format(len(val_src_sentences)))

    train_trans_sentences = load_tokenized(args.train_translated_sentences, preprocess)
    log("Loaded {} training translated sentences.".format(len(train_trans_sentences)))
    val_trans_sentences = load_tokenized(args.val_translated_sentences, preprocess)
    log("Loaded {} validation translated sentences.".format(len(val_trans_sentences)))

    if args.test_output_file:
        if not args.test_source_sentences or not args.test_translated_sentences:
            raise Exception("must supply src and trans sentences when want to translate test set")

        test_src_sentences = load_tokenized(args.test_source_sentences)
        log("Loaded {} test src_sentences.".format(len(test_src_sentences)))
        test_trans_sentences = load_tokenized(args.test_translated_sentences)
        log("Loaded {} test trans_sentences.".format(len(test_trans_sentences)))


    tgt_vocabulary = \
        Vocabulary(tokenized_text=[w for s in train_tgt_sentences for w in s])
    tgt_vocabulary.add_tokenized_text([w for s in train_trans_sentences for w in s])
    src_vocabulary = \
        Vocabulary(tokenized_text=[w for s in train_src_sentences for w in s])

    log("Training tgt_vocabulary has {} words".format(len(tgt_vocabulary)))

    log("Buiding the TensorFlow computation graph.")
    dropout_placeholder = tf.placeholder(tf.float32, name="dropout_keep_prob")
    training_placeholder = tf.placeholder(tf.bool, name="is_training")
    encoder_src = SentenceEncoder(args.maximum_output, src_vocabulary,
                                  args.embeddings_size, args.encoder_rnn_size, dropout_placeholder,
                                  training_placeholder, use_noisy_activations=args.use_noisy_activations,
                                  attention_type=eval(args.use_attention), name="source_encoder")
    encoder_trans = SentenceEncoder(args.maximum_output, tgt_vocabulary,
                                    args.embeddings_size, args.encoder_rnn_size, dropout_placeholder,
                                    training_placeholder, use_noisy_activations=args.use_noisy_activations,
                                    attention_type=eval(args.use_attention), name="trans_encoder")

    copy_net = None
    if args.use_copy_net:
        copy_net = (encoder_trans.inputs, encoder_trans.attention_tensor)

    if args.shared_embeddings:
        reused_word_embeddings = encoder_trans.word_embeddings
    else:
        reused_word_embeddings = None

    decoder = Decoder([encoder_src, encoder_trans], tgt_vocabulary, args.decoder_rnn_size,
                      training_placeholder,
                      embedding_size=args.embeddings_size, use_attention=args.use_attention,
                      max_out_len=args.maximum_output, use_peepholes=True,
                      scheduled_sampling=args.scheduled_sampling, dropout_placeholder=dropout_placeholder,
                      copy_net=copy_net, reused_word_embeddings=reused_word_embeddings,
                      use_noisy_activations=args.use_noisy_activations)

    if args.use_copy_net:
        trainer = CopyNetTrainer(decoder, args.l2_regularization)
    else:
        trainer = CrossEntropyTrainer(decoder, args.l2_regularization)

    if args.mixer:
        xent_calls, moving_calls = args.mixer
        trainer = Mixer(decoder, trainer, xent_calls, moving_calls)

{'fraction_non_alphanumeric': 0.07347740667976424, 'fraction_numerical': 0.00137524557956778, 'mean_word_length': 3.8596658711217184, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6397711', 'n_tokens_mistral': 3078, 'n_tokens_neox': 2974, 'n_words': 552}