<commit_before># Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""TensorFlow NMT model implementation."""
from __future__ import print_function

import argparse
import os
import random
import sys

# import matplotlib.image as mpimg
import numpy as np
import tensorflow as tf

from . import inference
from . import train
from .utils import evaluation_utils
from .utils import misc_utils as utils
from .utils import vocab_utils

utils.check_tensorflow_version()

FLAGS = None


def create_hparams():
  """Create training hparams."""
  return tf.contrib.training.HParams(
      # Data
      src=FLAGS.src,
      tgt=FLAGS.tgt,
      train_prefix=FLAGS.train_prefix,
      dev_prefix=FLAGS.dev_prefix,
      test_prefix=FLAGS.test_prefix,
      vocab_prefix=FLAGS.vocab_prefix,
      out_dir=FLAGS.out_dir,

      # Networks
      num_units=FLAGS.num_units,
      num_layers=FLAGS.num_layers,
      dropout=FLAGS.dropout,
      unit_type=FLAGS.unit_type,
      encoder_type=FLAGS.encoder_type,
      residual=FLAGS.residual,
      time_major=FLAGS.time_major,

      # Attention mechanisms
      attention=FLAGS.attention,
      attention_architecture=FLAGS.attention_architecture,
      pass_hidden_state=FLAGS.pass_hidden_state,

      # Train
      optimizer=FLAGS.optimizer,
      num_train_steps=FLAGS.num_train_steps,
      batch_size=FLAGS.batch_size,
      init_weight=FLAGS.init_weight,
      max_gradient_norm=FLAGS.max_gradient_norm,
      learning_rate=FLAGS.learning_rate,
      start_decay_step=FLAGS.start_decay_step,
      decay_factor=FLAGS.decay_factor,
      decay_steps=FLAGS.decay_steps,
      colocate_gradients_with_ops=FLAGS.colocate_gradients_with_ops,

      # Data constraints
      num_buckets=FLAGS.num_buckets,
      max_train=FLAGS.max_train,
      src_max_len=FLAGS.src_max_len,
      tgt_max_len=FLAGS.tgt_max_len,
      source_reverse=FLAGS.source_reverse,

      # Inference
      src_max_len_infer=FLAGS.src_max_len_infer,
      tgt_max_len_infer=FLAGS.tgt_max_len_infer,
      infer_batch_size=FLAGS.infer_batch_size,
      beam_width=FLAGS.beam_width,
      length_penalty_weight=FLAGS.length_penalty_weight,

      # Vocab
      sos=FLAGS.sos if FLAGS.sos else vocab_utils.SOS,
      eos=FLAGS.eos if FLAGS.eos else vocab_utils.EOS,
      bpe_delimiter=FLAGS.bpe_delimiter,

      # Misc
      forget_bias=FLAGS.forget_bias,
      num_gpus=FLAGS.num_gpus,
      epoch_step=0,  # record where we were within an epoch.
      steps_per_stats=FLAGS.steps_per_stats,
      steps_per_external_eval=FLAGS.steps_per_external_eval,
      share_vocab=FLAGS.share_vocab,
      metrics=FLAGS.metrics.split(","),
      log_device_placement=FLAGS.log_device_placement,
      random_seed=FLAGS.random_seed,
  )


def extend_hparams(hparams):
  """Extend training hparams."""
  # Sanity checks
  if hparams.encoder_type == "bi" and hparams.num_layers % 2 != 0:
    raise ValueError("For bi, num_layers %d should be even" %
                     hparams.num_layers)
  if (hparams.attention_architecture in ["gnmt"] and
      hparams.num_layers < 2):
    raise ValueError("For gnmt attention architecture, "
                     "num_layers %d should be >= 2" % hparams.num_layers)

  # Flags
  utils.print_out("# hparams:")
  utils.print_out("  src=%s" % hparams.src)
  utils.print_out("  tgt=%s" % hparams.tgt)
  utils.print_out("  train_prefix=%s" % hparams.train_prefix)
  utils.print_out("  dev_prefix=%s" % hparams.dev_prefix)
  utils.print_out("  test_prefix=%s" % hparams.test_prefix)
  utils.print_out("  out_dir=%s" % hparams.out_dir)

  # Set num_residual_layers
  if hparams.residual and hparams.num_layers > 1:
    if hparams.encoder_type == "gnmt":
      # The first unidirectional layer (after the bi-directional layer) in
      # the GNMT encoder can't have residual connection due to the input is
      # the concatenation of fw_cell and bw_cell's outputs.
      num_residual_layers = hparams.num_layers - 2
    else:
      num_residual_layers = hparams.num_layers - 1
  else:
    num_residual_layers = 0
  hparams.add_hparam("num_residual_layers", num_residual_layers)

  ## Vocab
  # Get vocab file names first
  if hparams.vocab_prefix:
    src_vocab_file = hparams.vocab_prefix + "." + hparams.src
    tgt_vocab_file = hparams.vocab_prefix + "." + hparams.tgt
  else:
    raise ValueError("hparams.vocab_prefix must be provided.")

  # Source vocab
  src_vocab_size, src_vocab_file = vocab_utils.check_vocab(
      src_vocab_file,
      hparams.out_dir,
      sos=hparams.sos,
      eos=hparams.eos,
      unk=vocab_utils.UNK)

  # Target vocab
  if hparams.share_vocab:
    utils.print_out("  using source vocab for target")
    tgt_vocab_file = src_vocab_file
    tgt_vocab_size = src_vocab_size
  else:
    tgt_vocab_size, tgt_vocab_file = vocab_utils.check_vocab(
        tgt_vocab_file,
        hparams.out_dir,
        sos=hparams.sos,
        eos=hparams.eos,
        unk=vocab_utils.UNK)
  hparams.add_hparam("src_vocab_size", src_vocab_size)
  hparams.add_hparam("tgt_vocab_size", tgt_vocab_size)
  hparams.add_hparam("src_vocab_file", src_vocab_file)
  hparams.add_hparam("tgt_vocab_file", tgt_vocab_file)

  # Check out_dir
  if not tf.gfile.Exists(hparams.out_dir):
    utils.print_out("# Creating output directory %s ..." % hparams.out_dir)
    tf.gfile.MakeDirs(hparams.out_dir)

  # Evaluation
  for metric in hparams.metrics:
    hparams.add_hparam("best_" + metric, 0)  # larger is better
    best_metric_dir = os.path.join(hparams.out_dir, "best_" + metric)
    hparams.add_hparam("best_" + metric + "_dir", best_metric_dir)
    tf.gfile.MakeDirs(best_metric_dir)

  return hparams


def ensure_compatible_hparams(hparams):
  """Make sure the loaded hparams is compatible with new changes."""
  new_hparams = create_hparams()
  new_hparams = utils.maybe_parse_standard_hparams(
      new_hparams, FLAGS.hparams_path)
  new_hparams = extend_hparams(new_hparams)

  # For compatible reason, if there are new fields in new_hparams,
  #   we add them to the current hparams
  new_config = new_hparams.values()
  config = hparams.values()
  for key in new_config:
    if key not in config:
      hparams.add_hparam(key, new_config[key])

  # Make sure that the loaded model has latest values for the below keys
  updated_keys = [
      "out_dir", "num_gpus", "test_prefix", "beam_width",
      "length_penalty_weight", "num_train_steps"
  ]
  for key in updated_keys:
    if key in new_config and getattr(hparams, key) != new_config[key]:
      utils.print_out("# Updating hparams.%s: %s -> %s" %
                      (key, str(getattr(hparams, key)), str(new_config[key])))
      setattr(hparams, key, new_config[key])
  return hparams<commit_msg>Remove extend hparams in ensure_compatible_hparams().<commit_after># Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""TensorFlow NMT model implementation."""
from __future__ import print_function

import argparse
import os
import random
import sys

# import matplotlib.image as mpimg
import numpy as np
import tensorflow as tf

from . import inference
from . import train
from .utils import evaluation_utils
from .utils import misc_utils as utils
from .utils import vocab_utils

utils.check_tensorflow_version()

FLAGS = None


def create_hparams():
  """Create training hparams."""
  return tf.contrib.training.HParams(
      # Data
      src=FLAGS.src,
      tgt=FLAGS.tgt,
      train_prefix=FLAGS.train_prefix,
      dev_prefix=FLAGS.dev_prefix,
      test_prefix=FLAGS.test_prefix,
      vocab_prefix=FLAGS.vocab_prefix,
      out_dir=FLAGS.out_dir,

      # Networks
      num_units=FLAGS.num_units,
      num_layers=FLAGS.num_layers,
      dropout=FLAGS.dropout,
      unit_type=FLAGS.unit_type,
      encoder_type=FLAGS.encoder_type,
      residual=FLAGS.residual,
      time_major=FLAGS.time_major,

      # Attention mechanisms
      attention=FLAGS.attention,
      attention_architecture=FLAGS.attention_architecture,
      pass_hidden_state=FLAGS.pass_hidden_state,

      # Train
      optimizer=FLAGS.optimizer,
      num_train_steps=FLAGS.num_train_steps,
      batch_size=FLAGS.batch_size,
      init_weight=FLAGS.init_weight,
      max_gradient_norm=FLAGS.max_gradient_norm,
      learning_rate=FLAGS.learning_rate,
      start_decay_step=FLAGS.start_decay_step,
      decay_factor=FLAGS.decay_factor,
      decay_steps=FLAGS.decay_steps,
      colocate_gradients_with_ops=FLAGS.colocate_gradients_with_ops,

      # Data constraints
      num_buckets=FLAGS.num_buckets,
      max_train=FLAGS.max_train,
      src_max_len=FLAGS.src_max_len,
      tgt_max_len=FLAGS.tgt_max_len,
      source_reverse=FLAGS.source_reverse,

      # Inference
      src_max_len_infer=FLAGS.src_max_len_infer,
      tgt_max_len_infer=FLAGS.tgt_max_len_infer,
      infer_batch_size=FLAGS.infer_batch_size,
      beam_width=FLAGS.beam_width,
      length_penalty_weight=FLAGS.length_penalty_weight,

      # Vocab
      sos=FLAGS.sos if FLAGS.sos else vocab_utils.SOS,
      eos=FLAGS.eos if FLAGS.eos else vocab_utils.EOS,
      bpe_delimiter=FLAGS.bpe_delimiter,

      # Misc
      forget_bias=FLAGS.forget_bias,
      num_gpus=FLAGS.num_gpus,
      epoch_step=0,  # record where we were within an epoch.
      steps_per_stats=FLAGS.steps_per_stats,
      steps_per_external_eval=FLAGS.steps_per_external_eval,
      share_vocab=FLAGS.share_vocab,
      metrics=FLAGS.metrics.split(","),
      log_device_placement=FLAGS.log_device_placement,
      random_seed=FLAGS.random_seed,
  )


def extend_hparams(hparams):
  """Extend training hparams."""
  # Sanity checks
  if hparams.encoder_type == "bi" and hparams.num_layers % 2 != 0:
    raise ValueError("For bi, num_layers %d should be even" %
                     hparams.num_layers)
  if (hparams.attention_architecture in ["gnmt"] and
      hparams.num_layers < 2):
    raise ValueError("For gnmt attention architecture, "
                     "num_layers %d should be >= 2" % hparams.num_layers)

  # Flags
  utils.print_out("# hparams:")
  utils.print_out("  src=%s" % hparams.src)
  utils.print_out("  tgt=%s" % hparams.tgt)
  utils.print_out("  train_prefix=%s" % hparams.train_prefix)
  utils.print_out("  dev_prefix=%s" % hparams.dev_prefix)
  utils.print_out("  test_prefix=%s" % hparams.test_prefix)
  utils.print_out("  out_dir=%s" % hparams.out_dir)

  # Set num_residual_layers
  if hparams.residual and hparams.num_layers > 1:
    if hparams.encoder_type == "gnmt":
      # The first unidirectional layer (after the bi-directional layer) in
      # the GNMT encoder can't have residual connection due to the input is
      # the concatenation of fw_cell and bw_cell's outputs.
      num_residual_layers = hparams.num_layers - 2
    else:
      num_residual_layers = hparams.num_layers - 1
  else:
    num_residual_layers = 0
  hparams.add_hparam("num_residual_layers", num_residual_layers)

  ## Vocab
  # Get vocab file names first
  if hparams.vocab_prefix:
    src_vocab_file = hparams.vocab_prefix + "." + hparams.src
    tgt_vocab_file = hparams.vocab_prefix + "." + hparams.tgt
  else:
    raise ValueError("hparams.vocab_prefix must be provided.")

  # Source vocab
  src_vocab_size, src_vocab_file = vocab_utils.check_vocab(
      src_vocab_file,
      hparams.out_dir,
      sos=hparams.sos,
      eos=hparams.eos,
      unk=vocab_utils.UNK)

  # Target vocab
  if hparams.share_vocab:
    utils.print_out("  using source vocab for target")
    tgt_vocab_file = src_vocab_file
    tgt_vocab_size = src_vocab_size
  else:
    tgt_vocab_size, tgt_vocab_file = vocab_utils.check_vocab(
        tgt_vocab_file,
        hparams.out_dir,
        sos=hparams.sos,
        eos=hparams.eos,
        unk=vocab_utils.UNK)
  hparams.add_hparam("src_vocab_size", src_vocab_size)
  hparams.add_hparam("tgt_vocab_size", tgt_vocab_size)
  hparams.add_hparam("src_vocab_file", src_vocab_file)
  hparams.add_hparam("tgt_vocab_file", tgt_vocab_file)

  # Check out_dir
  if not tf.gfile.Exists(hparams.out_dir):
    utils.print_out("# Creating output directory %s ..." % hparams.out_dir)
    tf.gfile.MakeDirs(hparams.out_dir)

  # Evaluation
  for metric in hparams.metrics:
    hparams.add_hparam("best_" + metric, 0)  # larger is better
    best_metric_dir = os.path.join(hparams.out_dir, "best_" + metric)
    hparams.add_hparam("best_" + metric + "_dir", best_metric_dir)
    tf.gfile.MakeDirs(best_metric_dir)

  return hparams


def ensure_compatible_hparams(hparams):
  """Make sure the loaded hparams is compatible with new changes."""
  new_hparams = create_hparams()
  new_hparams = utils.maybe_parse_standard_hparams(
      new_hparams, FLAGS.hparams_path)

  # For compatible reason, if there are new fields in new_hparams,
  #   we add them to the current hparams
  new_config = new_hparams.values()
  config = hparams.values()
  for key in new_config:
    if key not in config:
      hparams.add_hparam(key, new_config[key])

  # Make sure that the loaded model has latest values for the below keys
  updated_keys = [
      "out_dir", "num_gpus", "test_prefix", "beam_width",
      "length_penalty_weight", "num_train_steps"
  ]
  for key in updated_keys:
    if key in new_config and getattr(hparams, key) != new_config[key]:
      utils.print_out("# Updating hparams.%s: %s -> %s" %
                      (key, str(getattr(hparams, key)), str(new_config[key])))
      setattr(hparams, key, new_config[key])
  return hparams
{'fraction_non_alphanumeric': 0.09935853691824757, 'fraction_numerical': 0.0024566671216050225, 'mean_word_length': 3.713734319716951, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1239116', 'n_tokens_mistral': 5129, 'n_tokens_neox': 4960, 'n_words': 1047}