<commit_before># -*- coding: utf-8 -*-
# Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Miscellaneous training utility functions.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import inspect
import os
from collections import defaultdict
from pydoc import locate

import json

import tensorflow as tf
from tensorflow import gfile

from seq2seq.contrib import rnn_cell


class TrainOptions(object):
  """A collectionf of options that are passed to the training script
  and can be saved to perform inference later.

  Args:
    task: Name of the training task class.
    task_params: A dictionary of parameters passed to the training task.
  """

  def __init__(self, model_class, model_params):
    self._model_class = model_class
    self._model_params = model_params

  @property
  def model_class(self):
    """Returns the training task parameters"""
    return self._model_class

  @property
  def model_params(self):
    """Returns the training task class"""
    return self._model_params

  @staticmethod
  def path(model_dir):
    """Returns the path to the options file.

    Args:
      model_dir: The model directory
    """
    return os.path.join(model_dir, "train_options.json")

  def dump(self, model_dir):
    """Dumps the options to a file in the model directory.

    Args:
      model_dir: Path to the model directory. The options will be
      dumped into a file in this directory.
    """
    gfile.MakeDirs(model_dir)
    options_dict = {
        "model_class": self.model_class,
        "model_params": self.model_params,
    }

    with gfile.GFile(TrainOptions.path(model_dir), "w") as file:
      file.write(json.dumps(options_dict).encode("utf-8"))

  @staticmethod
  def load(model_dir):
    """ Loads options from the given model directory.

    Args:
      model_dir: Path to the model directory.
    """
    with gfile.GFile(TrainOptions.path(model_dir), "r") as file:
      options_dict = json.loads(file.read().decode("utf-8"))
    options_dict = defaultdict(None, options_dict)

    return TrainOptions(
        model_class=options_dict["model_class"],
        model_params=options_dict["model_params"])


def cell_from_spec(cell_classname, cell_params):
  """Create a RNN Cell instance from a JSON string.

  Args:
    cell_classname: Name of the cell class, e.g. "BasicLSTMCell".
    cell_params: A dictionary of parameters to pass to the cell constructor.
<commit_msg>Read as unicode<commit_after># -*- coding: utf-8 -*-
# Copyright 2017 Google Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Miscellaneous training utility functions.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import inspect
import os
from collections import defaultdict
from pydoc import locate

import json

import tensorflow as tf
from tensorflow import gfile

from seq2seq.contrib import rnn_cell


class TrainOptions(object):
  """A collectionf of options that are passed to the training script
  and can be saved to perform inference later.

  Args:
    task: Name of the training task class.
    task_params: A dictionary of parameters passed to the training task.
  """

  def __init__(self, model_class, model_params):
    self._model_class = model_class
    self._model_params = model_params

  @property
  def model_class(self):
    """Returns the training task parameters"""
    return self._model_class

  @property
  def model_params(self):
    """Returns the training task class"""
    return self._model_params

  @staticmethod
  def path(model_dir):
    """Returns the path to the options file.

    Args:
      model_dir: The model directory
    """
    return os.path.join(model_dir, "train_options.json")

  def dump(self, model_dir):
    """Dumps the options to a file in the model directory.

    Args:
      model_dir: Path to the model directory. The options will be
      dumped into a file in this directory.
    """
    gfile.MakeDirs(model_dir)
    options_dict = {
        "model_class": self.model_class,
        "model_params": self.model_params,
    }

    with gfile.GFile(TrainOptions.path(model_dir), "wb") as file:
      file.write(json.dumps(options_dict).encode("utf-8"))

  @staticmethod
  def load(model_dir):
    """ Loads options from the given model directory.

    Args:
      model_dir: Path to the model directory.
    """
    with gfile.GFile(TrainOptions.path(model_dir)) as file:
      options_dict = json.loads(file.read().decode("utf-8"))
    options_dict = defaultdict(None, options_dict)

    return TrainOptions(
        model_class=options_dict["model_class"],
        model_params=options_dict["model_params"])


def cell_from_spec(cell_classname, cell_params):
  """Create a RNN Cell instance from a JSON string.

  Args:
    cell_classname: Name of the cell class, e.g. "BasicLSTMCell".
    cell_params: A dictionary of parameters to pass to the cell constructor.

{'fraction_non_alphanumeric': 0.08166283273085771, 'fraction_numerical': 0.003943476832073612, 'mean_word_length': 3.7332814930015554, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3069271', 'n_tokens_mistral': 1749, 'n_tokens_neox': 1699, 'n_words': 645}