# Path: utils/io_utils.py
# _PAD = b"_PAD"      # Append to unused space for both encoder/decoder.
# _GO = b"_GO"       # Prepend to each decoder input.
# _EOS = b"_EOS"      # Append to outputs only. Stopping signal when decoding.
# _UNK = b"_UNK"      # For any symbols not in our vocabulary.
# _START_VOCAB = [_PAD, _GO, _EOS, _UNK]
# PAD_ID = 0
# GO_ID = 1
# EOS_ID = 2
# UNK_ID = 3
# _WORD_SPLIT = re.compile(b"([.,!?\"':;)(])")
# _DIGIT_RE = re.compile(br"\d")
# _FLAGS = Flags(pretrained_dir=None,
#                config=None,
#                debug=None,
#                model='{}',
#                dataset='{}',
#                model_params='{}',
#                dataset_params='{}')
# def save_hyper_params(hyper_params, fname):
# def get_sentence(lower=True):
# def update_config(config=None,
#                   config_path=None,
#                   return_config=True,
#                   **kwargs):
# def get_yaml_config(path, save_path=True):
# def load_pretrained_config(pretrained_dir):
# def print_non_defaults(config):
# def flags_to_dict(flags):
# def merge_dicts(default_dict, preference_dict):
# def parse_config(flags=None, pretrained_dir=None, config_path=None):
# def basic_tokenizer(sentence):
# def num_lines(file_path):
# def get_word_freqs(path, counter, norm_digits=True):
# def create_vocabulary(vocab_path, from_path, to_path, max_vocab_size, norm_digits=True):
# def get_vocab_dicts(vocabulary_path):
# def sentence_to_token_ids(sentence, vocabulary, normalize_digits=True):
# def data_to_token_ids(data_path, target_path, vocabulary_path, normalize_digits=True):
# def prepare_data(data_dir,
#                  vocab_size,
#                  from_train_path=None,
#                  to_train_path=None,
#                  from_valid_path=None,
#                  to_valid_path=None,
#                  optimize=True,
#                  config_path=None):
#     def maybe_set_param(param, file_name):
#     def get_vocab_path(vocab_size):
#     def append_to_paths(s, **paths):


import logging
import tensorflow as tf
from utils import io_utils
from tensorflow.contrib.training import bucket_by_sequence_length
              space-efficient manner.

        Args:
            name: filename prefix for data. See Dataset class for naming conventions.

        Returns:
            2-tuple (lengths, sequences):
                lengths: (dict) parsed context feature from protobuf file.
                Supports keys in LENGTHS.
                sequences: (dict) parsed feature_list from protobuf file.
                Supports keys in SEQUENCES.
        """
        with tf.variable_scope(name + '_pipeline'):
            proto_text = self._read_line(self.paths[name + '_tfrecords'])
            context_pair, sequence_pair = self._assign_queue(proto_text)
            input_length = tf.add(context_pair['encoder_sequence_length'],
                                  context_pair['decoder_sequence_length'],
                                  name=name + 'length_add')
            return self._padded_bucket_batches(input_length, sequence_pair)

    @property
    def encoder_inputs(self):
        """Determines, via tensorflow control structures, which part of the pipeline to run
           and retrieve inputs to a Model encoder component. """
        if not self.is_chatting:
            return self._cond_input('encoder')
        else:
            return self._user_input

    @property

# Path: utils/io_utils.py
# _PAD = b"_PAD"      # Append to unused space for both encoder/decoder.
# _GO = b"_GO"       # Prepend to each decoder input.
# _EOS = b"_EOS"      # Append to outputs only. Stopping signal when decoding.
# _UNK = b"_UNK"      # For any symbols not in our vocabulary.
# _START_VOCAB = [_PAD, _GO, _EOS, _UNK]
# PAD_ID = 0
# GO_ID = 1
# EOS_ID = 2
# UNK_ID = 3
# _WORD_SPLIT = re.compile(b"([.,!?\"':;)(])")
# _DIGIT_RE = re.compile(br"\d")
# _FLAGS = Flags(pretrained_dir=None,
#                config=None,
#                debug=None,
#                model='{}',
#                dataset='{}',
#                model_params='{}',
#                dataset_params='{}')
# def save_hyper_params(hyper_params, fname):
# def get_sentence(lower=True):
# def update_config(config=None,
#                   config_path=None,
#                   return_config=True,
#                   **kwargs):
# def get_yaml_config(path, save_path=True):
# def load_pretrained_config(pretrained_dir):
# def print_non_defaults(config):
# def flags_to_dict(flags):
# def merge_dicts(default_dict, preference_dict):
# def parse_config(flags=None, pretrained_dir=None, config_path=None):
# def basic_tokenizer(sentence):
# def num_lines(file_path):
# def get_word_freqs(path, counter, norm_digits=True):
# def create_vocabulary(vocab_path, from_path, to_path, max_vocab_size, norm_digits=True):
# def get_vocab_dicts(vocabulary_path):
# def sentence_to_token_ids(sentence, vocabulary, normalize_digits=True):
# def data_to_token_ids(data_path, target_path, vocabulary_path, normalize_digits=True):
# def prepare_data(data_dir,
#                  vocab_size,
#                  from_train_path=None,
#                  to_train_path=None,
#                  from_valid_path=None,
#                  to_valid_path=None,
#                  optimize=True,
#                  config_path=None):
#     def maybe_set_param(param, file_name):
#     def get_vocab_path(vocab_size):
#     def append_to_paths(s, **paths):


# Path: chatbot/components/input_pipeline.py
import logging
import tensorflow as tf
from utils import io_utils
from tensorflow.contrib.training import bucket_by_sequence_length

              space-efficient manner.

        Args:
            name: filename prefix for data. See Dataset class for naming conventions.

        Returns:
            2-tuple (lengths, sequences):
                lengths: (dict) parsed context feature from protobuf file.
                Supports keys in LENGTHS.
                sequences: (dict) parsed feature_list from protobuf file.
                Supports keys in SEQUENCES.
        """
        with tf.variable_scope(name + '_pipeline'):
            proto_text = self._read_line(self.paths[name + '_tfrecords'])
            context_pair, sequence_pair = self._assign_queue(proto_text)
            input_length = tf.add(context_pair['encoder_sequence_length'],
                                  context_pair['decoder_sequence_length'],
                                  name=name + 'length_add')
            return self._padded_bucket_batches(input_length, sequence_pair)

    @property
    def encoder_inputs(self):
        """Determines, via tensorflow control structures, which part of the pipeline to run
           and retrieve inputs to a Model encoder component. """
        if not self.is_chatting:
            return self._cond_input('encoder')
        else:
            return self._user_input

    @property

    def decoder_inputs(self):