# Path: utils/io_utils.py
# _PAD = b"_PAD"      # Append to unused space for both encoder/decoder.
# _GO = b"_GO"       # Prepend to each decoder input.
# _EOS = b"_EOS"      # Append to outputs only. Stopping signal when decoding.
# _UNK = b"_UNK"      # For any symbols not in our vocabulary.
# _START_VOCAB = [_PAD, _GO, _EOS, _UNK]
# PAD_ID = 0
# GO_ID = 1
# EOS_ID = 2
# UNK_ID = 3
# _WORD_SPLIT = re.compile(b"([.,!?\"':;)(])")
# _DIGIT_RE = re.compile(br"\d")
# _FLAGS = Flags(pretrained_dir=None,
#                config=None,
#                debug=None,
#                model='{}',
#                dataset='{}',
#                model_params='{}',
#                dataset_params='{}')
# def save_hyper_params(hyper_params, fname):
# def get_sentence(lower=True):
# def update_config(config=None,
#                   config_path=None,
#                   return_config=True,
#                   **kwargs):
# def get_yaml_config(path, save_path=True):
# def load_pretrained_config(pretrained_dir):
# def print_non_defaults(config):
# def flags_to_dict(flags):
# def merge_dicts(default_dict, preference_dict):
# def parse_config(flags=None, pretrained_dir=None, config_path=None):
# def basic_tokenizer(sentence):
# def num_lines(file_path):
# def get_word_freqs(path, counter, norm_digits=True):
# def create_vocabulary(vocab_path, from_path, to_path, max_vocab_size, norm_digits=True):
# def get_vocab_dicts(vocabulary_path):
# def sentence_to_token_ids(sentence, vocabulary, normalize_digits=True):
# def data_to_token_ids(data_path, target_path, vocabulary_path, normalize_digits=True):
# def prepare_data(data_dir,
#                  vocab_size,
#                  from_train_path=None,
#                  to_train_path=None,
#                  from_valid_path=None,
#                  to_valid_path=None,
#                  optimize=True,
#                  config_path=None):
#     def maybe_set_param(param, file_name):
#     def get_vocab_path(vocab_size):
#     def append_to_paths(s, **paths):
# 
# Path: chatbot/globals.py
# DEFAULT_FULL_CONFIG = {
#     "model": "DynamicBot",
#     "dataset": "Cornell",
#     "model_params": {
#         "base_cell": "GRUCell",
#         "ckpt_dir": "out",  # Directory to store training checkpoints.
#         "decode": False,
#         "batch_size": 256,
#         "dropout_prob": 0.2,  # Drop rate applied at encoder/decoders output.
#         "decoder.class": "BasicDecoder",
#         "encoder.class": "BasicEncoder",
#         "embed_size": 128,
#         "learning_rate": 0.002,
#         "l1_reg": 1.0e-6,  # L1 regularization applied to word embeddings.
#         "lr_decay": 0.98,
#         "max_gradient": 5.0,
#         "max_steps": int(1e6),  # Max number of training iterations.
#         "num_layers": 1,  # Num layers for each of encoder, decoder.
#         "num_samples": 512,  # IF sampled_loss is true, default sample size.
#         "optimizer": "Adam",  # Options are those in OPTIMIZERS above.
#         "reset_model": True,
#         "sampled_loss": False,  # Whether to do sampled softmax.
#         "state_size": 512,
#         "steps_per_ckpt": 200,
#         "temperature": 0.0,  # Response temp for chat sessions. (default argmax)
#     },
#     "dataset_params": {
#         "data_dir": None,  # Require user to specify.
#         "vocab_size": 40000,
#         "max_seq_len": 10,  # Maximum length of sentence used to train bot.
#         "optimize_params": True  # Reduce vocab size if exceeds num unique words
#     },
# }


import os
import logging
import numpy as np
import tensorflow as tf
from utils import io_utils
from abc import ABCMeta, abstractmethod, abstractproperty
from chatbot.globals import DEFAULT_FULL_CONFIG
        return self._generator(
            self.paths['from_train'],
            self.paths['to_train'],
            batch_size)

    def valid_generator(self, batch_size):
        """[Note: not needed by DynamicBot since InputPipeline]"""
        return self._generator(
            self.paths['from_valid'],
            self.paths['to_valid'],
            batch_size)

    def _generator(self, from_path, to_path, batch_size):
        """(Used by BucketModels only). Returns a generator function that 
        reads data from file, and yields shuffled batches.

        Args:
            from_path: full path to file for encoder inputs.
            to_path: full path to file for decoder inputs.
            batch_size: number of samples to yield at once.
        """

        def longest_sentence(enc_list, dec_list):
            max_enc_len = max([len(s) for s in enc_list])
            max_dec_len = max([len(s) for s in dec_list])
            return max(max_enc_len, max_dec_len)

        def padded_batch(encoder_tokens, decoder_tokens):
            max_sent_len = longest_sentence(encoder_tokens, decoder_tokens)
            encoder_batch = np.array(

# Path: utils/io_utils.py
# _PAD = b"_PAD"      # Append to unused space for both encoder/decoder.
# _GO = b"_GO"       # Prepend to each decoder input.
# _EOS = b"_EOS"      # Append to outputs only. Stopping signal when decoding.
# _UNK = b"_UNK"      # For any symbols not in our vocabulary.
# _START_VOCAB = [_PAD, _GO, _EOS, _UNK]
# PAD_ID = 0
# GO_ID = 1
# EOS_ID = 2
# UNK_ID = 3
# _WORD_SPLIT = re.compile(b"([.,!?\"':;)(])")
# _DIGIT_RE = re.compile(br"\d")
# _FLAGS = Flags(pretrained_dir=None,
#                config=None,
#                debug=None,
#                model='{}',
#                dataset='{}',
#                model_params='{}',
#                dataset_params='{}')
# def save_hyper_params(hyper_params, fname):
# def get_sentence(lower=True):
# def update_config(config=None,
#                   config_path=None,
#                   return_config=True,
#                   **kwargs):
# def get_yaml_config(path, save_path=True):
# def load_pretrained_config(pretrained_dir):
# def print_non_defaults(config):
# def flags_to_dict(flags):
# def merge_dicts(default_dict, preference_dict):
# def parse_config(flags=None, pretrained_dir=None, config_path=None):
# def basic_tokenizer(sentence):
# def num_lines(file_path):
# def get_word_freqs(path, counter, norm_digits=True):
# def create_vocabulary(vocab_path, from_path, to_path, max_vocab_size, norm_digits=True):
# def get_vocab_dicts(vocabulary_path):
# def sentence_to_token_ids(sentence, vocabulary, normalize_digits=True):
# def data_to_token_ids(data_path, target_path, vocabulary_path, normalize_digits=True):
# def prepare_data(data_dir,
#                  vocab_size,
#                  from_train_path=None,
#                  to_train_path=None,
#                  from_valid_path=None,
#                  to_valid_path=None,
#                  optimize=True,
#                  config_path=None):
#     def maybe_set_param(param, file_name):
#     def get_vocab_path(vocab_size):
#     def append_to_paths(s, **paths):
# 
# Path: chatbot/globals.py
# DEFAULT_FULL_CONFIG = {
#     "model": "DynamicBot",
#     "dataset": "Cornell",
#     "model_params": {
#         "base_cell": "GRUCell",
#         "ckpt_dir": "out",  # Directory to store training checkpoints.
#         "decode": False,
#         "batch_size": 256,
#         "dropout_prob": 0.2,  # Drop rate applied at encoder/decoders output.
#         "decoder.class": "BasicDecoder",
#         "encoder.class": "BasicEncoder",
#         "embed_size": 128,
#         "learning_rate": 0.002,
#         "l1_reg": 1.0e-6,  # L1 regularization applied to word embeddings.
#         "lr_decay": 0.98,
#         "max_gradient": 5.0,
#         "max_steps": int(1e6),  # Max number of training iterations.
#         "num_layers": 1,  # Num layers for each of encoder, decoder.
#         "num_samples": 512,  # IF sampled_loss is true, default sample size.
#         "optimizer": "Adam",  # Options are those in OPTIMIZERS above.
#         "reset_model": True,
#         "sampled_loss": False,  # Whether to do sampled softmax.
#         "state_size": 512,
#         "steps_per_ckpt": 200,
#         "temperature": 0.0,  # Response temp for chat sessions. (default argmax)
#     },
#     "dataset_params": {
#         "data_dir": None,  # Require user to specify.
#         "vocab_size": 40000,
#         "max_seq_len": 10,  # Maximum length of sentence used to train bot.
#         "optimize_params": True  # Reduce vocab size if exceeds num unique words
#     },
# }


# Path: data/_dataset.py
import os
import logging
import numpy as np
import tensorflow as tf
from utils import io_utils
from abc import ABCMeta, abstractmethod, abstractproperty
from chatbot.globals import DEFAULT_FULL_CONFIG

        return self._generator(
            self.paths['from_train'],
            self.paths['to_train'],
            batch_size)

    def valid_generator(self, batch_size):
        """[Note: not needed by DynamicBot since InputPipeline]"""
        return self._generator(
            self.paths['from_valid'],
            self.paths['to_valid'],
            batch_size)

    def _generator(self, from_path, to_path, batch_size):
        """(Used by BucketModels only). Returns a generator function that 
        reads data from file, and yields shuffled batches.

        Args:
            from_path: full path to file for encoder inputs.
            to_path: full path to file for decoder inputs.
            batch_size: number of samples to yield at once.
        """

        def longest_sentence(enc_list, dec_list):
            max_enc_len = max([len(s) for s in enc_list])
            max_dec_len = max([len(s) for s in dec_list])
            return max(max_enc_len, max_dec_len)

        def padded_batch(encoder_tokens, decoder_tokens):
            max_sent_len = longest_sentence(encoder_tokens, decoder_tokens)
            encoder_batch = np.array(

                [s + [io_utils.PAD_ID] * (max_sent_len - len(s))