# Path: utils/io_utils.py
# _PAD = b"_PAD"      # Append to unused space for both encoder/decoder.
# _GO = b"_GO"       # Prepend to each decoder input.
# _EOS = b"_EOS"      # Append to outputs only. Stopping signal when decoding.
# _UNK = b"_UNK"      # For any symbols not in our vocabulary.
# _START_VOCAB = [_PAD, _GO, _EOS, _UNK]
# PAD_ID = 0
# GO_ID = 1
# EOS_ID = 2
# UNK_ID = 3
# _WORD_SPLIT = re.compile(b"([.,!?\"':;)(])")
# _DIGIT_RE = re.compile(br"\d")
# _FLAGS = Flags(pretrained_dir=None,
#                config=None,
#                debug=None,
#                model='{}',
#                dataset='{}',
#                model_params='{}',
#                dataset_params='{}')
# def save_hyper_params(hyper_params, fname):
# def get_sentence(lower=True):
# def update_config(config=None,
#                   config_path=None,
#                   return_config=True,
#                   **kwargs):
# def get_yaml_config(path, save_path=True):
# def load_pretrained_config(pretrained_dir):
# def print_non_defaults(config):
# def flags_to_dict(flags):
# def merge_dicts(default_dict, preference_dict):
# def parse_config(flags=None, pretrained_dir=None, config_path=None):
# def basic_tokenizer(sentence):
# def num_lines(file_path):
# def get_word_freqs(path, counter, norm_digits=True):
# def create_vocabulary(vocab_path, from_path, to_path, max_vocab_size, norm_digits=True):
# def get_vocab_dicts(vocabulary_path):
# def sentence_to_token_ids(sentence, vocabulary, normalize_digits=True):
# def data_to_token_ids(data_path, target_path, vocabulary_path, normalize_digits=True):
# def prepare_data(data_dir,
#                  vocab_size,
#                  from_train_path=None,
#                  to_train_path=None,
#                  from_valid_path=None,
#                  to_valid_path=None,
#                  optimize=True,
#                  config_path=None):
#     def maybe_set_param(param, file_name):
#     def get_vocab_path(vocab_size):
#     def append_to_paths(s, **paths):


import os
import pydoc
import yaml
import logging
import unittest
import tensorflow as tf
import chatbot
import data
from utils import io_utils
from tests.utils import *

        # Manually set vocab size to huge (non-optimal for TestData) value.
        config = io_utils.update_config(config=config, vocab_size=99999)
        self.assertEqual(config['dataset_params']['vocab_size'], 99999)
        self.assertEqual(config['dataset_params']['config_path'], TEST_CONFIG_PATH)

        # Instantiate a new dataset.
        # This where the 'optimize' flag comes into play, since
        # the dataset object is responsible for things like checking
        # data file paths and unique words.
        logging.info("Setting up %s dataset.", config['dataset'])
        logging.info("Passing %r for dataset_params", config['dataset_params'])
        dataset_class = pydoc.locate(config['dataset']) \
                        or getattr(data, config['dataset'])
        dataset = dataset_class(config['dataset_params'])
        self.assertIsInstance(dataset, data.TestData)
        self.assertNotEqual(dataset.vocab_size, 99999)

    def test_update_config(self):
        """Test the new function in io_utils.py"""

        logging.info(os.getcwd())
        config = io_utils.get_yaml_config(TEST_CONFIG_PATH)
        config['model_params']['ckpt_dir'] = TEST_FLAGS.model_params['ckpt_dir']
        self.assertIsInstance(config, dict)
        self.assertTrue('model' in config)
        self.assertTrue('dataset' in config)
        self.assertTrue('dataset_params' in config)
        self.assertTrue('model_params' in config)


# Path: utils/io_utils.py
# _PAD = b"_PAD"      # Append to unused space for both encoder/decoder.
# _GO = b"_GO"       # Prepend to each decoder input.
# _EOS = b"_EOS"      # Append to outputs only. Stopping signal when decoding.
# _UNK = b"_UNK"      # For any symbols not in our vocabulary.
# _START_VOCAB = [_PAD, _GO, _EOS, _UNK]
# PAD_ID = 0
# GO_ID = 1
# EOS_ID = 2
# UNK_ID = 3
# _WORD_SPLIT = re.compile(b"([.,!?\"':;)(])")
# _DIGIT_RE = re.compile(br"\d")
# _FLAGS = Flags(pretrained_dir=None,
#                config=None,
#                debug=None,
#                model='{}',
#                dataset='{}',
#                model_params='{}',
#                dataset_params='{}')
# def save_hyper_params(hyper_params, fname):
# def get_sentence(lower=True):
# def update_config(config=None,
#                   config_path=None,
#                   return_config=True,
#                   **kwargs):
# def get_yaml_config(path, save_path=True):
# def load_pretrained_config(pretrained_dir):
# def print_non_defaults(config):
# def flags_to_dict(flags):
# def merge_dicts(default_dict, preference_dict):
# def parse_config(flags=None, pretrained_dir=None, config_path=None):
# def basic_tokenizer(sentence):
# def num_lines(file_path):
# def get_word_freqs(path, counter, norm_digits=True):
# def create_vocabulary(vocab_path, from_path, to_path, max_vocab_size, norm_digits=True):
# def get_vocab_dicts(vocabulary_path):
# def sentence_to_token_ids(sentence, vocabulary, normalize_digits=True):
# def data_to_token_ids(data_path, target_path, vocabulary_path, normalize_digits=True):
# def prepare_data(data_dir,
#                  vocab_size,
#                  from_train_path=None,
#                  to_train_path=None,
#                  from_valid_path=None,
#                  to_valid_path=None,
#                  optimize=True,
#                  config_path=None):
#     def maybe_set_param(param, file_name):
#     def get_vocab_path(vocab_size):
#     def append_to_paths(s, **paths):


# Path: tests/test_config.py
import os
import pydoc
import yaml
import logging
import unittest
import tensorflow as tf
import chatbot
import data
from utils import io_utils
from tests.utils import *


        # Manually set vocab size to huge (non-optimal for TestData) value.
        config = io_utils.update_config(config=config, vocab_size=99999)
        self.assertEqual(config['dataset_params']['vocab_size'], 99999)
        self.assertEqual(config['dataset_params']['config_path'], TEST_CONFIG_PATH)

        # Instantiate a new dataset.
        # This where the 'optimize' flag comes into play, since
        # the dataset object is responsible for things like checking
        # data file paths and unique words.
        logging.info("Setting up %s dataset.", config['dataset'])
        logging.info("Passing %r for dataset_params", config['dataset_params'])
        dataset_class = pydoc.locate(config['dataset']) \
                        or getattr(data, config['dataset'])
        dataset = dataset_class(config['dataset_params'])
        self.assertIsInstance(dataset, data.TestData)
        self.assertNotEqual(dataset.vocab_size, 99999)

    def test_update_config(self):
        """Test the new function in io_utils.py"""

        logging.info(os.getcwd())
        config = io_utils.get_yaml_config(TEST_CONFIG_PATH)
        config['model_params']['ckpt_dir'] = TEST_FLAGS.model_params['ckpt_dir']
        self.assertIsInstance(config, dict)
        self.assertTrue('model' in config)
        self.assertTrue('dataset' in config)
        self.assertTrue('dataset_params' in config)
        self.assertTrue('model_params' in config)


        config = io_utils.update_config(