# Path: nmt_chainer/__main__.py
# def main(arguments=None):
#     # create the top-level parser
#     parser = argparse.ArgumentParser(description="Kyoto-NMT: an Implementation of the RNNSearch model",
#                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter,
#                                      prog="knmt")
# 
#     parser.add_argument("--run_in_pdb", default=False, action="store_true", help="run knmt in pdb (python debugger)")
# 
#     parser.add_argument("--run_in_cprofile", default=False, action="store_true", help="run knmt in cProfile")
# 
#     subparsers = parser.add_subparsers(dest="__subcommand_name")
# 
#     # create the parser for the "make_data" command
#     parser_make_data = subparsers.add_parser('make_data', description="Prepare data for training.", help="Prepare data for training", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
#     make_data.define_parser(parser_make_data)
# 
#     # create the parser for the "train" command
#     parser_train = subparsers.add_parser('train', description="Train a model.", help="Train a model", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
#     train.define_parser(parser_train)
# 
#     # create the parser for the "eval" command
#     parser_eval = subparsers.add_parser('eval', description="Use a model.", help="Use a model", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
#     eval_module.define_parser(parser_eval)
# 
#     # create the parser for the "version" command
#     parser_version = subparsers.add_parser('version', description="Get version infos.", help="Get version infos", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# 
#     parser_utils = subparsers.add_parser('utils', description="Call a utility script.", help="Call a utility script", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
#     utils_command.define_parser(parser_utils)
# 
# #     import argcomplete
#     if argcomplete is not None:
#         argcomplete.autocomplete(parser)
# 
#     args = parser.parse_args(args=arguments)
#     if arguments is not None:
#         args.__original_argument_list = arguments
#     else:
#         args.__original_argument_list = sys.argv
# 
#     if args.__subcommand_name is None:
#         parser.print_help()
#         sys.exit(1)
# 
#     func = {"make_data": make_data.do_make_data,
#             "train": train.do_train,
#             "eval": eval_module.do_eval,
#             "version": versioning_tools.main,
#             "utils": utils_command.do_utils}[args.__subcommand_name]
# 
#     if args.run_in_pdb:
#         try:
#             import ipdb
#             run_in_ipdb(func, args)
#         except ImportError:
#             run_in_pdb(func, args)
#         # import pdb
#         # pdb.runcall(func, args)
#         # def debug_signal_handler(signal, frame):
#         #     import pdb
#         #     pdb.set_trace()
#         # import signal
#         # signal.signal(signal.SIGINT, debug_signal_handler)
# 
#     elif args.run_in_cprofile:
#         import cProfile
#         cProfile.runctx("func(args)", globals(), locals(), sort="cumtime")
#     
#     else:
#         try:
#             func(args)
#         except train.CommandLineValuesException as e:
#             parser_train.error(e.args[0])
#         except eval_module.CommandLineValuesException as e:
#             parser_eval.error(e.args[0])
#         except make_data.CommandLineValuesException as e:
#             parser_make_data.error(e.args[0])
# 
# Path: nmt_chainer/utilities/utils.py
# def de_batch(batch, mask=None, eos_idx=None, is_variable=False, raw=False):
#     """ Utility function for "de-batching".
#         batch is a list of Variable/numpy/cupy of shape[0] <= mb_size
# 
#         returns a list of the sequences in the batch
#     """
#     res = []
#     mb_size = len(batch[0].data) if is_variable else len(batch[0])
#     if mask is not None:
#         mask_offset = len(batch) - len(mask)
#         assert mask_offset >= 0
#     for sent_num in six.moves.range(mb_size):
#         assert sent_num == len(res)
#         res.append([])
#         for src_pos in six.moves.range(len(batch)):
#             current_batch_size = batch[src_pos].data.shape[0] if is_variable else batch[src_pos].shape[0]
#             if (mask is None or
#                     (src_pos < mask_offset or mask[src_pos - mask_offset][sent_num])) and current_batch_size > sent_num:
#                 #                 print(sent_num, src_pos, batch[src_pos].data)
#                 idx = batch[src_pos].data[sent_num] if is_variable else batch[src_pos][sent_num]
#                 if not raw:
#                     idx = int(idx)
#                 res[sent_num].append(None)
#                 res[sent_num][src_pos] = idx
#                 if eos_idx is not None and idx == eos_idx:
#                     break
#     return res


import numpy as np
import chainer
import chainer.functions as F
import chainer.links as L
import nmt_chainer.models as models
import nmt_chainer.utilities.utils as utils
import nmt_chainer.dataprocessing.make_data as make_data
import nmt_chainer.training_module.train as train
import os.path
from chainer import Link, Chain, ChainList, Variable
from nmt_chainer.__main__ import main
from nmt_chainer.utilities.utils import de_batch
#!/usr/bin/env python
"""utils_tests.py: Some correctness tests"""
from __future__ import absolute_import, division, print_function, unicode_literals
__author__ = "Fabien Cromieres"
__license__ = "undecided"
__version__ = "1.0"
__email__ = "fabien.cromieres@gmail.com"
__status__ = "Development"






class TestDeBatch:
    def test_multiple_length(self):
        batch = [np.array([1, 3, 4, 8]), np.array([1, 5, 6, 9]), np.array([7, 5]), np.array([10])]

# Path: nmt_chainer/__main__.py
# def main(arguments=None):
#     # create the top-level parser
#     parser = argparse.ArgumentParser(description="Kyoto-NMT: an Implementation of the RNNSearch model",
#                                      formatter_class=argparse.ArgumentDefaultsHelpFormatter,
#                                      prog="knmt")
# 
#     parser.add_argument("--run_in_pdb", default=False, action="store_true", help="run knmt in pdb (python debugger)")
# 
#     parser.add_argument("--run_in_cprofile", default=False, action="store_true", help="run knmt in cProfile")
# 
#     subparsers = parser.add_subparsers(dest="__subcommand_name")
# 
#     # create the parser for the "make_data" command
#     parser_make_data = subparsers.add_parser('make_data', description="Prepare data for training.", help="Prepare data for training", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
#     make_data.define_parser(parser_make_data)
# 
#     # create the parser for the "train" command
#     parser_train = subparsers.add_parser('train', description="Train a model.", help="Train a model", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
#     train.define_parser(parser_train)
# 
#     # create the parser for the "eval" command
#     parser_eval = subparsers.add_parser('eval', description="Use a model.", help="Use a model", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
#     eval_module.define_parser(parser_eval)
# 
#     # create the parser for the "version" command
#     parser_version = subparsers.add_parser('version', description="Get version infos.", help="Get version infos", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# 
#     parser_utils = subparsers.add_parser('utils', description="Call a utility script.", help="Call a utility script", formatter_class=argparse.ArgumentDefaultsHelpFormatter)
#     utils_command.define_parser(parser_utils)
# 
# #     import argcomplete
#     if argcomplete is not None:
#         argcomplete.autocomplete(parser)
# 
#     args = parser.parse_args(args=arguments)
#     if arguments is not None:
#         args.__original_argument_list = arguments
#     else:
#         args.__original_argument_list = sys.argv
# 
#     if args.__subcommand_name is None:
#         parser.print_help()
#         sys.exit(1)
# 
#     func = {"make_data": make_data.do_make_data,
#             "train": train.do_train,
#             "eval": eval_module.do_eval,
#             "version": versioning_tools.main,
#             "utils": utils_command.do_utils}[args.__subcommand_name]
# 
#     if args.run_in_pdb:
#         try:
#             import ipdb
#             run_in_ipdb(func, args)
#         except ImportError:
#             run_in_pdb(func, args)
#         # import pdb
#         # pdb.runcall(func, args)
#         # def debug_signal_handler(signal, frame):
#         #     import pdb
#         #     pdb.set_trace()
#         # import signal
#         # signal.signal(signal.SIGINT, debug_signal_handler)
# 
#     elif args.run_in_cprofile:
#         import cProfile
#         cProfile.runctx("func(args)", globals(), locals(), sort="cumtime")
#     
#     else:
#         try:
#             func(args)
#         except train.CommandLineValuesException as e:
#             parser_train.error(e.args[0])
#         except eval_module.CommandLineValuesException as e:
#             parser_eval.error(e.args[0])
#         except make_data.CommandLineValuesException as e:
#             parser_make_data.error(e.args[0])
# 
# Path: nmt_chainer/utilities/utils.py
# def de_batch(batch, mask=None, eos_idx=None, is_variable=False, raw=False):
#     """ Utility function for "de-batching".
#         batch is a list of Variable/numpy/cupy of shape[0] <= mb_size
# 
#         returns a list of the sequences in the batch
#     """
#     res = []
#     mb_size = len(batch[0].data) if is_variable else len(batch[0])
#     if mask is not None:
#         mask_offset = len(batch) - len(mask)
#         assert mask_offset >= 0
#     for sent_num in six.moves.range(mb_size):
#         assert sent_num == len(res)
#         res.append([])
#         for src_pos in six.moves.range(len(batch)):
#             current_batch_size = batch[src_pos].data.shape[0] if is_variable else batch[src_pos].shape[0]
#             if (mask is None or
#                     (src_pos < mask_offset or mask[src_pos - mask_offset][sent_num])) and current_batch_size > sent_num:
#                 #                 print(sent_num, src_pos, batch[src_pos].data)
#                 idx = batch[src_pos].data[sent_num] if is_variable else batch[src_pos][sent_num]
#                 if not raw:
#                     idx = int(idx)
#                 res[sent_num].append(None)
#                 res[sent_num][src_pos] = idx
#                 if eos_idx is not None and idx == eos_idx:
#                     break
#     return res


# Path: tests/suite1/utils_test.py
import numpy as np
import chainer
import chainer.functions as F
import chainer.links as L
import nmt_chainer.models as models
import nmt_chainer.utilities.utils as utils
import nmt_chainer.dataprocessing.make_data as make_data
import nmt_chainer.training_module.train as train
import os.path
from chainer import Link, Chain, ChainList, Variable
from nmt_chainer.__main__ import main
from nmt_chainer.utilities.utils import de_batch

#!/usr/bin/env python
"""utils_tests.py: Some correctness tests"""
from __future__ import absolute_import, division, print_function, unicode_literals
__author__ = "Fabien Cromieres"
__license__ = "undecided"
__version__ = "1.0"
__email__ = "fabien.cromieres@gmail.com"
__status__ = "Development"






class TestDeBatch:
    def test_multiple_length(self):
        batch = [np.array([1, 3, 4, 8]), np.array([1, 5, 6, 9]), np.array([7, 5]), np.array([10])]

        seq_list = de_batch(batch)