# Path: data_generator.py
# class Tokenizer(object):
#     def __init__(self, args):
#         self.args = args
# 
#     def get_dict(self, seqs):
#         s = set()
#         for seq in seqs:
#             for elem in seq:
#                 s.add(elem)
#         return {e: i + 1 for i, e in enumerate(s)}
# 
#     def convert_sequence(self, seqs, dic):
#         result = []
#         for seq in seqs:
#             a = []
#             for elem in seq:
#                 if elem not in dic:
#                     unk = '<unk>'
#                     if unk not in dic:
#                         dic[unk] = len(dic) + 1
#                     a.append(dic[unk])
#                 else:
#                     a.append(dic[elem])
#             result.append(a)
#         return result
# 
#     def padding(self, seqs, el, pad=0):
#         lengths = []
#         for seq in seqs:
#             lengths.append(len(seq) + 1)
#             for _ in range(el - len(seq)):
#                 seq.append(pad)
#         return seqs, lengths
# 
#     def initialize_basic(self, X, Y, X_test, Y_test):
#         voc = self.get_dict(X)
#         act = self.get_dict(Y)
# 
#         x_out = self.convert_sequence(X, voc)
#         y_out = self.convert_sequence(Y, act)
#         x_test_out = self.convert_sequence(X_test, voc)
#         y_test_out = self.convert_sequence(Y_test, act)
#         return x_out, y_out, x_test_out, y_test_out, voc, act
# 
#     def get_maximum_length(self, train, test):
#         train_max = max([len(x) for x in train])
#         test_max = max([len(x) for x in test])
#         return max(train_max, test_max) + 1
# 
#     def initialize(self, X, Y, X_test, Y_test):
#         X, Y, X_test, Y_test, voc, act = self.initialize_basic(
#             X, Y, X_test, Y_test)
#         max_input = self.get_maximum_length(X, X_test)
#         max_output = self.get_maximum_length(Y, Y_test)
#         X, X_len = self.padding(X, max_input)
#         Y, Y_len = self.padding(Y, max_output)
#         X_test, X_test_len = self.padding(X_test, max_input)
#         Y_test, Y_test_len = self.padding(Y_test, max_output)
#         samples = X, Y, X_test, Y_test
#         dicts = voc, act
#         lengths = X_len, Y_len, X_test_len, Y_test_len
#         maxs = max_input, max_output
#         return samples, dicts, lengths, maxs
# 
# Path: data_generator.py
# def get_data_generator(name, args):
#     if name == 'fewshot':
#         return FewShotDataGenerator(args)
#     elif name == 'scan':
#         return SCANGenerator(args)
#     elif name == 'toy':
#         return ToyDataGenerator(args)
#     else:
#         raise ValueError("Data generator name is not defined: " + name)
# 
# Path: model.py
# def get_model(name, args):
#     if name == 'simple':
#         return SimpleModel(args)
#     elif name == 'lstm':
#         return LSTMModel(args)
#     elif name == 's2s':
#         return S2SModel(args)
#     elif name == 's2s_att':
#         return S2SAttentionModel(args)
#     elif name == 'rand_reg':
#         return RandRegModel(args)
#     elif name == 'normal':
#         return NormalModel(args)
#     else:
#         raise ValueError("Model name is not defined: " + name)


import argparse
import random
import editdistance
import os
import numpy as np
from data_generator import Tokenizer
from data_generator import get_data_generator
from model import get_model
                        help='random batch.')
    parser.add_argument('--epochs', type=int, default=5000,
                        help='epochs')
    parser.add_argument('--data_name', type=str, default='scan',
                        help='name of data set')
    parser.add_argument('--train_file', type=str,
                        default='SCAN/add_prim_split/tasks_train_addprim_jump.txt',
                        help='train file name')
    parser.add_argument('--test_file', type=str,
                        default='SCAN/add_prim_split/tasks_test_addprim_jump.txt',
                        help='test file name')
    parser.add_argument('--switch_temperature', type=float, default=1.0,
                        help='switch temperature')
    parser.add_argument('--attention_temperature', type=float, default=10.0,
                        help='attention temperature')
    parser.add_argument('--num_units', type=int, default=16,
                        help='num units')
    parser.add_argument('--bidirectional_encoder', action='store_true', default=False,
                        help='bidirectional encoder.')
    parser.add_argument('--max_gradient_norm', type=float, default=-1.0,
                        help='max gradient norm')
    parser.add_argument('--decay_steps', type=int, default=-1,
                        help='decay steps')
    parser.add_argument('--use_input_length', action='store_true', default=False,
                        help='use input length.')
    parser.add_argument('--use_embedding', action='store_true', default=False,
                        help='use embedding.')
    parser.add_argument('--embedding_size', type=int, default=32,
                        help='embedding size')
    parser.add_argument('--function_embedding_size', type=int, default=-1,

# Path: data_generator.py
# class Tokenizer(object):
#     def __init__(self, args):
#         self.args = args
# 
#     def get_dict(self, seqs):
#         s = set()
#         for seq in seqs:
#             for elem in seq:
#                 s.add(elem)
#         return {e: i + 1 for i, e in enumerate(s)}
# 
#     def convert_sequence(self, seqs, dic):
#         result = []
#         for seq in seqs:
#             a = []
#             for elem in seq:
#                 if elem not in dic:
#                     unk = '<unk>'
#                     if unk not in dic:
#                         dic[unk] = len(dic) + 1
#                     a.append(dic[unk])
#                 else:
#                     a.append(dic[elem])
#             result.append(a)
#         return result
# 
#     def padding(self, seqs, el, pad=0):
#         lengths = []
#         for seq in seqs:
#             lengths.append(len(seq) + 1)
#             for _ in range(el - len(seq)):
#                 seq.append(pad)
#         return seqs, lengths
# 
#     def initialize_basic(self, X, Y, X_test, Y_test):
#         voc = self.get_dict(X)
#         act = self.get_dict(Y)
# 
#         x_out = self.convert_sequence(X, voc)
#         y_out = self.convert_sequence(Y, act)
#         x_test_out = self.convert_sequence(X_test, voc)
#         y_test_out = self.convert_sequence(Y_test, act)
#         return x_out, y_out, x_test_out, y_test_out, voc, act
# 
#     def get_maximum_length(self, train, test):
#         train_max = max([len(x) for x in train])
#         test_max = max([len(x) for x in test])
#         return max(train_max, test_max) + 1
# 
#     def initialize(self, X, Y, X_test, Y_test):
#         X, Y, X_test, Y_test, voc, act = self.initialize_basic(
#             X, Y, X_test, Y_test)
#         max_input = self.get_maximum_length(X, X_test)
#         max_output = self.get_maximum_length(Y, Y_test)
#         X, X_len = self.padding(X, max_input)
#         Y, Y_len = self.padding(Y, max_output)
#         X_test, X_test_len = self.padding(X_test, max_input)
#         Y_test, Y_test_len = self.padding(Y_test, max_output)
#         samples = X, Y, X_test, Y_test
#         dicts = voc, act
#         lengths = X_len, Y_len, X_test_len, Y_test_len
#         maxs = max_input, max_output
#         return samples, dicts, lengths, maxs
# 
# Path: data_generator.py
# def get_data_generator(name, args):
#     if name == 'fewshot':
#         return FewShotDataGenerator(args)
#     elif name == 'scan':
#         return SCANGenerator(args)
#     elif name == 'toy':
#         return ToyDataGenerator(args)
#     else:
#         raise ValueError("Data generator name is not defined: " + name)
# 
# Path: model.py
# def get_model(name, args):
#     if name == 'simple':
#         return SimpleModel(args)
#     elif name == 'lstm':
#         return LSTMModel(args)
#     elif name == 's2s':
#         return S2SModel(args)
#     elif name == 's2s_att':
#         return S2SAttentionModel(args)
#     elif name == 'rand_reg':
#         return RandRegModel(args)
#     elif name == 'normal':
#         return NormalModel(args)
#     else:
#         raise ValueError("Model name is not defined: " + name)


# Path: main.py
import argparse
import random
import editdistance
import os
import numpy as np
from data_generator import Tokenizer
from data_generator import get_data_generator
from model import get_model

                        help='random batch.')
    parser.add_argument('--epochs', type=int, default=5000,
                        help='epochs')
    parser.add_argument('--data_name', type=str, default='scan',
                        help='name of data set')
    parser.add_argument('--train_file', type=str,
                        default='SCAN/add_prim_split/tasks_train_addprim_jump.txt',
                        help='train file name')
    parser.add_argument('--test_file', type=str,
                        default='SCAN/add_prim_split/tasks_test_addprim_jump.txt',
                        help='test file name')
    parser.add_argument('--switch_temperature', type=float, default=1.0,
                        help='switch temperature')
    parser.add_argument('--attention_temperature', type=float, default=10.0,
                        help='attention temperature')
    parser.add_argument('--num_units', type=int, default=16,
                        help='num units')
    parser.add_argument('--bidirectional_encoder', action='store_true', default=False,
                        help='bidirectional encoder.')
    parser.add_argument('--max_gradient_norm', type=float, default=-1.0,
                        help='max gradient norm')
    parser.add_argument('--decay_steps', type=int, default=-1,
                        help='decay steps')
    parser.add_argument('--use_input_length', action='store_true', default=False,
                        help='use input length.')
    parser.add_argument('--use_embedding', action='store_true', default=False,
                        help='use embedding.')
    parser.add_argument('--embedding_size', type=int, default=32,
                        help='embedding size')
    parser.add_argument('--function_embedding_size', type=int, default=-1,

                        help='function embedding size')