# Path: tupa/classifiers/nn/constants.py
# ACTIVATIONS = {
#     "cube": "cube",
#     "tanh": "tanh",
#     "sigmoid": "logistic",
#     "relu": "rectify",
# }
# 
# INITIALIZERS = {
#     # "saxe": "SaxeInitializer",
#     "glorot_uniform": "GlorotInitializer",
#     "normal": "NormalInitializer",
# }
# 
# class CategoricalParameter:
#     def __init__(self, values, string):
#         self._value = self._string = None
#         self.values = values
#         self.string = string
# 
#     def __call__(self):
#         import dynet as dy
#         return getattr(dy, self._value)
# 
#     @property
#     def string(self):
#         return self._string
# 
#     @string.setter
#     def string(self, s):
#         self._string = s
#         self._value = self.values.get(s)
# 
#     def __str__(self):
#         return self._string
# 
# Path: tupa/classifiers/nn/sub_model.py
# class SubModel:
#     def __init__(self, params=None, save_path=(), shared=False, copy_shared=False):
#         self.params = OrderedDict() if params is None else params  # string (param identifier) -> parameter
#         self.save_path = save_path
#         self.shared = shared
#         self.copy_shared = copy_shared
# 
#     def save_sub_model(self, d, *args):
#         self.get_sub_dict(d).update(args + (("param_keys", list(self.params.keys())),))
#         return list(self.params.values())
# 
#     def load_sub_model(self, d, *args, load_path=None):
#         d = self.get_sub_dict(d, load_path=load_path)
#         param_keys = d.get("param_keys", ())
#         assert len(param_keys) <= len(args), "%s loaded values: expected %d, got %d" % ("/".join(self.save_path),
#                                                                                         len(param_keys), len(args))
#         self.params.clear()
#         self.params.update(zip(param_keys, args))
#         return d
# 
#     def get_sub_dict(self, d, load_path=None):
#         for element in load_path or self.save_path:
#             d = d.setdefault(element, OrderedDict())
#         return d
# 
#     def params_str(self):
#         return "/".join(self.save_path) + (": " if self.save_path else "") + ", ".join(self.params.keys())
# 
#     def invalidate_caches(self):
#         for model in self.sub_models():
#             model.invalidate_caches()
# 
#     def sub_models(self):
#         return ()
# 
# Path: tupa/classifiers/nn/util.py
# def randomize_orthonormal(*parameters, activation=None):  # Saxe et al., 2014 (https://arxiv.org/abs/1312.6120)
#     for param in parameters:
#         shape = param.shape()
#         if len(shape) == 2 and shape[0] == shape[1] > 1:
#             init, _, _ = np.linalg.svd(np.random.randn(*shape))
#             if str(activation) == "relu":
#                 init *= np.sqrt(2)
#             param.set_value(init)


from itertools import groupby
from .constants import ACTIVATIONS, INITIALIZERS, CategoricalParameter
from .sub_model import SubModel
from .util import randomize_orthonormal
import dynet as dy
import re
        return self.layers + (1 if self.num_labels else 0)

    def init_params(self, input_dim):
        assert input_dim, "Zero input dimension to MLP"
        self.input_dim = input_dim
        if self.layers > 0:
            W0 = self.params.get("W0")
            if W0:
                value = W0.as_array()
                old_input_dim = value.shape[1]
                extra_dim = self.input_dim - old_input_dim
                if extra_dim:  # Duplicate suffix of input dimension to accommodate extra input
                    assert extra_dim > 0, "%s: input dimension reduced from %d to %d" % (
                        "/".join(self.save_path), old_input_dim, self.input_dim)
                    value[:, -extra_dim:] /= 2
                    W0.set_value(value)
                    extra_value = value[:, -extra_dim:]
                    self.params["W0+"] = extra = self.model.add_parameters(extra_value.shape)
                    extra.set_value(extra_value)
            else:
                hidden_dim = (self.layers - 1) * [self.layer_dim]
                i_dim = [input_dim] + hidden_dim
                o_dim = hidden_dim + [self.output_dim]
                if self.num_labels:  # Adding another layer at the top
                    i_dim.append(self.output_dim)
                    o_dim.append(self.num_labels)
                self.params.update((prefix + str(i), self.model.add_parameters(dims[i], init=self.init()()))
                                   for prefix, dims in (("W", list(zip(o_dim, i_dim))), ("b", o_dim))
                                   for i, dim in enumerate(dims))
                self.verify_dims()

# Path: tupa/classifiers/nn/constants.py
# ACTIVATIONS = {
#     "cube": "cube",
#     "tanh": "tanh",
#     "sigmoid": "logistic",
#     "relu": "rectify",
# }
# 
# INITIALIZERS = {
#     # "saxe": "SaxeInitializer",
#     "glorot_uniform": "GlorotInitializer",
#     "normal": "NormalInitializer",
# }
# 
# class CategoricalParameter:
#     def __init__(self, values, string):
#         self._value = self._string = None
#         self.values = values
#         self.string = string
# 
#     def __call__(self):
#         import dynet as dy
#         return getattr(dy, self._value)
# 
#     @property
#     def string(self):
#         return self._string
# 
#     @string.setter
#     def string(self, s):
#         self._string = s
#         self._value = self.values.get(s)
# 
#     def __str__(self):
#         return self._string
# 
# Path: tupa/classifiers/nn/sub_model.py
# class SubModel:
#     def __init__(self, params=None, save_path=(), shared=False, copy_shared=False):
#         self.params = OrderedDict() if params is None else params  # string (param identifier) -> parameter
#         self.save_path = save_path
#         self.shared = shared
#         self.copy_shared = copy_shared
# 
#     def save_sub_model(self, d, *args):
#         self.get_sub_dict(d).update(args + (("param_keys", list(self.params.keys())),))
#         return list(self.params.values())
# 
#     def load_sub_model(self, d, *args, load_path=None):
#         d = self.get_sub_dict(d, load_path=load_path)
#         param_keys = d.get("param_keys", ())
#         assert len(param_keys) <= len(args), "%s loaded values: expected %d, got %d" % ("/".join(self.save_path),
#                                                                                         len(param_keys), len(args))
#         self.params.clear()
#         self.params.update(zip(param_keys, args))
#         return d
# 
#     def get_sub_dict(self, d, load_path=None):
#         for element in load_path or self.save_path:
#             d = d.setdefault(element, OrderedDict())
#         return d
# 
#     def params_str(self):
#         return "/".join(self.save_path) + (": " if self.save_path else "") + ", ".join(self.params.keys())
# 
#     def invalidate_caches(self):
#         for model in self.sub_models():
#             model.invalidate_caches()
# 
#     def sub_models(self):
#         return ()
# 
# Path: tupa/classifiers/nn/util.py
# def randomize_orthonormal(*parameters, activation=None):  # Saxe et al., 2014 (https://arxiv.org/abs/1312.6120)
#     for param in parameters:
#         shape = param.shape()
#         if len(shape) == 2 and shape[0] == shape[1] > 1:
#             init, _, _ = np.linalg.svd(np.random.randn(*shape))
#             if str(activation) == "relu":
#                 init *= np.sqrt(2)
#             param.set_value(init)


# Path: tupa/classifiers/nn/mlp.py
from itertools import groupby
from .constants import ACTIVATIONS, INITIALIZERS, CategoricalParameter
from .sub_model import SubModel
from .util import randomize_orthonormal
import dynet as dy
import re

        return self.layers + (1 if self.num_labels else 0)

    def init_params(self, input_dim):
        assert input_dim, "Zero input dimension to MLP"
        self.input_dim = input_dim
        if self.layers > 0:
            W0 = self.params.get("W0")
            if W0:
                value = W0.as_array()
                old_input_dim = value.shape[1]
                extra_dim = self.input_dim - old_input_dim
                if extra_dim:  # Duplicate suffix of input dimension to accommodate extra input
                    assert extra_dim > 0, "%s: input dimension reduced from %d to %d" % (
                        "/".join(self.save_path), old_input_dim, self.input_dim)
                    value[:, -extra_dim:] /= 2
                    W0.set_value(value)
                    extra_value = value[:, -extra_dim:]
                    self.params["W0+"] = extra = self.model.add_parameters(extra_value.shape)
                    extra.set_value(extra_value)
            else:
                hidden_dim = (self.layers - 1) * [self.layer_dim]
                i_dim = [input_dim] + hidden_dim
                o_dim = hidden_dim + [self.output_dim]
                if self.num_labels:  # Adding another layer at the top
                    i_dim.append(self.output_dim)
                    o_dim.append(self.num_labels)
                self.params.update((prefix + str(i), self.model.add_parameters(dims[i], init=self.init()()))
                                   for prefix, dims in (("W", list(zip(o_dim, i_dim))), ("b", o_dim))
                                   for i, dim in enumerate(dims))
                self.verify_dims()

                randomize_orthonormal(*[v for k, v in self.params.items() if MLP_PARAM_PATTERN.match(k)],