<commit_before>import numpy as np
import lasagne

from .. import utils
from .. import core
from .. import nodes

# ############################ utils for wrapping ############################


def wrap_lasagne_node(network, in_vw, param_kwargs, constructor, kwargs):
    """
    param_kwargs:
    dict from param name to map of keyword arguments for constructing
    (eg. inits, tags, etc.)
    """
    l_in = lasagne.layers.InputLayer(
        input_var=in_vw.variable,
        shape=in_vw.shape)
    l_out = constructor(l_in, **kwargs)
    output = lasagne.layers.get_output(l_out)
    output_shape = lasagne.layers.get_output_shape(l_out)
    params = lasagne.layers.get_all_params(l_out)
    to_replace = {}
    for param in params:
        name = param.name
        assert name in param_kwargs
        assert "tags" in param_kwargs[name]
        assert "default_inits" in param_kwargs[name]
        vw = network.create_vw(
            name=name,
            is_shared=True,
            shape=param.get_value().shape,
            **param_kwargs[name]
        )
        to_replace[param] = vw.variable
    new_output, = utils.deep_clone([output], to_replace)
    network.create_vw(
        name="default",
        variable=new_output,
        shape=output_shape,
        tags={"output"},
    )


class LasagneUpdatesNode(nodes.StandardUpdatesNode):

    """
    node that wraps lasagne update functions
    """

    def _new_update_deltas(self, network, parameters, grads):
        parameter_variables = [p.variable for p in parameters]
        updates = self._lasagne_updates(network, parameter_variables, grads)
        if isinstance(updates, dict):
            updates = list(updates.items())
        # HACK around lasagne not giving names to temporary variables
        counter = 0
        for var, _ in updates:
            if var.name is None:
                counter += 1
                var.name = "%s_unnamed_state%d" % (self.name, counter)
        return core.UpdateDeltas.from_updates(updates)

    def _lasagne_updates(self, network, parameter_variables, grads):
        raise NotImplementedError

# ################################## nodes ##################################


@core.register_node("lasagne_dense")
class DenseNode(core.NodeImpl):

    """
    node wrapping lasagne's DenseLayer
    """

    hyperparameter_names = ("inits",
                            "dense_num_units",
                            "num_units")

    def compute_output(self, network, in_vw):
        num_units = network.find_hyperparameter(["dense_num_units",
                                                 "num_units"])
        wrap_lasagne_node(
            network=network,
            in_vw=in_vw,
            param_kwargs=dict(
                W=dict(tags={"parameter", "weight"},
                       default_inits=[]),
                b=dict(tags={"parameter", "bias"},
                       default_inits=[])
            ),
            constructor=lasagne.layers.DenseLayer,
            kwargs=dict(
                num_units=num_units,
                nonlinearity=lasagne.nonlinearities.identity,
            )
        )


def ReLUNode(name):
    return nodes.ApplyNode(name,
                           fn=lasagne.nonlinearities.rectify,
                           shape_fn=utils.identity)

# ################################# updates #################################


@core.register_node("lasagne_sgd")
class SGDNode(LasagneUpdatesNode):

    """
    node that provides updates via SGD
    """

    hyperparameter_names = ("sgd_learning_rate",
                            "learning_rate")

    def _lasagne_updates(self, network, parameter_variables, grads):
        learning_rate = network.find_hyperparameter(["sgd_learning_rate",
                                                     "learning_rate"])
        return lasagne.updates.sgd(grads,
                                   parameter_variables,
                                   learning_rate=learning_rate)


@core.register_node("lasagne_nesterov_momentum")
class NesterovMomentumNode(LasagneUpdatesNode):

    """
    node that provides updates via SGD
    """

    hyperparameter_names = ("learning_rate",
                            "momentum")

    def _lasagne_updates(self, network, parameter_variables, grads):
        learning_rate = network.find_hyperparameter(["learning_rate"])
        momentum = network.find_hyperparameter(["momentum"], 0.9)
        return lasagne.updates.nesterov_momentum(grads,
                                                 parameter_variables,
                                                 learning_rate=learning_rate,
                                                 momentum=momentum)

# ############################### convolutions ###############################


@core.register_node("lasagne_conv2d")
class Conv2DNode(core.NodeImpl):

    """
    node wrapping lasagne's Conv2DLayer
    """

    hyperparameter_names = ("inits",
                            "num_filters",
                            "filter_size",
                            "conv_stride",
                            "stride",<commit_msg>create lasagne apply updates node<commit_after>import numpy as np
import lasagne

from .. import utils
from .. import core
from .. import nodes

# ############################ utils for wrapping ############################


def wrap_lasagne_node(network, in_vw, param_kwargs, constructor, kwargs):
    """
    param_kwargs:
    dict from param name to map of keyword arguments for constructing
    (eg. inits, tags, etc.)
    """
    l_in = lasagne.layers.InputLayer(
        input_var=in_vw.variable,
        shape=in_vw.shape)
    l_out = constructor(l_in, **kwargs)
    output = lasagne.layers.get_output(l_out)
    output_shape = lasagne.layers.get_output_shape(l_out)
    params = lasagne.layers.get_all_params(l_out)
    to_replace = {}
    for param in params:
        name = param.name
        assert name in param_kwargs
        assert "tags" in param_kwargs[name]
        assert "default_inits" in param_kwargs[name]
        vw = network.create_vw(
            name=name,
            is_shared=True,
            shape=param.get_value().shape,
            **param_kwargs[name]
        )
        to_replace[param] = vw.variable
    new_output, = utils.deep_clone([output], to_replace)
    network.create_vw(
        name="default",
        variable=new_output,
        shape=output_shape,
        tags={"output"},
    )


class BaseLasagneUpdatesNode(nodes.StandardUpdatesNode):

    """
    node that wraps lasagne update functions
    """

    def _new_update_deltas(self, network, parameters, grads):
        parameter_variables = [p.variable for p in parameters]
        updates = self._lasagne_updates(network, parameter_variables, grads)
        if isinstance(updates, dict):
            updates = list(updates.items())
        # HACK around lasagne not giving names to temporary variables
        counter = 0
        for var, _ in updates:
            if var.name is None:
                counter += 1
                var.name = "%s_unnamed_state%d" % (self.name, counter)
        return core.UpdateDeltas.from_updates(updates)

    def _lasagne_updates(self, network, parameter_variables, grads):
        raise NotImplementedError

# ################################## nodes ##################################


@core.register_node("lasagne_dense")
class DenseNode(core.NodeImpl):

    """
    node wrapping lasagne's DenseLayer
    """

    hyperparameter_names = ("inits",
                            "dense_num_units",
                            "num_units")

    def compute_output(self, network, in_vw):
        num_units = network.find_hyperparameter(["dense_num_units",
                                                 "num_units"])
        wrap_lasagne_node(
            network=network,
            in_vw=in_vw,
            param_kwargs=dict(
                W=dict(tags={"parameter", "weight"},
                       default_inits=[]),
                b=dict(tags={"parameter", "bias"},
                       default_inits=[])
            ),
            constructor=lasagne.layers.DenseLayer,
            kwargs=dict(
                num_units=num_units,
                nonlinearity=lasagne.nonlinearities.identity,
            )
        )


def ReLUNode(name):
    return nodes.ApplyNode(name,
                           fn=lasagne.nonlinearities.rectify,
                           shape_fn=utils.identity)

# ################################# updates #################################


@core.register_node("lasagne_apply_updates")
class LasagneApplyUpdatesNode(BaseLasagneUpdatesNode):

    """
    node that provides updates via a provided fn, given cost and parameters
    """

    hyperparameter_names = ("fn",)

    def _lasagne_updates(self, network, parameter_variables, grads):
        update_fn = network.find_hyperparameter(["fn"])
        return update_fn(grads, parameter_variables)


@core.register_node("lasagne_sgd")
class SGDNode(BaseLasagneUpdatesNode):

    """
    node that provides updates via SGD
    """

    hyperparameter_names = ("sgd_learning_rate",
                            "learning_rate")

    def _lasagne_updates(self, network, parameter_variables, grads):
        learning_rate = network.find_hyperparameter(["sgd_learning_rate",
                                                     "learning_rate"])
        return lasagne.updates.sgd(grads,
                                   parameter_variables,
                                   learning_rate=learning_rate)


@core.register_node("lasagne_nesterov_momentum")
class NesterovMomentumNode(BaseLasagneUpdatesNode):

    """
    node that provides updates via SGD
    """

    hyperparameter_names = ("learning_rate",
                            "momentum")

    def _lasagne_updates(self, network, parameter_variables, grads):
        learning_rate = network.find_hyperparameter(["learning_rate"])
        momentum = network.find_hyperparameter(["momentum"], 0.9)
        return lasagne.updates.nesterov_momentum(grads,
                                                 parameter_variables,
                                                 learning_rate=learning_rate,
                                                 momentum=momentum)

# ############################### convolutions ###############################


@core.register_node("lasagne_conv2d")
class Conv2DNode(core.NodeImpl):

    """
    node wrapping lasagne's Conv2DLayer
    """

    hyperparameter_names = ("inits",
                            "num_filters",
                            "filter_size",
                            "conv_stride",
                            "stride",
{'fraction_non_alphanumeric': 0.13259203887123902, 'fraction_numerical': 0.001308166697813493, 'mean_word_length': 2.0924588269286333, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5253285', 'n_tokens_mistral': 2927, 'n_tokens_neox': 2822, 'n_words': 540}