<commit_before>## @package gradient_checker
# Module caffe2.python.gradient_checker
import numpy as np
from caffe2.python import core, workspace
from caffe2.proto import caffe2_pb2


class NetGradientChecker(object):
    @staticmethod
    def Check(net, outputs_with_grad, input_values,
              input_to_check, step_size=0.0001, threshold=0.05):
        assert input_to_check in input_values.keys()

        net_copy = net.Clone(net.Name() + "_copy")

        grad_map = net_copy.AddGradientOperators(outputs_with_grad)
        for name, value in input_values.items():
            workspace.blobs[name] = value

        def GetLoss(new_value):
            workspace.blobs[input_to_check] = new_value
            workspace.RunNetOnce(net_copy)
            return sum([
                workspace.blobs[output]
                for output in outputs_with_grad
            ])

        def GetValue(dim, delta):
            input_value = input_values[input_to_check].copy()
            input_value.flat[dim] += delta
            return input_value

        workspace.RunNetOnce(net_copy)
        analitic_grad = workspace.blobs[grad_map[input_to_check]]

        grad_estimate = np.zeros_like(input_values[input_to_check])
        for dim in range(input_values[input_to_check].size):
            pos_loss = GetLoss(GetValue(dim, step_size))
            neg_loss = GetLoss(GetValue(dim, -step_size))
            grad_estimate.flat[dim] = (pos_loss - neg_loss) / step_size / 2

        err_msg = "Error in gradient check for net_copy {}: {}".format(
            net.Name(), net.Proto())

        np.testing.assert_allclose(
            analitic_grad, grad_estimate,
            atol=threshold, rtol=threshold,
            err_msg=err_msg,
        )


class GradientChecker:
    """A gradient checker in Python.

    This is not the most efficient way to check gradients, as the Python
    interface will involve a lot of copy back and forth operations. Use at your
    own risk.
    """

    def __init__(
        self,
        stepsize,
        threshold,
        device_option=caffe2_pb2.DeviceOption(),
        workspace_name="gradient_check"
    ):
        self._stepsize = stepsize
        self._threshold = threshold
        self._device_option = device_option
        self._workspace_name = workspace_name

    def GetLossAndGrad(
        self, op, grad_ops, x, input_name, grad_name, outputs_with_grads<commit_msg>add net gradient check<commit_after>## @package gradient_checker
# Module caffe2.python.gradient_checker
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import numpy as np

from caffe2.python import core, workspace
from caffe2.proto import caffe2_pb2


class NetGradientChecker(object):
    @staticmethod
    def Check(net, outputs_with_grad, input_values,
              input_to_check, step_size=0.0001,
              threshold=0.05, print_net=True):
        assert input_to_check in input_values.keys()

        net_copy = net.Clone(net.Name() + "_copy")

        grad_map = net_copy.AddGradientOperators(outputs_with_grad)
        assert input_to_check in grad_map, (
            '{} has no gradient, cannot check net gradient.'.format(
                input_to_check))

        for name, value in input_values.items():
            workspace.blobs[name] = value

        def GetLoss(new_value):
            workspace.blobs[input_to_check] = new_value
            workspace.RunNetOnce(net_copy)
            return sum([
                workspace.blobs[output]
                for output in outputs_with_grad
            ])

        def GetValue(dim, delta):
            input_value = input_values[input_to_check].copy()
            input_value.flat[dim] += delta
            return input_value

        workspace.RunNetOnce(net_copy)
        grad_blob = grad_map[input_to_check]

        def get_analytic_grad(grad_blob):
            if isinstance(grad_blob, core.BlobReference):
                return workspace.blobs[grad_blob]

            # If grad_blob is not a single blob, it should be a gradient slice.
            # To make it comparable with the estimiated gradient which is dense,
            # we need to first convert grad_blob to dense gradient.
            assert isinstance(grad_blob, core.GradientSlice)
            dense_grad = 'tmp_dense_grad'
            sparse_to_dense_op = core.CreateOperator(
                'SparseToDense',
                [grad_blob.indices, grad_blob.values, input_to_check],
                dense_grad,
            )
            workspace.RunOperatorOnce(sparse_to_dense_op)
            return workspace.blobs[dense_grad]

        analytic_grad = get_analytic_grad(grad_blob)

        grad_estimate = np.zeros_like(input_values[input_to_check])
        for dim in range(input_values[input_to_check].size):
            pos_loss = GetLoss(GetValue(dim, step_size))
            neg_loss = GetLoss(GetValue(dim, -step_size))
            grad_estimate.flat[dim] = (pos_loss - neg_loss) / step_size / 2

        err_msg = "Error in gradient check for net_copy {}".format(
            net.Name())
        if print_net:
            err_msg += ": {}".format(net.Proto())

        np.testing.assert_allclose(
            analytic_grad, grad_estimate,
            atol=threshold, rtol=threshold,
            err_msg=err_msg,
        )

        delta = np.abs(grad_estimate - analytic_grad).flatten()
        return np.mean(delta), max(delta)


class GradientChecker:
    """A gradient checker in Python.

    This is not the most efficient way to check gradients, as the Python
    interface will involve a lot of copy back and forth operations. Use at your
    own risk.
    """

    def __init__(
        self,
        stepsize,
        threshold,
        device_option=caffe2_pb2.DeviceOption(),
        workspace_name="gradient_check"
    ):
        self._stepsize = stepsize
        self._threshold = threshold
        self._device_option = device_option
        self._workspace_name = workspace_name

    def GetLossAndGrad(
        self, op, grad_ops, x, input_name, grad_name, outputs_with_grads
{'fraction_non_alphanumeric': 0.07773450178513469, 'fraction_numerical': 0.00519311911716975, 'mean_word_length': 2.423888888888889, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3607781', 'n_tokens_mistral': 1829, 'n_tokens_neox': 1756, 'n_words': 420}