<commit_before># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for multi_class_head.py."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import six
import tensorflow as tf
from tensorflow.python.framework import test_util
from tensorflow_estimator.python.estimator.canned import dnn
from tensorflow_estimator.python.estimator.canned import metric_keys
from tensorflow_estimator.python.estimator.canned import prediction_keys
from tensorflow_estimator.python.estimator.head import head_utils as test_lib
from tensorflow_estimator.python.estimator.head import multi_class_head as head_lib
from tensorflow_estimator.python.estimator.mode_keys import ModeKeys


@test_util.run_all_in_graph_and_eager_modes
class MultiClassHead(tf.test.TestCase):

  def test_n_classes_is_none(self):
    with self.assertRaisesRegexp(ValueError, 'n_classes cannot be None'):
      head_lib.MultiClassHead(n_classes=None)

  def test_n_classes_is_2(self):
    with self.assertRaisesRegexp(ValueError, 'n_classes must be > 2'):
      head_lib.MultiClassHead(n_classes=2)

  def test_invalid_loss_reduction(self):
    with self.assertRaisesRegexp(
        ValueError, r'Invalid loss_reduction: invalid_loss_reduction'):
      head_lib.MultiClassHead(
          n_classes=3, loss_reduction='invalid_loss_reduction')
    with self.assertRaisesRegexp(ValueError, r'Invalid loss_reduction: none'):
      head_lib.MultiClassHead(
          n_classes=3, loss_reduction=tf.compat.v2.keras.losses.Reduction.NONE)

  def test_loss_fn_arg_labels_missing(self):

    def _loss_fn(logits):
      del logits  # Unused

    with self.assertRaisesRegexp(
        ValueError, r'loss_fn must contain argument: labels\. '
        r'Given arguments: \(\'logits\',\)'):
      head_lib.MultiClassHead(n_classes=3, loss_fn=_loss_fn)

  def test_loss_fn_arg_logits_missing(self):

    def _loss_fn(labels):
      del labels  # unused

    with self.assertRaisesRegexp(
        ValueError, r'loss_fn must contain argument: logits\. '
        r'Given arguments: \(\'labels\',\)'):
      head_lib.MultiClassHead(n_classes=3, loss_fn=_loss_fn)

  def test_loss_fn_arg_features_ok(self):

    def _loss_fn(labels, logits, features):
      del labels, logits, features  # Unused

    head_lib.MultiClassHead(n_classes=3, loss_fn=_loss_fn)

  def test_loss_fn_arg_invalid(self):

    def _loss_fn(labels, logits, name=None):
      del labels, logits, name  # Unused

    with self.assertRaisesRegexp(ValueError,
                                 r'loss_fn has unexpected args: \[\'name\'\]'):
      head_lib.MultiClassHead(n_classes=3, loss_fn=_loss_fn)

  def test_invalid_logits_shape(self):
    n_classes = 3
    head = head_lib.MultiClassHead(n_classes)
    self.assertEqual(n_classes, head.logits_dimension)

    # Logits should be shape (batch_size, 3).
    logits_2x2 = np.array((
        (45., 44.),
        (41., 42.),
    ))
    pred_key = prediction_keys.PredictionKeys.PROBABILITIES
    # Static shape.
    with self.assertRaisesRegexp(ValueError, 'logits shape'):
      preds = head.predictions(logits_2x2, [pred_key])
      self.evaluate(preds[pred_key])
    if tf.executing_eagerly():
      return

    # Dynamic shape only works in Graph mode.
    logits_placeholder = tf.compat.v1.placeholder(dtype=tf.dtypes.float32)
    spec = head.create_estimator_spec(
        features={'x': np.array((
            (30.,),
            (42.,),
        ))},
        mode=ModeKeys.PREDICT,
        logits=logits_placeholder,
        trainable_variables=[tf.Variable([1.0, 2.0], dtype=tf.dtypes.float32)])
    with self.cached_session():
      with self.assertRaisesRegexp(tf.errors.OpError, 'logits shape'):
        spec.predictions[pred_key].eval({logits_placeholder: logits_2x2})

  def test_invalid_labels_shape(self):
    n_classes = 3
    head = head_lib.MultiClassHead(n_classes)
    self.assertEqual(n_classes, head.logits_dimension)

    # Logits should be shape (batch_size, 3).
    # Labels should be shape (batch_size, 1).
    labels_2x2 = np.array((
        (1, 2),
        (0, 1),
    ), dtype=np.int)
    logits_2x3 = np.array((
        (1., 2., 3.),
        (1., 2., 3.),
    ))
    features = {'x': np.array(((42.,),))}

    # Static shape.
    with self.assertRaisesRegexp(ValueError, 'Mismatched label shape'):
      training_loss = head.loss(
          logits=logits_2x3,
          labels=labels_2x2,
          features=features,
          mode=ModeKeys.EVAL)
      self.evaluate(training_loss)
    if tf.executing_eagerly():
      return

    # Dynamic shape only works in Graph mode.
    labels_placeholder = tf.compat.v1.placeholder(dtype=tf.dtypes.int64)
    logits_placeholder = tf.compat.v1.placeholder(dtype=tf.dtypes.float32)
    training_loss = head.loss(
        logits=logits_placeholder,
        labels=labels_placeholder,
        features=features,
        mode=ModeKeys.EVAL)
    with self.cached_session():
      with self.assertRaisesRegexp(
          tf.errors.InvalidArgumentError,
          r'\[expected_labels_shape: \] \[2 1\] \[labels_shape: \] \[2 2\]'):
        training_loss.eval({
            logits_placeholder: logits_2x3,
            labels_placeholder: labels_2x2
        })

  def test_invalid_labels_type(self):
    n_classes = 3
    head = head_lib.MultiClassHead(n_classes)
    self.assertEqual(n_classes, head.logits_dimension)

    # Logits should be shape (batch_size, 3).
    # Labels should be shape (batch_size, 1).
    labels_2x1 = np.array((
        (1.,),
        (1.,),
    ))
    logits_2x3 = np.array((
        (1., 2., 3.),
        (1., 2., 3.),
    ))
    features = {'x': np.array(((42.,),))}

    # Static shape.
    with self.assertRaisesRegexp(ValueError, 'Labels dtype'):
      head.loss(
          logits=logits_2x3,
          labels=labels_2x1,
          features=features,
          mode=ModeKeys.EVAL)
    if tf.executing_eagerly():
      return

    # Dynamic shape only works in Graph mode.
    labels_placeholder = tf.compat.v1.placeholder(dtype=tf.dtypes.float32)
    logits_placeholder = tf.compat.v1.placeholder(dtype=tf.dtypes.float32)
    with self.assertRaisesRegexp(ValueError, 'Labels dtype'):
      head.loss(
          logits=logits_placeholder,
          labels=labels_placeholder,
          features=features,
          mode=ModeKeys.EVAL)

  def test_invalid_labels_values(self):
    n_classes = 3
    head = head_lib.MultiClassHead(n_classes)
    self.assertEqual(n_classes, head.logits_dimension)

    labels_2x1_with_large_id = np.array((
        (45,),
        (1,),
    ), dtype=np.int)
    labels_2x1_with_negative_id = np.array((
        (-5,),
        (1,),
    ), dtype=np.int)
    logits_2x3 = np.array((
        (1., 2., 4.),
        (1., 2., 3.),
    ))
    features = {'x': np.array(((42.,),))}

    if tf.executing_eagerly():
      with self.assertRaisesRegexp(ValueError, 'Labels must be <= 3 - 1'):
        training_loss = head.loss(
            logits=logits_2x3,
            labels=labels_2x1_with_large_id,
            features=features,
            mode=ModeKeys.EVAL)

      with self.assertRaisesRegexp(ValueError, 'Labels must be >= 0'):
        training_loss = head.loss(
            logits=logits_2x3,
            labels=labels_2x1_with_negative_id,
            features=features,
            mode=ModeKeys.EVAL)
      return

    # Dynamic shape only works in Graph mode.
    labels_placeholder = tf.compat.v1.placeholder(dtype=tf.dtypes.int64)
    logits_placeholder = tf.compat.v1.placeholder(dtype=tf.dtypes.float32)
    training_loss = head.loss(
        logits=logits_placeholder,
        labels=labels_placeholder,
        features=features,
        mode=ModeKeys.EVAL)
    with self.cached_session():
      with self.assertRaisesOpError('Labels must be <= n_classes - 1'):
        training_loss.eval({
            labels_placeholder: labels_2x1_with_large_id,
            logits_placeholder: logits_2x3
        })

    with self.cached_session():
      with self.assertRaisesOpError('Labels must be >= 0'):
        training_loss.eval({
            labels_placeholder: labels_2x1_with_negative_id,
            logits_placeholder: logits_2x3
        })

  def test_invalid_labels_sparse_tensor(self):
    n_classes = 3
    head = head_lib.MultiClassHead(n_classes)
    self.assertEqual(n_classes, head.logits_dimension)

    labels_2x1 = tf.sparse.SparseTensor(
        values=['english', 'italian'],
        indices=[[0, 0], [1, 0]],
        dense_shape=[2, 1])
    logits_2x3 = np.array((
        (1., 2., 4.),
        (1., 2., 3.),
    ))

    with self.assertRaisesRegexp(ValueError,
                                 'SparseTensor labels are not supported.'):
      loss = head.loss(
          logits=logits_2x3,
          labels=labels_2x1,
          features={'x': np.array(((42.,),))},
          mode=ModeKeys.EVAL)
      self.evaluate(loss)

  def test_incompatible_labels_shape(self):
    n_classes = 3
    head = head_lib.MultiClassHead(n_classes)
    self.assertEqual(n_classes, head.logits_dimension)

    # Logits should be shape (batch_size, 3).
    # Labels should be shape (batch_size, 1).
    # Here batch sizes are different.
    values_3x1 = np.array((
        (1,),
        (1,),
        (1,),
    ))
    values_2x3 = np.array((
        (1., 2., 3.),
        (1., 2., 3.),
    ))
    features = {'x': values_2x3}

    # Static shape.
    # Eager mode.
    if tf.executing_eagerly():
      with self.assertRaisesRegexp(ValueError, 'labels shape'):
        head.loss(
            logits=values_2x3,
            labels=values_3x1,
            features=features,
            mode=ModeKeys.EVAL)
      return
    # Graph mode.
    with self.assertRaisesRegexp(
        ValueError,
        r'Shape mismatch: The shape of labels \(received \(3,\)\) should equal '
        r'the shape of logits except for the last dimension '
        r'\(received \(2, 3\)\)\.'):
      head.loss(
          logits=values_2x3,
          labels=values_3x1,
          features=features,
          mode=ModeKeys.EVAL)

    # Dynamic shape only works in Graph mode.
    labels_placeholder = tf.compat.v1.placeholder(dtype=tf.dtypes.int64)
    logits_placeholder = tf.compat.v1.placeholder(dtype=tf.dtypes.float32)
    training_loss = head.loss(
        logits=logits_placeholder,
        labels=labels_placeholder,
        features=features,
        mode=ModeKeys.EVAL)
    with self.cached_session():
      with self.assertRaisesRegexp(
          tf.errors.InvalidArgumentError,<commit_msg>Improve error messages for nn_ops<commit_after># Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests for multi_class_head.py."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np
import six
import tensorflow as tf
from tensorflow.python.framework import test_util
from tensorflow_estimator.python.estimator.canned import dnn
from tensorflow_estimator.python.estimator.canned import metric_keys
from tensorflow_estimator.python.estimator.canned import prediction_keys
from tensorflow_estimator.python.estimator.head import head_utils as test_lib
from tensorflow_estimator.python.estimator.head import multi_class_head as head_lib
from tensorflow_estimator.python.estimator.mode_keys import ModeKeys


@test_util.run_all_in_graph_and_eager_modes
class MultiClassHead(tf.test.TestCase):

  def test_n_classes_is_none(self):
    with self.assertRaisesRegexp(ValueError, 'n_classes cannot be None'):
      head_lib.MultiClassHead(n_classes=None)

  def test_n_classes_is_2(self):
    with self.assertRaisesRegexp(ValueError, 'n_classes must be > 2'):
      head_lib.MultiClassHead(n_classes=2)

  def test_invalid_loss_reduction(self):
    with self.assertRaisesRegexp(
        ValueError, r'Invalid loss_reduction: invalid_loss_reduction'):
      head_lib.MultiClassHead(
          n_classes=3, loss_reduction='invalid_loss_reduction')
    with self.assertRaisesRegexp(ValueError, r'Invalid loss_reduction: none'):
      head_lib.MultiClassHead(
          n_classes=3, loss_reduction=tf.compat.v2.keras.losses.Reduction.NONE)

  def test_loss_fn_arg_labels_missing(self):

    def _loss_fn(logits):
      del logits  # Unused

    with self.assertRaisesRegexp(
        ValueError, r'loss_fn must contain argument: labels\. '
        r'Given arguments: \(\'logits\',\)'):
      head_lib.MultiClassHead(n_classes=3, loss_fn=_loss_fn)

  def test_loss_fn_arg_logits_missing(self):

    def _loss_fn(labels):
      del labels  # unused

    with self.assertRaisesRegexp(
        ValueError, r'loss_fn must contain argument: logits\. '
        r'Given arguments: \(\'labels\',\)'):
      head_lib.MultiClassHead(n_classes=3, loss_fn=_loss_fn)

  def test_loss_fn_arg_features_ok(self):

    def _loss_fn(labels, logits, features):
      del labels, logits, features  # Unused

    head_lib.MultiClassHead(n_classes=3, loss_fn=_loss_fn)

  def test_loss_fn_arg_invalid(self):

    def _loss_fn(labels, logits, name=None):
      del labels, logits, name  # Unused

    with self.assertRaisesRegexp(ValueError,
                                 r'loss_fn has unexpected args: \[\'name\'\]'):
      head_lib.MultiClassHead(n_classes=3, loss_fn=_loss_fn)

  def test_invalid_logits_shape(self):
    n_classes = 3
    head = head_lib.MultiClassHead(n_classes)
    self.assertEqual(n_classes, head.logits_dimension)

    # Logits should be shape (batch_size, 3).
    logits_2x2 = np.array((
        (45., 44.),
        (41., 42.),
    ))
    pred_key = prediction_keys.PredictionKeys.PROBABILITIES
    # Static shape.
    with self.assertRaisesRegexp(ValueError, 'logits shape'):
      preds = head.predictions(logits_2x2, [pred_key])
      self.evaluate(preds[pred_key])
    if tf.executing_eagerly():
      return

    # Dynamic shape only works in Graph mode.
    logits_placeholder = tf.compat.v1.placeholder(dtype=tf.dtypes.float32)
    spec = head.create_estimator_spec(
        features={'x': np.array((
            (30.,),
            (42.,),
        ))},
        mode=ModeKeys.PREDICT,
        logits=logits_placeholder,
        trainable_variables=[tf.Variable([1.0, 2.0], dtype=tf.dtypes.float32)])
    with self.cached_session():
      with self.assertRaisesRegexp(tf.errors.OpError, 'logits shape'):
        spec.predictions[pred_key].eval({logits_placeholder: logits_2x2})

  def test_invalid_labels_shape(self):
    n_classes = 3
    head = head_lib.MultiClassHead(n_classes)
    self.assertEqual(n_classes, head.logits_dimension)

    # Logits should be shape (batch_size, 3).
    # Labels should be shape (batch_size, 1).
    labels_2x2 = np.array((
        (1, 2),
        (0, 1),
    ), dtype=np.int)
    logits_2x3 = np.array((
        (1., 2., 3.),
        (1., 2., 3.),
    ))
    features = {'x': np.array(((42.,),))}

    # Static shape.
    with self.assertRaisesRegexp(ValueError, 'Mismatched label shape'):
      training_loss = head.loss(
          logits=logits_2x3,
          labels=labels_2x2,
          features=features,
          mode=ModeKeys.EVAL)
      self.evaluate(training_loss)
    if tf.executing_eagerly():
      return

    # Dynamic shape only works in Graph mode.
    labels_placeholder = tf.compat.v1.placeholder(dtype=tf.dtypes.int64)
    logits_placeholder = tf.compat.v1.placeholder(dtype=tf.dtypes.float32)
    training_loss = head.loss(
        logits=logits_placeholder,
        labels=labels_placeholder,
        features=features,
        mode=ModeKeys.EVAL)
    with self.cached_session():
      with self.assertRaisesRegexp(
          tf.errors.InvalidArgumentError,
          r'\[expected_labels_shape: \] \[2 1\] \[labels_shape: \] \[2 2\]'):
        training_loss.eval({
            logits_placeholder: logits_2x3,
            labels_placeholder: labels_2x2
        })

  def test_invalid_labels_type(self):
    n_classes = 3
    head = head_lib.MultiClassHead(n_classes)
    self.assertEqual(n_classes, head.logits_dimension)

    # Logits should be shape (batch_size, 3).
    # Labels should be shape (batch_size, 1).
    labels_2x1 = np.array((
        (1.,),
        (1.,),
    ))
    logits_2x3 = np.array((
        (1., 2., 3.),
        (1., 2., 3.),
    ))
    features = {'x': np.array(((42.,),))}

    # Static shape.
    with self.assertRaisesRegexp(ValueError, 'Labels dtype'):
      head.loss(
          logits=logits_2x3,
          labels=labels_2x1,
          features=features,
          mode=ModeKeys.EVAL)
    if tf.executing_eagerly():
      return

    # Dynamic shape only works in Graph mode.
    labels_placeholder = tf.compat.v1.placeholder(dtype=tf.dtypes.float32)
    logits_placeholder = tf.compat.v1.placeholder(dtype=tf.dtypes.float32)
    with self.assertRaisesRegexp(ValueError, 'Labels dtype'):
      head.loss(
          logits=logits_placeholder,
          labels=labels_placeholder,
          features=features,
          mode=ModeKeys.EVAL)

  def test_invalid_labels_values(self):
    n_classes = 3
    head = head_lib.MultiClassHead(n_classes)
    self.assertEqual(n_classes, head.logits_dimension)

    labels_2x1_with_large_id = np.array((
        (45,),
        (1,),
    ), dtype=np.int)
    labels_2x1_with_negative_id = np.array((
        (-5,),
        (1,),
    ), dtype=np.int)
    logits_2x3 = np.array((
        (1., 2., 4.),
        (1., 2., 3.),
    ))
    features = {'x': np.array(((42.,),))}

    if tf.executing_eagerly():
      with self.assertRaisesRegexp(ValueError, 'Labels must be <= 3 - 1'):
        training_loss = head.loss(
            logits=logits_2x3,
            labels=labels_2x1_with_large_id,
            features=features,
            mode=ModeKeys.EVAL)

      with self.assertRaisesRegexp(ValueError, 'Labels must be >= 0'):
        training_loss = head.loss(
            logits=logits_2x3,
            labels=labels_2x1_with_negative_id,
            features=features,
            mode=ModeKeys.EVAL)
      return

    # Dynamic shape only works in Graph mode.
    labels_placeholder = tf.compat.v1.placeholder(dtype=tf.dtypes.int64)
    logits_placeholder = tf.compat.v1.placeholder(dtype=tf.dtypes.float32)
    training_loss = head.loss(
        logits=logits_placeholder,
        labels=labels_placeholder,
        features=features,
        mode=ModeKeys.EVAL)
    with self.cached_session():
      with self.assertRaisesOpError('Labels must be <= n_classes - 1'):
        training_loss.eval({
            labels_placeholder: labels_2x1_with_large_id,
            logits_placeholder: logits_2x3
        })

    with self.cached_session():
      with self.assertRaisesOpError('Labels must be >= 0'):
        training_loss.eval({
            labels_placeholder: labels_2x1_with_negative_id,
            logits_placeholder: logits_2x3
        })

  def test_invalid_labels_sparse_tensor(self):
    n_classes = 3
    head = head_lib.MultiClassHead(n_classes)
    self.assertEqual(n_classes, head.logits_dimension)

    labels_2x1 = tf.sparse.SparseTensor(
        values=['english', 'italian'],
        indices=[[0, 0], [1, 0]],
        dense_shape=[2, 1])
    logits_2x3 = np.array((
        (1., 2., 4.),
        (1., 2., 3.),
    ))

    with self.assertRaisesRegexp(ValueError,
                                 'SparseTensor labels are not supported.'):
      loss = head.loss(
          logits=logits_2x3,
          labels=labels_2x1,
          features={'x': np.array(((42.,),))},
          mode=ModeKeys.EVAL)
      self.evaluate(loss)

  def test_incompatible_labels_shape(self):
    n_classes = 3
    head = head_lib.MultiClassHead(n_classes)
    self.assertEqual(n_classes, head.logits_dimension)

    # Logits should be shape (batch_size, 3).
    # Labels should be shape (batch_size, 1).
    # Here batch sizes are different.
    values_3x1 = np.array((
        (1,),
        (1,),
        (1,),
    ))
    values_2x3 = np.array((
        (1., 2., 3.),
        (1., 2., 3.),
    ))
    features = {'x': values_2x3}

    # Static shape.
    # Eager mode.
    if tf.executing_eagerly():
      with self.assertRaisesRegex(ValueError, 'labels shape'):
        head.loss(
            logits=values_2x3,
            labels=values_3x1,
            features=features,
            mode=ModeKeys.EVAL)
      return
    # Graph mode.
    with self.assertRaisesRegex(ValueError, r'shape.*\(3,\).*\(2, 3\)'):
      head.loss(
          logits=values_2x3,
          labels=values_3x1,
          features=features,
          mode=ModeKeys.EVAL)

    # Dynamic shape only works in Graph mode.
    labels_placeholder = tf.compat.v1.placeholder(dtype=tf.dtypes.int64)
    logits_placeholder = tf.compat.v1.placeholder(dtype=tf.dtypes.float32)
    training_loss = head.loss(
        logits=logits_placeholder,
        labels=labels_placeholder,
        features=features,
        mode=ModeKeys.EVAL)
    with self.cached_session():
      with self.assertRaisesRegex(
          tf.errors.InvalidArgumentError,
{'fraction_non_alphanumeric': 0.11603005353848922, 'fraction_numerical': 0.01943582129842084, 'mean_word_length': 2.864394993045897, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3020718', 'n_tokens_mistral': 7382, 'n_tokens_neox': 7108, 'n_words': 1440}