<commit_before># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""API for Tensorflow Model Analysis."""

# TODO(b/149126671): Put ValidationResultsWriter in a separate file.

import os
import tempfile
from typing import Any, Dict, Iterable, Iterator, List, Optional, Set, Union

from absl import logging
import apache_beam as beam
import pandas as pd
import pyarrow as pa
import tensorflow as tf
from tensorflow_model_analysis import constants
from tensorflow_model_analysis import types
from tensorflow_model_analysis.eval_saved_model import constants as eval_constants
from tensorflow_model_analysis.evaluators import evaluator
from tensorflow_model_analysis.evaluators import legacy_metrics_and_plots_evaluator
from tensorflow_model_analysis.evaluators import metrics_plots_and_validations_evaluator
from tensorflow_model_analysis.extractors import example_weights_extractor
from tensorflow_model_analysis.extractors import extractor
from tensorflow_model_analysis.extractors import features_extractor
from tensorflow_model_analysis.extractors import labels_extractor
from tensorflow_model_analysis.extractors import legacy_predict_extractor
from tensorflow_model_analysis.extractors import materialized_predictions_extractor
from tensorflow_model_analysis.extractors import predictions_extractor
from tensorflow_model_analysis.extractors import slice_key_extractor
from tensorflow_model_analysis.extractors import sql_slice_key_extractor
from tensorflow_model_analysis.extractors import tfjs_predict_extractor
from tensorflow_model_analysis.extractors import tflite_predict_extractor
from tensorflow_model_analysis.extractors import transformed_features_extractor
from tensorflow_model_analysis.extractors import unbatch_extractor
from tensorflow_model_analysis.post_export_metrics import post_export_metrics
from tensorflow_model_analysis.proto import config_pb2
from tensorflow_model_analysis.proto import metrics_for_slice_pb2
from tensorflow_model_analysis.proto import validation_result_pb2
from tensorflow_model_analysis.slicer import slicer_lib as slicer
from tensorflow_model_analysis.utils import config_util
from tensorflow_model_analysis.utils import model_util
from tensorflow_model_analysis.validators import validator
from tensorflow_model_analysis.view import util as view_util
from tensorflow_model_analysis.view import view_types
from tensorflow_model_analysis.writers import eval_config_writer
from tensorflow_model_analysis.writers import metrics_plots_and_validations_writer
from tensorflow_model_analysis.writers import writer
from tfx_bsl.arrow import table_util
from tfx_bsl.tfxio import raw_tf_record
from tfx_bsl.tfxio import tensor_adapter
from tfx_bsl.tfxio import tf_example_record

from tensorflow_metadata.proto.v0 import schema_pb2


def _assert_tensorflow_version():
  """Check that we're using a compatible TF version."""
  # Fail with a clear error in case we are not using a compatible TF version.
  major, minor, _ = tf.version.VERSION.split('.')
  if (int(major) not in (1, 2)) or (int(major) == 1 and int(minor) < 15):
    raise RuntimeError(
        'Tensorflow version >= 1.15, < 3 is required. Found (%s). Please '
        'install the latest 1.x or 2.x version from '
        'https://github.com/tensorflow/tensorflow. ' % tf.version.VERSION)
  if int(major) == 2:
    logging.warning(
        'Tensorflow version (%s) found. Note that TFMA support for TF 2.0 '
        'is currently in beta', tf.version.VERSION)


def _is_legacy_eval(
    config_version: Optional[int],
    eval_shared_model: Optional[types.MaybeMultipleEvalSharedModels],
    eval_config: Optional[config_pb2.EvalConfig]):
  """Returns True if legacy evaluation is being used.

  A legacy evaluation is an evalution that uses only a single EvalSharedModel,
  has no tags (or uses "eval" as its tag), and does not specify an eval_config
  The legacy evaluation is based on using add_metrics_callbacks to create a
  modified version of the graph saved with an EvalSavedModel. The newer version
  of evaluation supports both add_metrics_callbacks as well as metrics defined
  in MetricsSpecs inside of EvalConfig. The newer version works with both "eval"
  and serving models and also supports multi-model evaluation. This function is
  used by code to support backwards compatibility for callers that have not
  updated to use the new EvalConfig.

  Args:
    config_version: Optionally, An explicit version of the config determined
      elsewhere. This is used to handle cases where the provided eval_config was
      generated internally, and thus not a reliable indicator of user intent.
    eval_shared_model: Optionally, the model to be evaluated.
    eval_config: Optionally, an EvalConfig specifying v2 config.

  Returns:
    Whether the user inputs should trigger a legacy evaluation.
  """
  return ((config_version is not None and config_version == 1) or
          (eval_shared_model and not isinstance(eval_shared_model, dict) and
           not isinstance(eval_shared_model, list) and
           (not eval_shared_model.model_loader.tags or eval_constants.EVAL_TAG
            in eval_shared_model.model_loader.tags) and not eval_config))


def _default_eval_config(eval_shared_models: List[types.EvalSharedModel],
                         slice_spec: Optional[List[slicer.SingleSliceSpec]],
                         write_config: Optional[bool],
                         compute_confidence_intervals: Optional[bool],
                         min_slice_size: int):
  """Creates default EvalConfig (for use in legacy evaluations)."""
  model_specs = []
  for shared_model in eval_shared_models:
    example_weight_key = shared_model.example_weight_key
    example_weight_keys = {}
    if example_weight_key and isinstance(example_weight_key, dict):
      example_weight_keys = example_weight_key
      example_weight_key = ''
    model_specs.append(
        config_pb2.ModelSpec(
            name=shared_model.model_name,
            example_weight_key=example_weight_key,
            example_weight_keys=example_weight_keys))
  slicing_specs = None
  if slice_spec:
    slicing_specs = [s.to_proto() for s in slice_spec]
  options = config_pb2.Options()
  options.compute_confidence_intervals.value = compute_confidence_intervals
  options.min_slice_size.value = min_slice_size
  if not write_config:
    options.disabled_outputs.values.append(eval_config_writer.EVAL_CONFIG_FILE)
  return config_pb2.EvalConfig(
      model_specs=model_specs, slicing_specs=slicing_specs, options=options)


def _model_types(
    eval_shared_model: Optional[types.MaybeMultipleEvalSharedModels]
) -> Optional[Set[str]]:
  """Returns model types associated with given EvalSharedModels."""
  eval_shared_models = model_util.verify_and_update_eval_shared_models(
      eval_shared_model)
  if not eval_shared_models:
    return None
  else:
    return set([m.model_type for m in eval_shared_models])


def _update_eval_config_with_defaults(
    eval_config: config_pb2.EvalConfig,
    eval_shared_model: Optional[types.MaybeMultipleEvalSharedModels]
) -> config_pb2.EvalConfig:
  """Returns updated eval config with default values."""
  eval_shared_models = model_util.verify_and_update_eval_shared_models(
      eval_shared_model)
  has_baseline = eval_shared_models and len(eval_shared_models) == 2
  return config_util.update_eval_config_with_defaults(
      eval_config=eval_config,
      has_baseline=has_baseline,
      rubber_stamp=model_util.has_rubber_stamp(eval_shared_models))


def _get_extract_num_bytes(extract: types.Extracts) -> int:
  """Returns the number of bytes in the input."""
  if constants.ARROW_RECORD_BATCH_KEY in extract:
    return extract[constants.ARROW_RECORD_BATCH_KEY].nbytes
  if constants.INPUT_KEY in extract:
    if isinstance(extract[constants.INPUT_KEY], bytes):
      return len(extract[constants.INPUT_KEY])
  logging.warning('Failed to extract number of input bytes.')
  return 0


def _increment_counter(counter_name: str, value: int) -> int:
  """Increments the specified counter by the value."""
  counter = beam.metrics.Metrics.counter(constants.METRICS_NAMESPACE,
                                         counter_name)
  counter.inc(value)
  return value
<commit_msg>Allow providing multiple candidate models for a single baseline.<commit_after># Copyright 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""API for Tensorflow Model Analysis."""

# TODO(b/149126671): Put ValidationResultsWriter in a separate file.

import os
import tempfile
from typing import Any, Dict, Iterable, Iterator, List, Optional, Set, Union

from absl import logging
import apache_beam as beam
import pandas as pd
import pyarrow as pa
import tensorflow as tf
from tensorflow_model_analysis import constants
from tensorflow_model_analysis import types
from tensorflow_model_analysis.eval_saved_model import constants as eval_constants
from tensorflow_model_analysis.evaluators import evaluator
from tensorflow_model_analysis.evaluators import legacy_metrics_and_plots_evaluator
from tensorflow_model_analysis.evaluators import metrics_plots_and_validations_evaluator
from tensorflow_model_analysis.extractors import example_weights_extractor
from tensorflow_model_analysis.extractors import extractor
from tensorflow_model_analysis.extractors import features_extractor
from tensorflow_model_analysis.extractors import labels_extractor
from tensorflow_model_analysis.extractors import legacy_predict_extractor
from tensorflow_model_analysis.extractors import materialized_predictions_extractor
from tensorflow_model_analysis.extractors import predictions_extractor
from tensorflow_model_analysis.extractors import slice_key_extractor
from tensorflow_model_analysis.extractors import sql_slice_key_extractor
from tensorflow_model_analysis.extractors import tfjs_predict_extractor
from tensorflow_model_analysis.extractors import tflite_predict_extractor
from tensorflow_model_analysis.extractors import transformed_features_extractor
from tensorflow_model_analysis.extractors import unbatch_extractor
from tensorflow_model_analysis.post_export_metrics import post_export_metrics
from tensorflow_model_analysis.proto import config_pb2
from tensorflow_model_analysis.proto import metrics_for_slice_pb2
from tensorflow_model_analysis.proto import validation_result_pb2
from tensorflow_model_analysis.slicer import slicer_lib as slicer
from tensorflow_model_analysis.utils import config_util
from tensorflow_model_analysis.utils import model_util
from tensorflow_model_analysis.validators import validator
from tensorflow_model_analysis.view import util as view_util
from tensorflow_model_analysis.view import view_types
from tensorflow_model_analysis.writers import eval_config_writer
from tensorflow_model_analysis.writers import metrics_plots_and_validations_writer
from tensorflow_model_analysis.writers import writer
from tfx_bsl.arrow import table_util
from tfx_bsl.tfxio import raw_tf_record
from tfx_bsl.tfxio import tensor_adapter
from tfx_bsl.tfxio import tf_example_record

from tensorflow_metadata.proto.v0 import schema_pb2


def _assert_tensorflow_version():
  """Check that we're using a compatible TF version."""
  # Fail with a clear error in case we are not using a compatible TF version.
  major, minor, _ = tf.version.VERSION.split('.')
  if (int(major) not in (1, 2)) or (int(major) == 1 and int(minor) < 15):
    raise RuntimeError(
        'Tensorflow version >= 1.15, < 3 is required. Found (%s). Please '
        'install the latest 1.x or 2.x version from '
        'https://github.com/tensorflow/tensorflow. ' % tf.version.VERSION)
  if int(major) == 2:
    logging.warning(
        'Tensorflow version (%s) found. Note that TFMA support for TF 2.0 '
        'is currently in beta', tf.version.VERSION)


def _is_legacy_eval(
    config_version: Optional[int],
    eval_shared_model: Optional[types.MaybeMultipleEvalSharedModels],
    eval_config: Optional[config_pb2.EvalConfig]):
  """Returns True if legacy evaluation is being used.

  A legacy evaluation is an evalution that uses only a single EvalSharedModel,
  has no tags (or uses "eval" as its tag), and does not specify an eval_config
  The legacy evaluation is based on using add_metrics_callbacks to create a
  modified version of the graph saved with an EvalSavedModel. The newer version
  of evaluation supports both add_metrics_callbacks as well as metrics defined
  in MetricsSpecs inside of EvalConfig. The newer version works with both "eval"
  and serving models and also supports multi-model evaluation. This function is
  used by code to support backwards compatibility for callers that have not
  updated to use the new EvalConfig.

  Args:
    config_version: Optionally, An explicit version of the config determined
      elsewhere. This is used to handle cases where the provided eval_config was
      generated internally, and thus not a reliable indicator of user intent.
    eval_shared_model: Optionally, the model to be evaluated.
    eval_config: Optionally, an EvalConfig specifying v2 config.

  Returns:
    Whether the user inputs should trigger a legacy evaluation.
  """
  return ((config_version is not None and config_version == 1) or
          (eval_shared_model and not isinstance(eval_shared_model, dict) and
           not isinstance(eval_shared_model, list) and
           (not eval_shared_model.model_loader.tags or eval_constants.EVAL_TAG
            in eval_shared_model.model_loader.tags) and not eval_config))


def _default_eval_config(eval_shared_models: List[types.EvalSharedModel],
                         slice_spec: Optional[List[slicer.SingleSliceSpec]],
                         write_config: Optional[bool],
                         compute_confidence_intervals: Optional[bool],
                         min_slice_size: int):
  """Creates default EvalConfig (for use in legacy evaluations)."""
  model_specs = []
  for shared_model in eval_shared_models:
    example_weight_key = shared_model.example_weight_key
    example_weight_keys = {}
    if example_weight_key and isinstance(example_weight_key, dict):
      example_weight_keys = example_weight_key
      example_weight_key = ''
    model_specs.append(
        config_pb2.ModelSpec(
            name=shared_model.model_name,
            example_weight_key=example_weight_key,
            example_weight_keys=example_weight_keys))
  slicing_specs = None
  if slice_spec:
    slicing_specs = [s.to_proto() for s in slice_spec]
  options = config_pb2.Options()
  options.compute_confidence_intervals.value = compute_confidence_intervals
  options.min_slice_size.value = min_slice_size
  if not write_config:
    options.disabled_outputs.values.append(eval_config_writer.EVAL_CONFIG_FILE)
  return config_pb2.EvalConfig(
      model_specs=model_specs, slicing_specs=slicing_specs, options=options)


def _model_types(
    eval_shared_model: Optional[types.MaybeMultipleEvalSharedModels]
) -> Optional[Set[str]]:
  """Returns model types associated with given EvalSharedModels."""
  eval_shared_models = model_util.verify_and_update_eval_shared_models(
      eval_shared_model)
  if not eval_shared_models:
    return None
  else:
    return set([m.model_type for m in eval_shared_models])


def _update_eval_config_with_defaults(
    eval_config: config_pb2.EvalConfig,
    eval_shared_model: Optional[types.MaybeMultipleEvalSharedModels]
) -> config_pb2.EvalConfig:
  """Returns updated eval config with default values."""
  eval_shared_models = model_util.verify_and_update_eval_shared_models(
      eval_shared_model)
  has_baseline = eval_shared_models and len(eval_shared_models) > 1
  return config_util.update_eval_config_with_defaults(
      eval_config=eval_config,
      has_baseline=has_baseline,
      rubber_stamp=model_util.has_rubber_stamp(eval_shared_models))


def _get_extract_num_bytes(extract: types.Extracts) -> int:
  """Returns the number of bytes in the input."""
  if constants.ARROW_RECORD_BATCH_KEY in extract:
    return extract[constants.ARROW_RECORD_BATCH_KEY].nbytes
  if constants.INPUT_KEY in extract:
    if isinstance(extract[constants.INPUT_KEY], bytes):
      return len(extract[constants.INPUT_KEY])
  logging.warning('Failed to extract number of input bytes.')
  return 0


def _increment_counter(counter_name: str, value: int) -> int:
  """Increments the specified counter by the value."""
  counter = beam.metrics.Metrics.counter(constants.METRICS_NAMESPACE,
                                         counter_name)
  counter.inc(value)
  return value

{'fraction_non_alphanumeric': 0.05584045584045584, 'fraction_numerical': 0.005242165242165242, 'mean_word_length': 4.929391891891892, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 14, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1362683', 'n_tokens_mistral': 4915, 'n_tokens_neox': 4828, 'n_words': 1490}