<commit_before># Copyright 2017 Google LLC.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
#    contributors may be used to endorse or promote products derived from this
#    software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
"""Evaluates a DeepVariant model during training.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import sys
if 'google' in sys.modules and 'google.protobuf' not in sys.modules:
  del sys.modules['google']


import json
import os



from absl import flags
from absl import logging
import tensorflow as tf

from third_party.nucleus.util import proto_utils
from tensorflow.python.framework.errors_impl import NotFoundError
from deepvariant import data_providers
from deepvariant import logging_level
from deepvariant import modeling
from deepvariant import tf_utils


slim = tf.contrib.slim
FLAGS = flags.FLAGS

increasing_metrics = [
    'Accuracy/All', 'Precision/All', 'Recall/All', 'FPs/All', 'FNs/All',
    'TPs/All', 'TNs/All', 'Recall/Het', 'Recall/HomVar', 'Precision/Het',
    'Precision/HomVar', 'F1/All'
]
decreasing_metrics = ['loss']

flags.DEFINE_integer('batch_size', 1024, 'The number of samples in each batch.')

# Cloud TPU Cluster Resolvers
flags.DEFINE_string(
    'gcp_project', None,
    'Project name for the Cloud TPU-enabled project. If not specified, we '
    'will attempt to automatically detect the GCE project from metadata.')
flags.DEFINE_string(
    'tpu_zone', None,
    'GCE zone where the Cloud TPU is located in. If not specified, we '
    'will attempt to automatically detect the GCE project from metadata.')
flags.DEFINE_string(
    'tpu_name', None,
    'Name of the Cloud TPU for Cluster Resolvers. You must specify either '
    'this flag or --master.')

flags.DEFINE_string(
    'master', None,
    'GRPC URL of the master (e.g. grpc://ip.address.of.tpu:8470). You '
    'must specify either this flag or --tpu_name.')

flags.DEFINE_string('checkpoint_dir', '/tmp/deepvariant/',
                    'Directory where the model was written to.')

flags.DEFINE_string(
    'eval_name', None,
    'Name of the evaluation if user needs to run multiple evaluations on '
    'different data sets, such as on training data vs test data. Metrics for '
    'different evaluations are saved in separate directories, and appear '
    'separately in tensorboard.  The directory will be named "eval_"+eval_name')

flags.DEFINE_string(
    'eval_dir', None,
    'This is used only to generate eval_name, if that is not provided.')

flags.DEFINE_integer('min_eval_interval_s', 180,
                     'Minimum seconds between evaluations.')

flags.DEFINE_integer(
    'eval_timeout', 10000,
    'Maximum seconds between checkpoints before evaluation '
    'terminates.')

flags.DEFINE_integer('max_evaluations', None,
                     'Max number of batches to evaluate')

flags.DEFINE_integer('max_examples', 1000000,
                     'Maximum number of examples to evaluate.')

flags.DEFINE_string('model_name', 'inception_v3',
                    'The name of the model to use for predictions.')

flags.DEFINE_string('dataset_config_pbtxt', None,
                    'The path to the dataset config file.')

flags.DEFINE_boolean('use_tpu', False, 'use tpu if available')

flags.DEFINE_string(
    'kmp_blocktime', '0',
    'Value to set the KMP_BLOCKTIME environment variable to for efficient MKL '
    'evaluation. See https://www.tensorflow.org/performance/performance_guide '
    'for more information. The default value is 0, which provides the best '
    'performance in our tests. Set this flag to "" to not set the variable.')

flags.DEFINE_enum('best_checkpoint_metric', 'F1/All',
                  increasing_metrics + decreasing_metrics,
                  'The metric for measuring the best checkpoint.')


def main(_):
  proto_utils.uses_fast_cpp_protos_or_die()

  if not FLAGS.dataset_config_pbtxt:
    logging.error('Need to specify --dataset_config_pbtxt')
  logging_level.set_from_flag()<commit_msg>Set the num_examples of model_eval.py to None by default so that all examples are evaluated by default.<commit_after># Copyright 2017 Google LLC.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# 3. Neither the name of the copyright holder nor the names of its
#    contributors may be used to endorse or promote products derived from this
#    software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
"""Evaluates a DeepVariant model during training.
"""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import sys
if 'google' in sys.modules and 'google.protobuf' not in sys.modules:
  del sys.modules['google']


import json
import os



from absl import flags
from absl import logging
import tensorflow as tf

from third_party.nucleus.util import proto_utils
from tensorflow.python.framework.errors_impl import NotFoundError
from deepvariant import data_providers
from deepvariant import logging_level
from deepvariant import modeling
from deepvariant import tf_utils


slim = tf.contrib.slim
FLAGS = flags.FLAGS

increasing_metrics = [
    'Accuracy/All', 'Precision/All', 'Recall/All', 'FPs/All', 'FNs/All',
    'TPs/All', 'TNs/All', 'Recall/Het', 'Recall/HomVar', 'Precision/Het',
    'Precision/HomVar', 'F1/All'
]
decreasing_metrics = ['loss']

flags.DEFINE_integer('batch_size', 1024, 'The number of samples in each batch.')

# Cloud TPU Cluster Resolvers
flags.DEFINE_string(
    'gcp_project', None,
    'Project name for the Cloud TPU-enabled project. If not specified, we '
    'will attempt to automatically detect the GCE project from metadata.')
flags.DEFINE_string(
    'tpu_zone', None,
    'GCE zone where the Cloud TPU is located in. If not specified, we '
    'will attempt to automatically detect the GCE project from metadata.')
flags.DEFINE_string(
    'tpu_name', None,
    'Name of the Cloud TPU for Cluster Resolvers. You must specify either '
    'this flag or --master.')

flags.DEFINE_string(
    'master', None,
    'GRPC URL of the master (e.g. grpc://ip.address.of.tpu:8470). You '
    'must specify either this flag or --tpu_name.')

flags.DEFINE_string('checkpoint_dir', '/tmp/deepvariant/',
                    'Directory where the model was written to.')

flags.DEFINE_string(
    'eval_name', None,
    'Name of the evaluation if user needs to run multiple evaluations on '
    'different data sets, such as on training data vs test data. Metrics for '
    'different evaluations are saved in separate directories, and appear '
    'separately in tensorboard.  The directory will be named "eval_"+eval_name')

flags.DEFINE_string(
    'eval_dir', None,
    'This is used only to generate eval_name, if that is not provided.')

flags.DEFINE_integer('min_eval_interval_s', 180,
                     'Minimum seconds between evaluations.')

flags.DEFINE_integer(
    'eval_timeout', 10000,
    'Maximum seconds between checkpoints before evaluation '
    'terminates.')

flags.DEFINE_integer('max_evaluations', None,
                     'Max number of batches to evaluate')

flags.DEFINE_integer(
    'max_examples', None,
    'Maximum number of examples to evaluate. Set to None (default) to evaluate '
    'all examples. If not None, must be a positive integer and at most '
    '`n_examples // max_example batches` will be evaluated.')

flags.DEFINE_string('model_name', 'inception_v3',
                    'The name of the model to use for predictions.')

flags.DEFINE_string('dataset_config_pbtxt', None,
                    'The path to the dataset config file.')

flags.DEFINE_boolean('use_tpu', False, 'use tpu if available')

flags.DEFINE_string(
    'kmp_blocktime', '0',
    'Value to set the KMP_BLOCKTIME environment variable to for efficient MKL '
    'evaluation. See https://www.tensorflow.org/performance/performance_guide '
    'for more information. The default value is 0, which provides the best '
    'performance in our tests. Set this flag to "" to not set the variable.')

flags.DEFINE_enum('best_checkpoint_metric', 'F1/All',
                  increasing_metrics + decreasing_metrics,
                  'The metric for measuring the best checkpoint.')


def main(_):
  proto_utils.uses_fast_cpp_protos_or_die()

  if not FLAGS.dataset_config_pbtxt:
    logging.error('Need to specify --dataset_config_pbtxt')
  logging_level.set_from_flag()
{'fraction_non_alphanumeric': 0.078039430449069, 'fraction_numerical': 0.005750273822562979, 'mean_word_length': 3.9289248762932973, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5492184', 'n_tokens_mistral': 3222, 'n_tokens_neox': 2942, 'n_words': 1247}