<commit_before># coding=utf-8
# Copyright 2022 The TensorFlow Datasets Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Benchmark utils."""

import dataclasses
import statistics
import textwrap
import time
from typing import Any, Dict, Iterable, List, Optional, Union

from absl import logging
from tensorflow_datasets.core.utils import tqdm_utils

try:
  import pandas as pd  # pylint: disable=g-import-not-at-top
except ImportError:
  pd = Any

# pylint: disable=logging-format-interpolation

StatDict = Dict[str, Union[int, float]]


def _ns_to_s(time_in_ns: int) -> float:
  return time_in_ns / 1e9


@dataclasses.dataclass(frozen=True)
class RawBenchmarkResult:
  """Raw results of running the benchmark.

  Attributes:
    num_iter: the number of iterations over examples that were done.
    num_examples: the number of examples that were processed in these
      iterations. Note that when examples are batched, then one iteration
      processes multiple examples.
    start_time: the time (in ns) at which when the benchmark started.
    first_batch_time: the time (in ns) at which the first iteration was
      processed.
    end_time: the time (in ns) at which the benchmark ended.
    batch_size: the number of examples in each iteration.
    durations_ns: the duration in ns of each iteration that was processed.
  """
  num_iter: int
  num_examples: int
  start_time: int
  first_batch_time: int
  end_time: int
  batch_size: int
  durations_ns: Optional[List[int]] = None

  def examples(self, include_first: bool = True) -> int:
    if include_first:
      return self.num_examples
    return self.num_examples - 1

  def total_time_s(self, include_first: bool = True) -> float:
    if include_first:
      return _ns_to_s(self.end_time - self.start_time)
    return _ns_to_s(self.end_time - self.first_batch_time)

  def examples_per_second(self, include_first: bool = True) -> float:
    return self.examples(include_first) / self.total_time_s(include_first)

  def time_until_first(self, include_first: bool = True) -> Optional[float]:
    """Time in seconds that it took to load the first example."""
    if include_first:
      return _ns_to_s(self.first_batch_time - self.start_time)
    if len(self.durations_ns) > 1:
      return _ns_to_s(self.durations_ns[1])
    return None

  def durations_s(self, include_first: bool = True) -> List[float]:
    if not include_first:
      return [_ns_to_s(d) for d in self.durations_ns[1:]]
    return [_ns_to_s(d) for d in self.durations_ns]

  def summary_statistics(
      self, include_first: bool) -> Dict[str, Union[float, List[float]]]:
    if self.durations_ns is None:
      return {}
    durations = self.durations_s(include_first)
    return {
        'mean': statistics.mean(durations),
        'variance': statistics.pvariance(durations),<commit_msg>Add more checks for None<commit_after># coding=utf-8
# Copyright 2022 The TensorFlow Datasets Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Benchmark utils."""

import dataclasses
import statistics
import textwrap
import time
from typing import Any, Dict, Iterable, List, Optional, Union

from absl import logging
from tensorflow_datasets.core.utils import tqdm_utils

try:
  import pandas as pd  # pylint: disable=g-import-not-at-top
except ImportError:
  pd = Any

# pylint: disable=logging-format-interpolation

StatDict = Dict[str, Union[int, float]]


def _ns_to_s(time_in_ns: int) -> float:
  return time_in_ns / 1e9


@dataclasses.dataclass(frozen=True)
class RawBenchmarkResult:
  """Raw results of running the benchmark.

  Attributes:
    num_iter: the number of iterations over examples that were done.
    num_examples: the number of examples that were processed in these
      iterations. Note that when examples are batched, then one iteration
      processes multiple examples.
    start_time: the time (in ns) at which when the benchmark started.
    first_batch_time: the time (in ns) at which the first iteration was
      processed.
    end_time: the time (in ns) at which the benchmark ended.
    batch_size: the number of examples in each iteration.
    durations_ns: the duration in ns of each iteration that was processed.
  """
  num_iter: int
  num_examples: int
  start_time: int
  first_batch_time: int
  end_time: int
  batch_size: int
  durations_ns: Optional[List[int]] = None

  def examples(self, include_first: bool = True) -> int:
    if include_first:
      return self.num_examples
    return self.num_examples - 1

  def total_time_s(self, include_first: bool = True) -> float:
    if include_first:
      return _ns_to_s(self.end_time - self.start_time)
    return _ns_to_s(self.end_time - self.first_batch_time)

  def examples_per_second(self, include_first: bool = True) -> float:
    return self.examples(include_first) / self.total_time_s(include_first)

  def time_until_first(self, include_first: bool = True) -> Optional[float]:
    """Time in seconds that it took to load the first example."""
    if include_first:
      return _ns_to_s(self.first_batch_time - self.start_time)
    if self.durations_ns is not None and len(self.durations_ns) > 1:
      return _ns_to_s(self.durations_ns[1])
    return None

  def durations_s(self, include_first: bool = True) -> List[float]:
    if not include_first:
      return [_ns_to_s(d) for d in self.durations_ns[1:]]
    return [_ns_to_s(d) for d in self.durations_ns]

  def summary_statistics(
      self, include_first: bool) -> Dict[str, Union[float, List[float]]]:
    if self.durations_ns is None:
      return {}
    durations = self.durations_s(include_first)
    return {
        'mean': statistics.mean(durations),
        'variance': statistics.pvariance(durations),
{'fraction_non_alphanumeric': 0.08232842042578532, 'fraction_numerical': 0.0044662795891022775, 'mean_word_length': 3.91441111923921, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '814369', 'n_tokens_mistral': 2109, 'n_tokens_neox': 2044, 'n_words': 714}