"""simple docstring"""


import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


UpperCAmelCase__     						=  datasets.utils.logging.get_logger(__name__)


@dataclass
class 				a (						datasets.BuilderConfig      ):
    _snake_case   :     int   							=							1_00_00
    _snake_case   :     Optional[List[str]]   							=							None
    _snake_case   :     Optional[datasets.Features]   							=							None



class 				a (						datasets.ArrowBasedBuilder      ):
    _snake_case   :     Optional[Any]   							=							ParquetConfig


    def    lowerCAmelCase_     (    self		:     Optional[int]    ):
      return datasets.DatasetInfo(features=self.config.features    )


    def    lowerCAmelCase_     (    self		:     Optional[Any]      , __lowerCAmelCase		:     str    ):
      if not self.config.data_files:
        raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}'''    )
      _UpperCAmelCase         =					dl_manager.download_and_extract(self.config.data_files    )
      if isinstance(__lowerCAmelCase      , (str, list, tuple)    ):
        _UpperCAmelCase         =					data_files
        if isinstance(__lowerCAmelCase      , __lowerCAmelCase    ):
          _UpperCAmelCase         =					[files]
        # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
        _UpperCAmelCase         =					[dl_manager.iter_files(__lowerCAmelCase    ) for file in files]
        return [datasets.SplitGenerator(name=datasets.Split.TRAIN      , gen_kwargs={"""files""": files}    )]
      _UpperCAmelCase         =					[]
      for split_name, files in data_files.items():
        if isinstance(__lowerCAmelCase      , __lowerCAmelCase    ):
          _UpperCAmelCase         =					[files]
        # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
        _UpperCAmelCase         =					[dl_manager.iter_files(__lowerCAmelCase    ) for file in files]
        # Infer features is they are stoed in the arrow schema
        if self.info.features is None:
          for file in itertools.chain.from_iterable(__lowerCAmelCase    ):
            with open(__lowerCAmelCase      , """rb"""    ) as f:
              _UpperCAmelCase         =					datasets.Features.from_arrow_schema(pq.read_schema(__lowerCAmelCase    )    )
            break
        splits.append(datasets.SplitGenerator(name=__lowerCAmelCase      , gen_kwargs={"""files""": files}    )    )
      return splits


    def    lowerCAmelCase_     (    self		:     Optional[Any]      , __lowerCAmelCase		:     pa.Table    ):
      if self.info.features is not None:
        # more expensive cast to support nested features with keys in a different order
        # allows str <-> int/float or str to Audio for example
        _UpperCAmelCase         =					table_cast(__lowerCAmelCase      , self.info.features.arrow_schema    )
      return pa_table


    def    lowerCAmelCase_     (    self		:     Any      , __lowerCAmelCase		:     Optional[int]    ):
      _UpperCAmelCase         =					self.info.features.arrow_schema if self.info.features is not None else None
      if self.info.features is not None and self.config.columns is not None:
        if sorted(field.name for field in schema    ) != sorted(self.config.columns    ):
          raise ValueError(
              f'''Tried to load parquet data with columns \'{self.config.columns}\' with mismatching features \'{self.info.features}\''''    )
      for file_idx, file in enumerate(itertools.chain.from_iterable(__lowerCAmelCase    )    ):
        with open(__lowerCAmelCase      , """rb"""    ) as f:
          _UpperCAmelCase         =					pq.ParquetFile(__lowerCAmelCase    )
          try:
            for batch_idx, record_batch in enumerate(
                parquet_file.iter_batches(batch_size=self.config.batch_size      , columns=self.config.columns    )    ):
              _UpperCAmelCase         =					pa.Table.from_batches([record_batch]    )
              # Uncomment for debugging (will print the Arrow table size and elements)
              # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
              # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
              yield f'''{file_idx}_{batch_idx}''', self._cast_table(__lowerCAmelCase    )
          except ValueError as e:
            logger.error(f'''Failed to read file \'{file}\' with error {type(__lowerCAmelCase    )}: {e}'''    )
            raise

"""simple docstring"""


import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
    import tensorflow as tf
    from tensorflow.python.framework.errors_impl import ResourceExhaustedError

    from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
    import pyanvml.pyanvml as nvml

UpperCAmelCase__     						=  logging.get_logger(__name__)






def 			__UpperCAmelCase    (	lowercase      ,lowercase						):







  """simple docstring"""
  def run_func(lowercase						):
    @wraps(lowercase						)
    def run_in_eager_mode(*lowercase      ,**lowercase						):
      return func(*lowercase      ,**lowercase						)

    @wraps(lowercase						)
    @tf.function(experimental_compile=lowercase						)
    def run_in_graph_mode(*lowercase      ,**lowercase						):
      return func(*lowercase      ,**lowercase						)

    if do_eager_mode is True:
      if use_xla is not False:
        raise ValueError(
            """Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`."""						)
      return run_in_eager_mode
    else:
      return run_in_graph_mode

  return run_func






def 			__UpperCAmelCase    (	lowercase      ,lowercase      ,lowercase						):







  """simple docstring"""
  _UpperCAmelCase         =					random.Random()
  _UpperCAmelCase         =					[rng.randint(0      ,vocab_size - 1						) for i in range(batch_size * sequence_length						)]
  return tf.constant(lowercase      ,shape=(batch_size, sequence_length)      ,dtype=tf.intaa						)


class 				a (						lowerCAmelCase_      ):
    _snake_case   :     TensorFlowBenchmarkArguments
    _snake_case   :     PretrainedConfig
    _snake_case   :     str   							=							"TensorFlow"


    @property
    def    lowerCAmelCase_     (    self		:     Union[str, Any]    ):
      return tf.__version__


    def    lowerCAmelCase_     (    self		:     List[str]      , __lowerCAmelCase		:     str      , __lowerCAmelCase		:     int      , __lowerCAmelCase		:     int    ):
      # initialize GPU on separate process
      _UpperCAmelCase         =					self.args.strategy
      if strategy is None:
        raise ValueError("""A device strategy has to be initialized before using TensorFlow."""    )
      _UpperCAmelCase         =					self._prepare_inference_func(__lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase    )
      return self._measure_speed(_inference    )


    def    lowerCAmelCase_     (    self		:     Optional[int]      , __lowerCAmelCase		:     str      , __lowerCAmelCase		:     int      , __lowerCAmelCase		:     int    ):
      _UpperCAmelCase         =					self.args.strategy
      if strategy is None:
        raise ValueError("""A device strategy has to be initialized before using TensorFlow."""    )
      _UpperCAmelCase         =					self._prepare_train_func(__lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase    )
      return self._measure_speed(_train    )


    def    lowerCAmelCase_     (    self		:     Optional[Any]      , __lowerCAmelCase		:     str      , __lowerCAmelCase		:     int      , __lowerCAmelCase		:     int    ):
      # initialize GPU on separate process
      if self.args.is_gpu:
        tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]      , __lowerCAmelCase    )
      _UpperCAmelCase         =					self.args.strategy
      if strategy is None:
        raise ValueError("""A device strategy has to be initialized before using TensorFlow."""    )
      _UpperCAmelCase         =					self._prepare_inference_func(__lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase    )
      return self._measure_memory(_inference    )


    def    lowerCAmelCase_     (    self		:     str      , __lowerCAmelCase		:     str      , __lowerCAmelCase		:     int      , __lowerCAmelCase		:     int    ):
      if self.args.is_gpu:
        tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]      , __lowerCAmelCase    )
      _UpperCAmelCase         =					self.args.strategy
      if strategy is None:
        raise ValueError("""A device strategy has to be initialized before using TensorFlow."""    )

      _UpperCAmelCase         =					self._prepare_train_func(__lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase    )
      return self._measure_memory(_train    )


    def    lowerCAmelCase_     (    self		:     List[Any]      , __lowerCAmelCase		:     str      , __lowerCAmelCase		:     int      , __lowerCAmelCase		:     int    ):
      _UpperCAmelCase         =					self.config_dict[model_name]

      if self.args.fpaa:
        raise NotImplementedError("""Mixed precision is currently not supported."""    )

      _UpperCAmelCase         =					(
          hasattr(__lowerCAmelCase      , """architectures"""    )
          and isinstance(config.architectures      , __lowerCAmelCase    )
          and len(config.architectures    ) > 0
      )
      if not self.args.only_pretrain_model and has_model_class_in_config:
        try:
          _UpperCAmelCase         =					"""TF""" + config.architectures[0]  # prepend 'TF' for tensorflow model
          _UpperCAmelCase         =					__import__("""transformers"""      , fromlist=[model_class]    )
          _UpperCAmelCase         =					getattr(__lowerCAmelCase      , __lowerCAmelCase    )
          _UpperCAmelCase         =					model_cls(__lowerCAmelCase    )
        except ImportError:
          raise ImportError(
              f'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
              """ set `--only_pretrain_model` or `args.only_pretrain_model=True`."""    )
      else:
        _UpperCAmelCase         =					TF_MODEL_MAPPING[config.__class__](__lowerCAmelCase    )

      # encoder-decoder has vocab size saved differently
      _UpperCAmelCase         =					config.vocab_size if hasattr(__lowerCAmelCase      , """vocab_size"""    ) else config.encoder.vocab_size
      _UpperCAmelCase         =					random_input_ids(__lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase    )

      @run_with_tf_optimizations(self.args.eager_mode      , self.args.use_xla    )
      def encoder_decoder_forward():
        return model(__lowerCAmelCase      , decoder_input_ids=__lowerCAmelCase      , training=__lowerCAmelCase    )

      @run_with_tf_optimizations(self.args.eager_mode      , self.args.use_xla    )
      def encoder_forward():
        return model(__lowerCAmelCase      , training=__lowerCAmelCase    )

      _UpperCAmelCase         =					encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

      return _inference


    def    lowerCAmelCase_     (    self		:     List[str]      , __lowerCAmelCase		:     str      , __lowerCAmelCase		:     int      , __lowerCAmelCase		:     int    ):
      _UpperCAmelCase         =					self.config_dict[model_name]

      if self.args.eager_mode is not False:
        raise ValueError("""Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`."""    )

      if self.args.fpaa:
        raise NotImplementedError("""Mixed precision is currently not supported."""    )

      _UpperCAmelCase         =					(
          hasattr(__lowerCAmelCase      , """architectures"""    )
          and isinstance(config.architectures      , __lowerCAmelCase    )
          and len(config.architectures    ) > 0
      )
      if not self.args.only_pretrain_model and has_model_class_in_config:
        try:
          _UpperCAmelCase         =					"""TF""" + config.architectures[0]  # prepend 'TF' for tensorflow model
          _UpperCAmelCase         =					__import__("""transformers"""      , fromlist=[model_class]    )
          _UpperCAmelCase         =					getattr(__lowerCAmelCase      , __lowerCAmelCase    )
          _UpperCAmelCase         =					model_cls(__lowerCAmelCase    )
        except ImportError:
          raise ImportError(
              f'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
              """ set `--only_pretrain_model` or `args.only_pretrain_model=True`."""    )
      else:
        _UpperCAmelCase         =					TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](__lowerCAmelCase    )

      # encoder-decoder has vocab size saved differently
      _UpperCAmelCase         =					config.vocab_size if hasattr(__lowerCAmelCase      , """vocab_size"""    ) else config.encoder.vocab_size
      _UpperCAmelCase         =					random_input_ids(__lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase    )

      @run_with_tf_optimizations(self.args.eager_mode      , self.args.use_xla    )
      def encoder_decoder_train():
        _UpperCAmelCase         =					model(__lowerCAmelCase      , decoder_input_ids=__lowerCAmelCase      , labels=__lowerCAmelCase      , training=__lowerCAmelCase    )[0]
        _UpperCAmelCase         =					tf.gradients(__lowerCAmelCase      , model.trainable_variables    )
        return gradients

      @run_with_tf_optimizations(self.args.eager_mode      , self.args.use_xla    )
      def encoder_train():
        _UpperCAmelCase         =					model(__lowerCAmelCase      , labels=__lowerCAmelCase      , training=__lowerCAmelCase    )[0]
        _UpperCAmelCase         =					tf.gradients(__lowerCAmelCase      , model.trainable_variables    )
        return gradients

      _UpperCAmelCase         =					encoder_decoder_train if config.is_encoder_decoder else encoder_train

      return _train


    def    lowerCAmelCase_     (    self		:     Union[str, Any]      , __lowerCAmelCase		:     Any    ):
      with self.args.strategy.scope():
        try:
          if self.args.is_tpu or self.args.use_xla:
            # run additional 10 times to stabilize compilation for tpu
            logger.info("""Do inference on TPU. Running model 5 times to stabilize compilation"""    )
            timeit.repeat(__lowerCAmelCase      , repeat=1      , number=5    )

          # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
          _UpperCAmelCase         =					timeit.repeat(
              __lowerCAmelCase      , repeat=self.args.repeat      , number=10      , )

          return min(__lowerCAmelCase    ) / 10.0
        except ResourceExhaustedError as e:
          self.print_fn(f'''Doesn\'t fit on GPU. {e}'''    )


    def    lowerCAmelCase_     (    self		:     List[Any]      , __lowerCAmelCase		:     Callable[[], None]    ):
      logger.info(
          """Note that TensorFlow allocates more memory than """
          """it might need to speed up computation. """
          """The memory reported here corresponds to the memory """
          """reported by `nvidia-smi`, which can vary depending """
          """on total available memory on the GPU that is used."""    )
      with self.args.strategy.scope():
        try:
          if self.args.trace_memory_line_by_line:
            if not self.args.eager_mode:
              raise ValueError(
                  """`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory"""
                  """ consumption line by line."""    )
            _UpperCAmelCase         =					start_memory_tracing("""transformers"""    )

          if self.args.is_tpu:
            # tpu
            raise NotImplementedError(
                """Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking"""
                """ with `args.memory=False`"""    )
          elif self.args.is_gpu:
            # gpu
            if not is_pyanvml_available():
              logger.warning(
                  """py3nvml not installed, we won't log GPU memory usage. """
                  """Install py3nvml (pip install py3nvml) to log information about GPU."""    )
              _UpperCAmelCase         =					"""N/A"""
            else:
              logger.info(
                  """Measuring total GPU usage on GPU device. Make sure to not have additional processes"""
                  """ running on the same GPU."""    )
              # init nvml
              nvml.nvmlInit()
              func()
              _UpperCAmelCase         =					nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx    )
              _UpperCAmelCase         =					nvml.nvmlDeviceGetMemoryInfo(__lowerCAmelCase    )
              _UpperCAmelCase         =					meminfo.used
              _UpperCAmelCase         =					Memory(__lowerCAmelCase    )
              # shutdown nvml
              nvml.nvmlShutdown()
          else:
            # cpu
            if self.args.trace_memory_line_by_line:
              logger.info(
                  """When enabling line by line tracing, the max peak memory for CPU is inaccurate in"""
                  """ TensorFlow."""    )
              _UpperCAmelCase         =					None
            else:
              _UpperCAmelCase         =					measure_peak_memory_cpu(__lowerCAmelCase    )
              _UpperCAmelCase         =					Memory(__lowerCAmelCase    ) if isinstance(__lowerCAmelCase      , __lowerCAmelCase    ) else memory_bytes
          if self.args.trace_memory_line_by_line:
            _UpperCAmelCase         =					stop_memory_tracing(__lowerCAmelCase    )
            if memory is None:
              _UpperCAmelCase         =					summary.total
          else:
            _UpperCAmelCase         =					None

          return memory, summary
        except ResourceExhaustedError as e:
          self.print_fn(f'''Doesn\'t fit on GPU. {e}'''    )
          return "N/A", None
