"""simple docstring"""


import gc
import math
import unittest

import torch

from diffusers import UNetaDModel
from diffusers.utils import floats_tensor, logging, slow, torch_all_close, torch_device
from diffusers.utils.testing_utils import enable_full_determinism

from .test_modeling_common import ModelTesterMixin, UNetTesterMixin


UpperCAmelCase__     						=  logging.get_logger(__name__)

enable_full_determinism()


class 				a (						lowerCAmelCase_		,							lowerCAmelCase_		,							unittest.TestCase      ):
    _snake_case   :     Optional[int]   							=							UNetaDModel
    _snake_case   :     List[str]   							=							'sample'


    @property
    def    lowerCAmelCase_     (    self		:     List[str]    ):
      _UpperCAmelCase         =					4
      _UpperCAmelCase         =					3
      _UpperCAmelCase         =					(32, 32)

      _UpperCAmelCase         =					floats_tensor((batch_size, num_channels) + sizes    ).to(__lowerCAmelCase    )
      _UpperCAmelCase         =					torch.tensor([10]    ).to(__lowerCAmelCase    )

      return {"sample": noise, "timestep": time_step}


    @property
    def    lowerCAmelCase_     (    self		:     List[Any]    ):
      return (3, 32, 32)


    @property
    def    lowerCAmelCase_     (    self		:     Optional[Any]    ):
      return (3, 32, 32)


    def    lowerCAmelCase_     (    self		:     Any    ):
      _UpperCAmelCase         =					{
          """block_out_channels""": (32, 64),
          """down_block_types""": ("""DownBlock2D""", """AttnDownBlock2D"""),
          """up_block_types""": ("""AttnUpBlock2D""", """UpBlock2D"""),
          """attention_head_dim""": 3,
          """out_channels""": 3,
          """in_channels""": 3,
          """layers_per_block""": 2,
          """sample_size""": 32,
      }
      _UpperCAmelCase         =					self.dummy_input
      return init_dict, inputs_dict



class 				a (						lowerCAmelCase_		,							lowerCAmelCase_		,							unittest.TestCase      ):
    _snake_case   :     int   							=							UNetaDModel
    _snake_case   :     Optional[Any]   							=							'sample'


    @property
    def    lowerCAmelCase_     (    self		:     Optional[Any]    ):
      _UpperCAmelCase         =					4
      _UpperCAmelCase         =					4
      _UpperCAmelCase         =					(32, 32)

      _UpperCAmelCase         =					floats_tensor((batch_size, num_channels) + sizes    ).to(__lowerCAmelCase    )
      _UpperCAmelCase         =					torch.tensor([10]    ).to(__lowerCAmelCase    )

      return {"sample": noise, "timestep": time_step}


    @property
    def    lowerCAmelCase_     (    self		:     Optional[Any]    ):
      return (4, 32, 32)


    @property
    def    lowerCAmelCase_     (    self		:     Dict    ):
      return (4, 32, 32)


    def    lowerCAmelCase_     (    self		:     List[Any]    ):
      _UpperCAmelCase         =					{
          """sample_size""": 32,
          """in_channels""": 4,
          """out_channels""": 4,
          """layers_per_block""": 2,
          """block_out_channels""": (32, 64),
          """attention_head_dim""": 32,
          """down_block_types""": ("""DownBlock2D""", """DownBlock2D"""),
          """up_block_types""": ("""UpBlock2D""", """UpBlock2D"""),
      }
      _UpperCAmelCase         =					self.dummy_input
      return init_dict, inputs_dict


    def    lowerCAmelCase_     (    self		:     List[str]    ):
      _UpperCAmelCase       , _UpperCAmelCase         =					UNetaDModel.from_pretrained("""fusing/unet-ldm-dummy-update"""      , output_loading_info=__lowerCAmelCase    )

      self.assertIsNotNone(__lowerCAmelCase    )
      self.assertEqual(len(loading_info["""missing_keys"""]    )      , 0    )

      model.to(__lowerCAmelCase    )
      _UpperCAmelCase         =					model(**self.dummy_input    ).sample

      assert image is not None, "Make sure output is not None"


    @unittest.skipIf(torch_device != """cuda"""      , """This test is supposed to run on GPU"""    )
    def    lowerCAmelCase_     (    self		:     Optional[int]    ):
      _UpperCAmelCase       , _UpperCAmelCase         =					UNetaDModel.from_pretrained("""fusing/unet-ldm-dummy-update"""      , output_loading_info=__lowerCAmelCase    )
      model.to(__lowerCAmelCase    )
      _UpperCAmelCase         =					model(**self.dummy_input    ).sample

      assert image is not None, "Make sure output is not None"


    @unittest.skipIf(torch_device != """cuda"""      , """This test is supposed to run on GPU"""    )
    def    lowerCAmelCase_     (    self		:     str    ):
      # by defautl model loading will use accelerate as `low_cpu_mem_usage=True`
      _UpperCAmelCase       , _UpperCAmelCase         =					UNetaDModel.from_pretrained("""fusing/unet-ldm-dummy-update"""      , output_loading_info=__lowerCAmelCase    )
      model_accelerate.to(__lowerCAmelCase    )
      model_accelerate.eval()

      _UpperCAmelCase         =					torch.randn(
          1      , model_accelerate.config.in_channels      , model_accelerate.config.sample_size      , model_accelerate.config.sample_size      , generator=torch.manual_seed(0    )      , )
      _UpperCAmelCase         =					noise.to(__lowerCAmelCase    )
      _UpperCAmelCase         =					torch.tensor([10] * noise.shape[0]    ).to(__lowerCAmelCase    )

      _UpperCAmelCase         =					model_accelerate(__lowerCAmelCase      , __lowerCAmelCase    )["""sample"""]

      # two models don't need to stay in the device at the same time
      del model_accelerate
      torch.cuda.empty_cache()
      gc.collect()

      _UpperCAmelCase       , _UpperCAmelCase         =					UNetaDModel.from_pretrained(
          """fusing/unet-ldm-dummy-update"""      , output_loading_info=__lowerCAmelCase      , low_cpu_mem_usage=__lowerCAmelCase    )
      model_normal_load.to(__lowerCAmelCase    )
      model_normal_load.eval()
      _UpperCAmelCase         =					model_normal_load(__lowerCAmelCase      , __lowerCAmelCase    )["""sample"""]

      assert torch_all_close(__lowerCAmelCase      , __lowerCAmelCase      , rtol=1e-3    )


    def    lowerCAmelCase_     (    self		:     Tuple    ):
      _UpperCAmelCase         =					UNetaDModel.from_pretrained("""fusing/unet-ldm-dummy-update"""    )
      model.eval()
      model.to(__lowerCAmelCase    )

      _UpperCAmelCase         =					torch.randn(
          1      , model.config.in_channels      , model.config.sample_size      , model.config.sample_size      , generator=torch.manual_seed(0    )      , )
      _UpperCAmelCase         =					noise.to(__lowerCAmelCase    )
      _UpperCAmelCase         =					torch.tensor([10] * noise.shape[0]    ).to(__lowerCAmelCase    )

      with torch.no_grad():
        _UpperCAmelCase         =					model(__lowerCAmelCase      , __lowerCAmelCase    ).sample

      _UpperCAmelCase         =					output[0, -1, -3:, -3:].flatten().cpu()
      # fmt: off
      _UpperCAmelCase         =					torch.tensor([-13.3_258, -20.1_100, -15.9_873, -17.6_617, -23.0_596, -17.9_419, -13.3_675, -16.1_889, -12.3_800]    )
      # fmt: on

      self.assertTrue(torch_all_close(__lowerCAmelCase      , __lowerCAmelCase      , rtol=1e-3    )    )



class 				a (						lowerCAmelCase_		,							lowerCAmelCase_		,							unittest.TestCase      ):
    _snake_case   :     Optional[Any]   							=							UNetaDModel
    _snake_case   :     str   							=							'sample'


    @property
    def    lowerCAmelCase_     (    self		:     Optional[Any]      , __lowerCAmelCase		:     str=(32, 32)    ):
      _UpperCAmelCase         =					4
      _UpperCAmelCase         =					3

      _UpperCAmelCase         =					floats_tensor((batch_size, num_channels) + sizes    ).to(__lowerCAmelCase    )
      _UpperCAmelCase         =					torch.tensor(batch_size * [10]    ).to(dtype=torch.intaa      , device=__lowerCAmelCase    )

      return {"sample": noise, "timestep": time_step}


    @property
    def    lowerCAmelCase_     (    self		:     Any    ):
      return (3, 32, 32)


    @property
    def    lowerCAmelCase_     (    self		:     Union[str, Any]    ):
      return (3, 32, 32)


    def    lowerCAmelCase_     (    self		:     Union[str, Any]    ):
      _UpperCAmelCase         =					{
          """block_out_channels""": [32, 64, 64, 64],
          """in_channels""": 3,
          """layers_per_block""": 1,
          """out_channels""": 3,
          """time_embedding_type""": """fourier""",
          """norm_eps""": 1e-6,
          """mid_block_scale_factor""": math.sqrt(2.0    ),
          """norm_num_groups""": None,
          """down_block_types""": [
              """SkipDownBlock2D""",
              """AttnSkipDownBlock2D""",
              """SkipDownBlock2D""",
              """SkipDownBlock2D""",
          ],
          """up_block_types""": [
              """SkipUpBlock2D""",
              """SkipUpBlock2D""",
              """AttnSkipUpBlock2D""",
              """SkipUpBlock2D""",
          ],
      }
      _UpperCAmelCase         =					self.dummy_input
      return init_dict, inputs_dict


    @slow
    def    lowerCAmelCase_     (    self		:     Optional[Any]    ):
      _UpperCAmelCase       , _UpperCAmelCase         =					UNetaDModel.from_pretrained("""google/ncsnpp-celebahq-256"""      , output_loading_info=__lowerCAmelCase    )
      self.assertIsNotNone(__lowerCAmelCase    )
      self.assertEqual(len(loading_info["""missing_keys"""]    )      , 0    )

      model.to(__lowerCAmelCase    )
      _UpperCAmelCase         =					self.dummy_input
      _UpperCAmelCase         =					floats_tensor((4, 3) + (256, 256)    ).to(__lowerCAmelCase    )
      _UpperCAmelCase         =					noise
      _UpperCAmelCase         =					model(**__lowerCAmelCase    )

      assert image is not None, "Make sure output is not None"


    @slow
    def    lowerCAmelCase_     (    self		:     Union[str, Any]    ):
      _UpperCAmelCase         =					UNetaDModel.from_pretrained("""google/ncsnpp-celebahq-256"""    )
      model.to(__lowerCAmelCase    )

      _UpperCAmelCase         =					4
      _UpperCAmelCase         =					3
      _UpperCAmelCase         =					(256, 256)

      _UpperCAmelCase         =					torch.ones((batch_size, num_channels) + sizes    ).to(__lowerCAmelCase    )
      _UpperCAmelCase         =					torch.tensor(batch_size * [1e-4]    ).to(__lowerCAmelCase    )

      with torch.no_grad():
        _UpperCAmelCase         =					model(__lowerCAmelCase      , __lowerCAmelCase    ).sample

      _UpperCAmelCase         =					output[0, -3:, -3:, -1].flatten().cpu()
      # fmt: off
      _UpperCAmelCase         =					torch.tensor([-4_842.8_691, -6_499.6_631, -3_800.1_953, -7_978.2_686, -10_980.7_129, -20_028.8_535, 8_148.2_822, 2_342.2_905, 567.7_608]    )
      # fmt: on

      self.assertTrue(torch_all_close(__lowerCAmelCase      , __lowerCAmelCase      , rtol=1e-2    )    )


    def    lowerCAmelCase_     (    self		:     str    ):
      _UpperCAmelCase         =					UNetaDModel.from_pretrained("""fusing/ncsnpp-ffhq-ve-dummy-update"""    )
      model.to(__lowerCAmelCase    )

      _UpperCAmelCase         =					4
      _UpperCAmelCase         =					3
      _UpperCAmelCase         =					(32, 32)

      _UpperCAmelCase         =					torch.ones((batch_size, num_channels) + sizes    ).to(__lowerCAmelCase    )
      _UpperCAmelCase         =					torch.tensor(batch_size * [1e-4]    ).to(__lowerCAmelCase    )

      with torch.no_grad():
        _UpperCAmelCase         =					model(__lowerCAmelCase      , __lowerCAmelCase    ).sample

      _UpperCAmelCase         =					output[0, -3:, -3:, -1].flatten().cpu()
      # fmt: off
      _UpperCAmelCase         =					torch.tensor([-0.0_325, -0.0_900, -0.0_869, -0.0_332, -0.0_725, -0.0_270, -0.0_101, 0.0_227, 0.0_256]    )
      # fmt: on

      self.assertTrue(torch_all_close(__lowerCAmelCase      , __lowerCAmelCase      , rtol=1e-2    )    )


    def    lowerCAmelCase_     (    self		:     List[str]    ):
      # not required for this model
      pass

"""simple docstring"""


import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
    import tensorflow as tf
    from tensorflow.python.framework.errors_impl import ResourceExhaustedError

    from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
    import pyanvml.pyanvml as nvml

UpperCAmelCase__     						=  logging.get_logger(__name__)






def 			__UpperCAmelCase    (	lowercase      ,lowercase						):







  """simple docstring"""
  def run_func(lowercase						):
    @wraps(lowercase						)
    def run_in_eager_mode(*lowercase      ,**lowercase						):
      return func(*lowercase      ,**lowercase						)

    @wraps(lowercase						)
    @tf.function(experimental_compile=lowercase						)
    def run_in_graph_mode(*lowercase      ,**lowercase						):
      return func(*lowercase      ,**lowercase						)

    if do_eager_mode is True:
      if use_xla is not False:
        raise ValueError(
            """Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`."""						)
      return run_in_eager_mode
    else:
      return run_in_graph_mode

  return run_func






def 			__UpperCAmelCase    (	lowercase      ,lowercase      ,lowercase						):







  """simple docstring"""
  _UpperCAmelCase         =					random.Random()
  _UpperCAmelCase         =					[rng.randint(0      ,vocab_size - 1						) for i in range(batch_size * sequence_length						)]
  return tf.constant(lowercase      ,shape=(batch_size, sequence_length)      ,dtype=tf.intaa						)


class 				a (						lowerCAmelCase_      ):
    _snake_case   :     TensorFlowBenchmarkArguments
    _snake_case   :     PretrainedConfig
    _snake_case   :     str   							=							"TensorFlow"


    @property
    def    lowerCAmelCase_     (    self		:     Union[str, Any]    ):
      return tf.__version__


    def    lowerCAmelCase_     (    self		:     List[str]      , __lowerCAmelCase		:     str      , __lowerCAmelCase		:     int      , __lowerCAmelCase		:     int    ):
      # initialize GPU on separate process
      _UpperCAmelCase         =					self.args.strategy
      if strategy is None:
        raise ValueError("""A device strategy has to be initialized before using TensorFlow."""    )
      _UpperCAmelCase         =					self._prepare_inference_func(__lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase    )
      return self._measure_speed(_inference    )


    def    lowerCAmelCase_     (    self		:     Optional[int]      , __lowerCAmelCase		:     str      , __lowerCAmelCase		:     int      , __lowerCAmelCase		:     int    ):
      _UpperCAmelCase         =					self.args.strategy
      if strategy is None:
        raise ValueError("""A device strategy has to be initialized before using TensorFlow."""    )
      _UpperCAmelCase         =					self._prepare_train_func(__lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase    )
      return self._measure_speed(_train    )


    def    lowerCAmelCase_     (    self		:     Optional[Any]      , __lowerCAmelCase		:     str      , __lowerCAmelCase		:     int      , __lowerCAmelCase		:     int    ):
      # initialize GPU on separate process
      if self.args.is_gpu:
        tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]      , __lowerCAmelCase    )
      _UpperCAmelCase         =					self.args.strategy
      if strategy is None:
        raise ValueError("""A device strategy has to be initialized before using TensorFlow."""    )
      _UpperCAmelCase         =					self._prepare_inference_func(__lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase    )
      return self._measure_memory(_inference    )


    def    lowerCAmelCase_     (    self		:     str      , __lowerCAmelCase		:     str      , __lowerCAmelCase		:     int      , __lowerCAmelCase		:     int    ):
      if self.args.is_gpu:
        tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]      , __lowerCAmelCase    )
      _UpperCAmelCase         =					self.args.strategy
      if strategy is None:
        raise ValueError("""A device strategy has to be initialized before using TensorFlow."""    )

      _UpperCAmelCase         =					self._prepare_train_func(__lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase    )
      return self._measure_memory(_train    )


    def    lowerCAmelCase_     (    self		:     List[Any]      , __lowerCAmelCase		:     str      , __lowerCAmelCase		:     int      , __lowerCAmelCase		:     int    ):
      _UpperCAmelCase         =					self.config_dict[model_name]

      if self.args.fpaa:
        raise NotImplementedError("""Mixed precision is currently not supported."""    )

      _UpperCAmelCase         =					(
          hasattr(__lowerCAmelCase      , """architectures"""    )
          and isinstance(config.architectures      , __lowerCAmelCase    )
          and len(config.architectures    ) > 0
      )
      if not self.args.only_pretrain_model and has_model_class_in_config:
        try:
          _UpperCAmelCase         =					"""TF""" + config.architectures[0]  # prepend 'TF' for tensorflow model
          _UpperCAmelCase         =					__import__("""transformers"""      , fromlist=[model_class]    )
          _UpperCAmelCase         =					getattr(__lowerCAmelCase      , __lowerCAmelCase    )
          _UpperCAmelCase         =					model_cls(__lowerCAmelCase    )
        except ImportError:
          raise ImportError(
              f'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
              """ set `--only_pretrain_model` or `args.only_pretrain_model=True`."""    )
      else:
        _UpperCAmelCase         =					TF_MODEL_MAPPING[config.__class__](__lowerCAmelCase    )

      # encoder-decoder has vocab size saved differently
      _UpperCAmelCase         =					config.vocab_size if hasattr(__lowerCAmelCase      , """vocab_size"""    ) else config.encoder.vocab_size
      _UpperCAmelCase         =					random_input_ids(__lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase    )

      @run_with_tf_optimizations(self.args.eager_mode      , self.args.use_xla    )
      def encoder_decoder_forward():
        return model(__lowerCAmelCase      , decoder_input_ids=__lowerCAmelCase      , training=__lowerCAmelCase    )

      @run_with_tf_optimizations(self.args.eager_mode      , self.args.use_xla    )
      def encoder_forward():
        return model(__lowerCAmelCase      , training=__lowerCAmelCase    )

      _UpperCAmelCase         =					encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

      return _inference


    def    lowerCAmelCase_     (    self		:     List[str]      , __lowerCAmelCase		:     str      , __lowerCAmelCase		:     int      , __lowerCAmelCase		:     int    ):
      _UpperCAmelCase         =					self.config_dict[model_name]

      if self.args.eager_mode is not False:
        raise ValueError("""Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`."""    )

      if self.args.fpaa:
        raise NotImplementedError("""Mixed precision is currently not supported."""    )

      _UpperCAmelCase         =					(
          hasattr(__lowerCAmelCase      , """architectures"""    )
          and isinstance(config.architectures      , __lowerCAmelCase    )
          and len(config.architectures    ) > 0
      )
      if not self.args.only_pretrain_model and has_model_class_in_config:
        try:
          _UpperCAmelCase         =					"""TF""" + config.architectures[0]  # prepend 'TF' for tensorflow model
          _UpperCAmelCase         =					__import__("""transformers"""      , fromlist=[model_class]    )
          _UpperCAmelCase         =					getattr(__lowerCAmelCase      , __lowerCAmelCase    )
          _UpperCAmelCase         =					model_cls(__lowerCAmelCase    )
        except ImportError:
          raise ImportError(
              f'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
              """ set `--only_pretrain_model` or `args.only_pretrain_model=True`."""    )
      else:
        _UpperCAmelCase         =					TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](__lowerCAmelCase    )

      # encoder-decoder has vocab size saved differently
      _UpperCAmelCase         =					config.vocab_size if hasattr(__lowerCAmelCase      , """vocab_size"""    ) else config.encoder.vocab_size
      _UpperCAmelCase         =					random_input_ids(__lowerCAmelCase      , __lowerCAmelCase      , __lowerCAmelCase    )

      @run_with_tf_optimizations(self.args.eager_mode      , self.args.use_xla    )
      def encoder_decoder_train():
        _UpperCAmelCase         =					model(__lowerCAmelCase      , decoder_input_ids=__lowerCAmelCase      , labels=__lowerCAmelCase      , training=__lowerCAmelCase    )[0]
        _UpperCAmelCase         =					tf.gradients(__lowerCAmelCase      , model.trainable_variables    )
        return gradients

      @run_with_tf_optimizations(self.args.eager_mode      , self.args.use_xla    )
      def encoder_train():
        _UpperCAmelCase         =					model(__lowerCAmelCase      , labels=__lowerCAmelCase      , training=__lowerCAmelCase    )[0]
        _UpperCAmelCase         =					tf.gradients(__lowerCAmelCase      , model.trainable_variables    )
        return gradients

      _UpperCAmelCase         =					encoder_decoder_train if config.is_encoder_decoder else encoder_train

      return _train


    def    lowerCAmelCase_     (    self		:     Union[str, Any]      , __lowerCAmelCase		:     Any    ):
      with self.args.strategy.scope():
        try:
          if self.args.is_tpu or self.args.use_xla:
            # run additional 10 times to stabilize compilation for tpu
            logger.info("""Do inference on TPU. Running model 5 times to stabilize compilation"""    )
            timeit.repeat(__lowerCAmelCase      , repeat=1      , number=5    )

          # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
          _UpperCAmelCase         =					timeit.repeat(
              __lowerCAmelCase      , repeat=self.args.repeat      , number=10      , )

          return min(__lowerCAmelCase    ) / 10.0
        except ResourceExhaustedError as e:
          self.print_fn(f'''Doesn\'t fit on GPU. {e}'''    )


    def    lowerCAmelCase_     (    self		:     List[Any]      , __lowerCAmelCase		:     Callable[[], None]    ):
      logger.info(
          """Note that TensorFlow allocates more memory than """
          """it might need to speed up computation. """
          """The memory reported here corresponds to the memory """
          """reported by `nvidia-smi`, which can vary depending """
          """on total available memory on the GPU that is used."""    )
      with self.args.strategy.scope():
        try:
          if self.args.trace_memory_line_by_line:
            if not self.args.eager_mode:
              raise ValueError(
                  """`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory"""
                  """ consumption line by line."""    )
            _UpperCAmelCase         =					start_memory_tracing("""transformers"""    )

          if self.args.is_tpu:
            # tpu
            raise NotImplementedError(
                """Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking"""
                """ with `args.memory=False`"""    )
          elif self.args.is_gpu:
            # gpu
            if not is_pyanvml_available():
              logger.warning(
                  """py3nvml not installed, we won't log GPU memory usage. """
                  """Install py3nvml (pip install py3nvml) to log information about GPU."""    )
              _UpperCAmelCase         =					"""N/A"""
            else:
              logger.info(
                  """Measuring total GPU usage on GPU device. Make sure to not have additional processes"""
                  """ running on the same GPU."""    )
              # init nvml
              nvml.nvmlInit()
              func()
              _UpperCAmelCase         =					nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx    )
              _UpperCAmelCase         =					nvml.nvmlDeviceGetMemoryInfo(__lowerCAmelCase    )
              _UpperCAmelCase         =					meminfo.used
              _UpperCAmelCase         =					Memory(__lowerCAmelCase    )
              # shutdown nvml
              nvml.nvmlShutdown()
          else:
            # cpu
            if self.args.trace_memory_line_by_line:
              logger.info(
                  """When enabling line by line tracing, the max peak memory for CPU is inaccurate in"""
                  """ TensorFlow."""    )
              _UpperCAmelCase         =					None
            else:
              _UpperCAmelCase         =					measure_peak_memory_cpu(__lowerCAmelCase    )
              _UpperCAmelCase         =					Memory(__lowerCAmelCase    ) if isinstance(__lowerCAmelCase      , __lowerCAmelCase    ) else memory_bytes
          if self.args.trace_memory_line_by_line:
            _UpperCAmelCase         =					stop_memory_tracing(__lowerCAmelCase    )
            if memory is None:
              _UpperCAmelCase         =					summary.total
          else:
            _UpperCAmelCase         =					None

          return memory, summary
        except ResourceExhaustedError as e:
          self.print_fn(f'''Doesn\'t fit on GPU. {e}'''    )
          return "N/A", None
