





import os
import re
import sys
import traceback
import warnings
from pathlib import Path
from typing import Dict, Optional, Union
from uuid import uuida

from huggingface_hub import HfFolder, ModelCard, ModelCardData, hf_hub_download, whoami
from huggingface_hub.file_download import REGEX_COMMIT_HASH
from huggingface_hub.utils import (
    EntryNotFoundError,
    RepositoryNotFoundError,
    RevisionNotFoundError,
    is_jinja_available,
)
from packaging import version
from requests import HTTPError

from .. import __version__
from .constants import (
    DEPRECATED_REVISION_ARGS,
    DIFFUSERS_CACHE,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    SAFETENSORS_WEIGHTS_NAME,
    WEIGHTS_NAME,
)
from .import_utils import (
    ENV_VARS_TRUE_VALUES,
    _flax_version,
    _jax_version,
    _onnxruntime_version,
    _torch_version,
    is_flax_available,
    is_onnx_available,
    is_torch_available,
)
from .logging import get_logger


snake_case__					:							Any													=     get_logger(__name__)


snake_case__					:							int													=     Path(__file__).parent / 'model_card_template.md'
snake_case__					:							Dict													=     uuida().hex
snake_case__					:							List[str]													=     os.getenv('HF_HUB_OFFLINE', '').upper() in ENV_VARS_TRUE_VALUES
snake_case__					:							List[str]													=     os.getenv('DISABLE_TELEMETRY', '').upper() in ENV_VARS_TRUE_VALUES
snake_case__					:							Tuple													=     HUGGINGFACE_CO_RESOLVE_ENDPOINT + '/api/telemetry/'



def _a				(	lowerCamelCase:   Union[Dict, str, None] = None )	->						str:







      '''simple docstring'''






      __A          =				F"""diffusers/{__version__}; python/{sys.version.split()[0]}; session_id/{SESSION_ID}"""
      if DISABLE_TELEMETRY or HF_HUB_OFFLINE:
            return ua + "; telemetry/off"
      if is_torch_available():
            ua += F"""; torch/{_torch_version}"""
      if is_flax_available():
            ua += F"""; jax/{_jax_version}"""
            ua += F"""; flax/{_flax_version}"""
      if is_onnx_available():
            ua += F"""; onnxruntime/{_onnxruntime_version}"""
      # CI will set this value to True
      if os.environ.get('''DIFFUSERS_IS_CI'''					,				'''''' ).upper() in ENV_VARS_TRUE_VALUES:
            ua += "; is_ci/true"
      if isinstance(lowerCamelCase					,				lowerCamelCase ):
            ua += "; " + "; ".join(F"""{k}/{v}""" for k, v in user_agent.items() )
      elif isinstance(lowerCamelCase					,				lowerCamelCase ):
            ua += "; " + user_agent
      return ua



def _a				(	lowerCamelCase:   str					,				lowerCamelCase:   Optional[str] = None					,				lowerCamelCase:   Optional[str] = None )	->						str:







      '''simple docstring'''






      if token is None:
            __A          =				HfFolder.get_token()
      if organization is None:
            __A          =				whoami(lowerCamelCase )['''name''']
            return F"""{username}/{model_id}"""
      else:
            return F"""{organization}/{model_id}"""



def _a				(	lowerCamelCase:   Union[str, Any]					,				lowerCamelCase:   Tuple )	->						Tuple:







      '''simple docstring'''






      if not is_jinja_available():
            raise ValueError(
                '''Modelcard rendering is based on Jinja templates.'''
                ''' Please make sure to have `jinja` installed before using `create_model_card`.'''
                ''' To install it, please run `pip install Jinja2`.''' )

      if hasattr(lowerCamelCase					,				'''local_rank''' ) and args.local_rank not in [-1, 0]:
            return

      __A          =				args.hub_token if hasattr(lowerCamelCase					,				'''hub_token''' ) else None
      __A          =				get_full_repo_name(lowerCamelCase					,				token=lowerCamelCase )

      __A          =				ModelCard.from_template(
          card_data=ModelCardData(  # Card metadata object that will be converted to YAML block
              language='''en'''					,				license='''apache-2.0'''					,				library_name='''diffusers'''					,				tags=[]					,				datasets=args.dataset_name					,				metrics=[]					,				)					,				template_path=lowerCamelCase					,				model_name=lowerCamelCase					,				repo_name=lowerCamelCase					,				dataset_name=args.dataset_name if hasattr(lowerCamelCase					,				'''dataset_name''' ) else None					,				learning_rate=args.learning_rate					,				train_batch_size=args.train_batch_size					,				eval_batch_size=args.eval_batch_size					,				gradient_accumulation_steps=(
              args.gradient_accumulation_steps if hasattr(lowerCamelCase					,				'''gradient_accumulation_steps''' ) else None
          )					,				adam_betaa=args.adam_betaa if hasattr(lowerCamelCase					,				'''adam_beta1''' ) else None					,				adam_betaa=args.adam_betaa if hasattr(lowerCamelCase					,				'''adam_beta2''' ) else None					,				adam_weight_decay=args.adam_weight_decay if hasattr(lowerCamelCase					,				'''adam_weight_decay''' ) else None					,				adam_epsilon=args.adam_epsilon if hasattr(lowerCamelCase					,				'''adam_epsilon''' ) else None					,				lr_scheduler=args.lr_scheduler if hasattr(lowerCamelCase					,				'''lr_scheduler''' ) else None					,				lr_warmup_steps=args.lr_warmup_steps if hasattr(lowerCamelCase					,				'''lr_warmup_steps''' ) else None					,				ema_inv_gamma=args.ema_inv_gamma if hasattr(lowerCamelCase					,				'''ema_inv_gamma''' ) else None					,				ema_power=args.ema_power if hasattr(lowerCamelCase					,				'''ema_power''' ) else None					,				ema_max_decay=args.ema_max_decay if hasattr(lowerCamelCase					,				'''ema_max_decay''' ) else None					,				mixed_precision=args.mixed_precision					,				)

      __A          =				os.path.join(args.output_dir					,				'''README.md''' )
      model_card.save(lowerCamelCase )



def _a				(	lowerCamelCase:   Optional[str]					,				lowerCamelCase:   Optional[str] = None )	->						Any:







      '''simple docstring'''






      if resolved_file is None or commit_hash is not None:
            return commit_hash
      __A          =				str(Path(lowerCamelCase ).as_posix() )
      __A          =				re.search(r'''snapshots/([^/]+)/'''					,				lowerCamelCase )
      if search is None:
            return None
      __A          =				search.groups()[0]
      return commit_hash if REGEX_COMMIT_HASH.match(lowerCamelCase ) else None


# Old default cache path, potentially to be migrated.
# This logic was more or less taken from `transformers`, with the following differences:
# - Diffusers doesn't use custom environment variables to specify the cache path.
# - There is no need to migrate the cache format, just move the files to the new location.
snake_case__					:							Optional[int]													=     os.path.expanduser(
    os.getenv('HF_HOME', os.path.join(os.getenv('XDG_CACHE_HOME', '~/.cache'), 'huggingface'))
)
snake_case__					:							str													=     os.path.join(hf_cache_home, 'diffusers')



def _a				(	lowerCamelCase:   Optional[str] = None					,				lowerCamelCase:   Optional[str] = None )	->						None:







      '''simple docstring'''






      if new_cache_dir is None:
            __A          =				DIFFUSERS_CACHE
      if old_cache_dir is None:
            __A          =				old_diffusers_cache

      __A          =				Path(lowerCamelCase ).expanduser()
      __A          =				Path(lowerCamelCase ).expanduser()
      for old_blob_path in old_cache_dir.glob('''**/blobs/*''' ):
            if old_blob_path.is_file() and not old_blob_path.is_symlink():
                  __A          =				new_cache_dir / old_blob_path.relative_to(lowerCamelCase )
                  new_blob_path.parent.mkdir(parents=lowerCamelCase					,				exist_ok=lowerCamelCase )
                  os.replace(lowerCamelCase					,				lowerCamelCase )
                  try:
                        os.symlink(lowerCamelCase					,				lowerCamelCase )
                  except OSError:
                        logger.warning(
                            '''Could not create symlink between old cache and new cache. If you use an older version of diffusers again, files will be re-downloaded.''' )
      # At this point, old_cache_dir contains symlinks to the new cache (it can still be used).


snake_case__					:							List[Any]													=     os.path.join(DIFFUSERS_CACHE, 'version_diffusers_cache.txt')
if not os.path.isfile(cache_version_file):
       snake_case__					:							str													=     0
else:
       with open(cache_version_file) as f:
              try:
                     snake_case__					:							List[str]													=     int(f.read())
              except ValueError:
                     snake_case__					:							Any													=     0

if cache_version < 1:
       snake_case__					:							Dict													=     os.path.isdir(old_diffusers_cache) and len(os.listdir(old_diffusers_cache)) > 0
       if old_cache_is_not_empty:
              logger.warning(
                  'The cache for model files in Diffusers v0.14.0 has moved to a new location. Moving your '
                  'existing cached models. This is a one-time operation, you can interrupt it or run it '
                  'later by calling `diffusers.utils.hub_utils.move_cache()`.'
              )
              try:
                     move_cache()
              except Exception as e:
                     snake_case__					:							Tuple													=     '\n'.join(traceback.format_tb(e.__traceback__))
                     logger.error(
                         f'There was a problem when trying to move your cache:\n\n{trace}\n{e.__class__.__name__}: {e}\n\nPlease '
                         'file an issue at https://github.com/huggingface/diffusers/issues/new/choose, copy paste this whole '
                         'message and we will do our best to help.'
                     )

if cache_version < 1:
       try:
              os.makedirs(DIFFUSERS_CACHE, exist_ok=True)
              with open(cache_version_file, 'w') as f:
                     f.write('1')
       except Exception:
              logger.warning(
                  f'There was a problem when trying to write in your cache folder ({DIFFUSERS_CACHE}). Please, ensure '
                  'the directory exists and can be written to.'
              )



def _a				(	lowerCamelCase:   str					,				lowerCamelCase:   Optional[str] = None )	->						str:







      '''simple docstring'''






      if variant is not None:
            __A          =				weights_name.split('''.''' )
            __A          =				splits[:-1] + [variant] + splits[-1:]
            __A          =				'''.'''.join(lowerCamelCase )

      return weights_name







def _a				(	lowerCamelCase:   Dict					,				*,
    lowerCamelCase:   Optional[int]					,				lowerCamelCase:   Optional[int]					,				lowerCamelCase:   str					,				lowerCamelCase:   Any					,				lowerCamelCase:   Tuple					,				lowerCamelCase:   Optional[int]					,				lowerCamelCase:   Union[str, Any]					,				lowerCamelCase:   str					,				lowerCamelCase:   Union[str, Any]					,				lowerCamelCase:   Union[str, Any]					,				lowerCamelCase:   Optional[Any]=None					,				)	->						Optional[Any]:







      '''simple docstring'''






      __A          =				str(lowerCamelCase )
      if os.path.isfile(lowerCamelCase ):
            return pretrained_model_name_or_path
      elif os.path.isdir(lowerCamelCase ):
            if os.path.isfile(os.path.join(lowerCamelCase					,				lowerCamelCase ) ):
                  # Load from a PyTorch checkpoint
                  __A          =				os.path.join(lowerCamelCase					,				lowerCamelCase )
                  return model_file
            elif subfolder is not None and os.path.isfile(
                os.path.join(lowerCamelCase					,				lowerCamelCase					,				lowerCamelCase ) ):
                  __A          =				os.path.join(lowerCamelCase					,				lowerCamelCase					,				lowerCamelCase )
                  return model_file
            else:
                  raise EnvironmentError(
                      F"""Error no file named {weights_name} found in directory {pretrained_model_name_or_path}.""" )
      else:
            # 1. First check if deprecated way of loading from branches is used
            if (
                revision in DEPRECATED_REVISION_ARGS
                and (weights_name == WEIGHTS_NAME or weights_name == SAFETENSORS_WEIGHTS_NAME)
                and version.parse(version.parse(lowerCamelCase ).base_version ) >= version.parse('''0.20.0''' )
            ):
                  try:
                        __A          =				hf_hub_download(
                            lowerCamelCase					,				filename=_add_variant(lowerCamelCase					,				lowerCamelCase )					,				cache_dir=lowerCamelCase					,				force_download=lowerCamelCase					,				proxies=lowerCamelCase					,				resume_download=lowerCamelCase					,				local_files_only=lowerCamelCase					,				use_auth_token=lowerCamelCase					,				user_agent=lowerCamelCase					,				subfolder=lowerCamelCase					,				revision=revision or commit_hash					,				)
                        warnings.warn(
                            F"""Loading the variant {revision} from {pretrained_model_name_or_path} via `revision='{revision}'` is deprecated. Loading instead from `revision='main'` with `variant={revision}`. Loading model variants via `revision='{revision}'` will be removed in diffusers v1. Please use `variant='{revision}'` instead."""					,				lowerCamelCase					,				)
                        return model_file
                  except:  # noqa: E722
                        warnings.warn(
                            F"""You are loading the variant {revision} from {pretrained_model_name_or_path} via `revision='{revision}'`. This behavior is deprecated and will be removed in diffusers v1. One should use `variant='{revision}'` instead. However, it appears that {pretrained_model_name_or_path} currently does not have a {_add_variant(lowerCamelCase					,				lowerCamelCase )} file in the 'main' branch of {pretrained_model_name_or_path}. \n The Diffusers team and community would be very grateful if you could open an issue: https://github.com/huggingface/diffusers/issues/new with the title '{pretrained_model_name_or_path} is missing {_add_variant(lowerCamelCase					,				lowerCamelCase )}' so that the correct variant file can be added."""					,				lowerCamelCase					,				)
            try:
                  # 2. Load model file as usual
                  __A          =				hf_hub_download(
                      lowerCamelCase					,				filename=lowerCamelCase					,				cache_dir=lowerCamelCase					,				force_download=lowerCamelCase					,				proxies=lowerCamelCase					,				resume_download=lowerCamelCase					,				local_files_only=lowerCamelCase					,				use_auth_token=lowerCamelCase					,				user_agent=lowerCamelCase					,				subfolder=lowerCamelCase					,				revision=revision or commit_hash					,				)
                  return model_file

            except RepositoryNotFoundError:
                  raise EnvironmentError(
                      F"""{pretrained_model_name_or_path} is not a local folder and is not a valid model identifier """
                      '''listed on \'https://huggingface.co/models\'\nIf this is a private repository, make sure to pass a '''
                      '''token having permission to this repo with `use_auth_token` or log in with `huggingface-cli '''
                      '''login`.''' )
            except RevisionNotFoundError:
                  raise EnvironmentError(
                      F"""{revision} is not a valid git identifier (branch name, tag name or commit id) that exists for """
                      '''this model name. Check the model page at '''
                      F"""'https://huggingface.co/{pretrained_model_name_or_path}' for available revisions.""" )
            except EntryNotFoundError:
                  raise EnvironmentError(
                      F"""{pretrained_model_name_or_path} does not appear to have a file named {weights_name}.""" )
            except HTTPError as err:
                  raise EnvironmentError(
                      F"""There was a specific connection error when trying to load {pretrained_model_name_or_path}:\n{err}""" )
            except ValueError:
                  raise EnvironmentError(
                      F"""We couldn't connect to '{HUGGINGFACE_CO_RESOLVE_ENDPOINT}' to load this model, couldn't find it"""
                      F""" in the cached files and it looks like {pretrained_model_name_or_path} is not the path to a"""
                      F""" directory containing a file named {weights_name} or"""
                      ''' \nCheckout your internet connection or see how to run the library in'''
                      ''' offline mode at \'https://huggingface.co/docs/diffusers/installation#offline-mode\'.''' )
            except EnvironmentError:
                  raise EnvironmentError(
                      F"""Can't load the model for '{pretrained_model_name_or_path}'. If you were trying to load it from """
                      '''\'https://huggingface.co/models\', make sure you don\'t have a local directory with the same name. '''
                      F"""Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a directory """
                      F"""containing a file named {weights_name}""" )












import json
import os
import tempfile

import transformers

import datasets
from utils import generate_example_dataset, get_duration


snake_case__					:							Union[str, Any]													=     500000

snake_case__						,    snake_case__					:							Optional[Any]													=     os.path.split(__file__)
snake_case__					:							List[Any]													=     os.path.join(RESULTS_BASEPATH, 'results', RESULTS_FILENAME.replace('.py', '.json'))



@get_duration
def _a				(	lowerCamelCase:   datasets.Dataset					,				**lowerCamelCase:   Optional[int] )	->						str:







      '''simple docstring'''






      __A          =				dataset.map(**lowerCamelCase )



@get_duration
def _a				(	lowerCamelCase:   datasets.Dataset					,				**lowerCamelCase:   Optional[int] )	->						str:







      '''simple docstring'''






      __A          =				dataset.filter(**lowerCamelCase )



def _a				(	)	->						List[Any]:







      '''simple docstring'''






      __A          =				{'''num examples''': SPEED_TEST_N_EXAMPLES}
      with tempfile.TemporaryDirectory() as tmp_dir:
            __A          =				datasets.Features({'''text''': datasets.Value('''string''' ), '''numbers''': datasets.Value('''float32''' )} )
            __A          =				generate_example_dataset(
                os.path.join(lowerCamelCase					,				'''dataset.arrow''' )					,				lowerCamelCase					,				num_examples=lowerCamelCase )

            __A          =				transformers.AutoTokenizer.from_pretrained('''bert-base-cased'''					,				use_fast=lowerCamelCase )

            def tokenize(lowerCamelCase:   List[str] ):
                  return tokenizer(examples['''text'''] )

            __A          =				map(lowerCamelCase )

            __A          =				map(lowerCamelCase					,				batched=lowerCamelCase )

            __A          =				map(lowerCamelCase					,				function=lambda lowerCamelCase : None					,				batched=lowerCamelCase )

            with dataset.formatted_as(type='''numpy''' ):
                  __A          =				map(lowerCamelCase					,				function=lambda lowerCamelCase : None					,				batched=lowerCamelCase )

            with dataset.formatted_as(type='''pandas''' ):
                  __A          =				map(lowerCamelCase					,				function=lambda lowerCamelCase : None					,				batched=lowerCamelCase )

            with dataset.formatted_as(type='''torch'''					,				columns='''numbers''' ):
                  __A          =				map(lowerCamelCase					,				function=lambda lowerCamelCase : None					,				batched=lowerCamelCase )

            with dataset.formatted_as(type='''tensorflow'''					,				columns='''numbers''' ):
                  __A          =				map(lowerCamelCase					,				function=lambda lowerCamelCase : None					,				batched=lowerCamelCase )

            __A          =				map(lowerCamelCase					,				function=lowerCamelCase					,				batched=lowerCamelCase )

            __A          =				filter(lowerCamelCase )

            # Activate later when tokenizer support batched inputs
            # with dataset.formatted_as(type='numpy'):
            #     times[func.__name__ + " fast-tokenizer batched numpy"] = func(dataset, function=tokenize, batched=True)

      with open(lowerCamelCase					,				'''wb''' ) as f:
            f.write(json.dumps(lowerCamelCase ).encode('''utf-8''' ) )


if __name__ == "__main__":  # useful to run the profiler
       benchmark_map_filter()





