


'''simple docstring'''

import os
import re
import sys
import traceback
import warnings
from pathlib import Path
from typing import Dict, Optional, Union
from uuid import uuida

from huggingface_hub import HfFolder, ModelCard, ModelCardData, hf_hub_download, whoami
from huggingface_hub.file_download import REGEX_COMMIT_HASH
from huggingface_hub.utils import (
    EntryNotFoundError,
    RepositoryNotFoundError,
    RevisionNotFoundError,
    is_jinja_available,
)
from packaging import version
from requests import HTTPError

from .. import __version__
from .constants import (
    DEPRECATED_REVISION_ARGS,
    DIFFUSERS_CACHE,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    SAFETENSORS_WEIGHTS_NAME,
    WEIGHTS_NAME,
)
from .import_utils import (
    ENV_VARS_TRUE_VALUES,
    _flax_version,
    _jax_version,
    _onnxruntime_version,
    _torch_version,
    is_flax_available,
    is_onnx_available,
    is_torch_available,
)
from .logging import get_logger


a_  :				Tuple						= get_logger(__name__)


a_  :				Union[str, Any]						= Path(__file__).parent / """model_card_template.md"""
a_  :				Any						= uuida().hex
a_  :				List[Any]						= os.getenv("""HF_HUB_OFFLINE""", """""").upper() in ENV_VARS_TRUE_VALUES
a_  :				Optional[Any]						= os.getenv("""DISABLE_TELEMETRY""", """""").upper() in ENV_VARS_TRUE_VALUES
a_  :				int						= HUGGINGFACE_CO_RESOLVE_ENDPOINT + """/api/telemetry/"""





def __snake_case					(      UpperCAmelCase_						:						Union[Dict, str, None] = None		):
 lowerCamelCase_      =  F'''diffusers/{__version__}; python/{sys.version.split()[0]}; session_id/{SESSION_ID}'''
 if DISABLE_TELEMETRY or HF_HUB_OFFLINE:
  return ua + "; telemetry/off"
 if is_torch_available():
  ua += F'''; torch/{_torch_version}'''
 if is_flax_available():
  ua += F'''; jax/{_jax_version}'''
  ua += F'''; flax/{_flax_version}'''
 if is_onnx_available():
  ua += F'''; onnxruntime/{_onnxruntime_version}'''
 # CI will set this value to True
 if os.environ.get("DIFFUSERS_IS_CI"					,				""		).upper() in ENV_VARS_TRUE_VALUES:
  ua += "; is_ci/true"
 if isinstance(UpperCAmelCase_					,				UpperCAmelCase_		):
  ua += "; " + "; ".join(F'''{k}/{v}''' for k, v in user_agent.items()		)
 elif isinstance(UpperCAmelCase_					,				UpperCAmelCase_		):
  ua += "; " + user_agent
 return ua





def __snake_case					(      UpperCAmelCase_						:						str					,				UpperCAmelCase_						:						Optional[str] = None					,				UpperCAmelCase_						:						Optional[str] = None		):
 if token is None:
  lowerCamelCase_      =  HfFolder.get_token()
 if organization is None:
  lowerCamelCase_      =  whoami(UpperCAmelCase_		)["name"]
  return F'''{username}/{model_id}'''
 else:
  return F'''{organization}/{model_id}'''





def __snake_case					(      UpperCAmelCase_						:						str					,				UpperCAmelCase_						:						str		):
 if not is_jinja_available():
  raise ValueError(
      "Modelcard rendering is based on Jinja templates."
      " Please make sure to have `jinja` installed before using `create_model_card`."
      " To install it, please run `pip install Jinja2`."		)

 if hasattr(UpperCAmelCase_					,				"local_rank"		) and args.local_rank not in [-1, 0]:
  return

 lowerCamelCase_      =  args.hub_token if hasattr(UpperCAmelCase_					,				"hub_token"		) else None
 lowerCamelCase_      =  get_full_repo_name(UpperCAmelCase_					,				token=UpperCAmelCase_		)

 lowerCamelCase_      =  ModelCard.from_template(
     card_data=ModelCardData(  # Card metadata object that will be converted to YAML block
         language="en"					,				license="apache-2.0"					,				library_name="diffusers"					,				tags=[]					,				datasets=args.dataset_name					,				metrics=[]					,				)					,				template_path=UpperCAmelCase_					,				model_name=UpperCAmelCase_					,				repo_name=UpperCAmelCase_					,				dataset_name=args.dataset_name if hasattr(UpperCAmelCase_					,				"dataset_name"		) else None					,				learning_rate=args.learning_rate					,				train_batch_size=args.train_batch_size					,				eval_batch_size=args.eval_batch_size					,				gradient_accumulation_steps=(
         args.gradient_accumulation_steps if hasattr(UpperCAmelCase_					,				"gradient_accumulation_steps"		) else None
     )					,				adam_betaa=args.adam_betaa if hasattr(UpperCAmelCase_					,				"adam_beta1"		) else None					,				adam_betaa=args.adam_betaa if hasattr(UpperCAmelCase_					,				"adam_beta2"		) else None					,				adam_weight_decay=args.adam_weight_decay if hasattr(UpperCAmelCase_					,				"adam_weight_decay"		) else None					,				adam_epsilon=args.adam_epsilon if hasattr(UpperCAmelCase_					,				"adam_epsilon"		) else None					,				lr_scheduler=args.lr_scheduler if hasattr(UpperCAmelCase_					,				"lr_scheduler"		) else None					,				lr_warmup_steps=args.lr_warmup_steps if hasattr(UpperCAmelCase_					,				"lr_warmup_steps"		) else None					,				ema_inv_gamma=args.ema_inv_gamma if hasattr(UpperCAmelCase_					,				"ema_inv_gamma"		) else None					,				ema_power=args.ema_power if hasattr(UpperCAmelCase_					,				"ema_power"		) else None					,				ema_max_decay=args.ema_max_decay if hasattr(UpperCAmelCase_					,				"ema_max_decay"		) else None					,				mixed_precision=args.mixed_precision					,				)

 lowerCamelCase_      =  os.path.join(args.output_dir					,				"README.md"		)
 model_card.save(UpperCAmelCase_		)





def __snake_case					(      UpperCAmelCase_						:						Optional[str]					,				UpperCAmelCase_						:						Optional[str] = None		):
 if resolved_file is None or commit_hash is not None:
  return commit_hash
 lowerCamelCase_      =  str(Path(UpperCAmelCase_		).as_posix()		)
 lowerCamelCase_      =  re.search(r"snapshots/([^/]+)/"					,				UpperCAmelCase_		)
 if search is None:
  return None
 lowerCamelCase_      =  search.groups()[0]
 return commit_hash if REGEX_COMMIT_HASH.match(UpperCAmelCase_		) else None


# Old default cache path, potentially to be migrated.
# This logic was more or less taken from `transformers`, with the following differences:
# - Diffusers doesn't use custom environment variables to specify the cache path.
# - There is no need to migrate the cache format, just move the files to the new location.
a_  :				Dict						= os.path.expanduser(
    os.getenv("""HF_HOME""", os.path.join(os.getenv("""XDG_CACHE_HOME""", """~/.cache"""), """huggingface"""))
)
a_  :				Any						= os.path.join(hf_cache_home, """diffusers""")





def __snake_case					(      UpperCAmelCase_						:						Optional[str] = None					,				UpperCAmelCase_						:						Optional[str] = None		):
 if new_cache_dir is None:
  lowerCamelCase_      =  DIFFUSERS_CACHE
 if old_cache_dir is None:
  lowerCamelCase_      =  old_diffusers_cache

 lowerCamelCase_      =  Path(UpperCAmelCase_		).expanduser()
 lowerCamelCase_      =  Path(UpperCAmelCase_		).expanduser()
 for old_blob_path in old_cache_dir.glob("**/blobs/*"		):
  if old_blob_path.is_file() and not old_blob_path.is_symlink():
   lowerCamelCase_      =  new_cache_dir / old_blob_path.relative_to(UpperCAmelCase_		)
   new_blob_path.parent.mkdir(parents=UpperCAmelCase_					,				exist_ok=UpperCAmelCase_		)
   os.replace(UpperCAmelCase_					,				UpperCAmelCase_		)
   try:
    os.symlink(UpperCAmelCase_					,				UpperCAmelCase_		)
   except OSError:
    logger.warning(
        "Could not create symlink between old cache and new cache. If you use an older version of diffusers again, files will be re-downloaded."		)
 # At this point, old_cache_dir contains symlinks to the new cache (it can still be used).


a_  :				str						= os.path.join(DIFFUSERS_CACHE, """version_diffusers_cache.txt""")
if not os.path.isfile(cache_version_file):
     a_  :				Union[str, Any]						= 0
else:
     with open(cache_version_file) as f:
          try:
               a_  :				int						= int(f.read())
          except ValueError:
               a_  :				List[str]						= 0

if cache_version < 1:
     a_  :				Optional[int]						= os.path.isdir(old_diffusers_cache) and len(os.listdir(old_diffusers_cache)) > 0
     if old_cache_is_not_empty:
          logger.warning(
              """The cache for model files in Diffusers v0.14.0 has moved to a new location. Moving your """
              """existing cached models. This is a one-time operation, you can interrupt it or run it """
              """later by calling `diffusers.utils.hub_utils.move_cache()`."""
          )
          try:
               move_cache()
          except Exception as e:
               a_  :				Any						= """\n""".join(traceback.format_tb(e.__traceback__))
               logger.error(
                   f'''There was a problem when trying to move your cache:\n\n{trace}\n{e.__class__.__name__}: {e}\n\nPlease '''
                   """file an issue at https://github.com/huggingface/diffusers/issues/new/choose, copy paste this whole """
                   """message and we will do our best to help."""
               )

if cache_version < 1:
     try:
          os.makedirs(DIFFUSERS_CACHE, exist_ok=True)
          with open(cache_version_file, """w""") as f:
               f.write("""1""")
     except Exception:
          logger.warning(
              f'''There was a problem when trying to write in your cache folder ({DIFFUSERS_CACHE}). Please, ensure '''
              """the directory exists and can be written to."""
          )





def __snake_case					(      UpperCAmelCase_						:						str					,				UpperCAmelCase_						:						Optional[str] = None		):
 if variant is not None:
  lowerCamelCase_      =  weights_name.split("."		)
  lowerCamelCase_      =  splits[:-1] + [variant] + splits[-1:]
  lowerCamelCase_      =  ".".join(UpperCAmelCase_		)

 return weights_name







def __snake_case					(      UpperCAmelCase_						:						List[str]					,				*,
    UpperCAmelCase_						:						Dict					,				UpperCAmelCase_						:						List[str]					,				UpperCAmelCase_						:						List[Any]					,				UpperCAmelCase_						:						List[Any]					,				UpperCAmelCase_						:						List[str]					,				UpperCAmelCase_						:						Dict					,				UpperCAmelCase_						:						int					,				UpperCAmelCase_						:						List[str]					,				UpperCAmelCase_						:						Dict					,				UpperCAmelCase_						:						Any					,				UpperCAmelCase_						:						Optional[int]=None					,				):
 lowerCamelCase_      =  str(UpperCAmelCase_		)
 if os.path.isfile(UpperCAmelCase_		):
  return pretrained_model_name_or_path
 elif os.path.isdir(UpperCAmelCase_		):
  if os.path.isfile(os.path.join(UpperCAmelCase_					,				UpperCAmelCase_		)		):
   # Load from a PyTorch checkpoint
   lowerCamelCase_      =  os.path.join(UpperCAmelCase_					,				UpperCAmelCase_		)
   return model_file
  elif subfolder is not None and os.path.isfile(
      os.path.join(UpperCAmelCase_					,				UpperCAmelCase_					,				UpperCAmelCase_		)		):
   lowerCamelCase_      =  os.path.join(UpperCAmelCase_					,				UpperCAmelCase_					,				UpperCAmelCase_		)
   return model_file
  else:
   raise EnvironmentError(
       F'''Error no file named {weights_name} found in directory {pretrained_model_name_or_path}.'''		)
 else:
  # 1. First check if deprecated way of loading from branches is used
  if (
      revision in DEPRECATED_REVISION_ARGS
      and (weights_name == WEIGHTS_NAME or weights_name == SAFETENSORS_WEIGHTS_NAME)
      and version.parse(version.parse(UpperCAmelCase_		).base_version		) >= version.parse("0.20.0"		)
  ):
   try:
    lowerCamelCase_      =  hf_hub_download(
        UpperCAmelCase_					,				filename=_add_variant(UpperCAmelCase_					,				UpperCAmelCase_		)					,				cache_dir=UpperCAmelCase_					,				force_download=UpperCAmelCase_					,				proxies=UpperCAmelCase_					,				resume_download=UpperCAmelCase_					,				local_files_only=UpperCAmelCase_					,				use_auth_token=UpperCAmelCase_					,				user_agent=UpperCAmelCase_					,				subfolder=UpperCAmelCase_					,				revision=revision or commit_hash					,				)
    warnings.warn(
        F'''Loading the variant {revision} from {pretrained_model_name_or_path} via `revision=\'{revision}\'` is deprecated. Loading instead from `revision=\'main\'` with `variant={revision}`. Loading model variants via `revision=\'{revision}\'` will be removed in diffusers v1. Please use `variant=\'{revision}\'` instead.'''					,				UpperCAmelCase_					,				)
    return model_file
   except:  # noqa: E722
    warnings.warn(
        F'''You are loading the variant {revision} from {pretrained_model_name_or_path} via `revision=\'{revision}\'`. This behavior is deprecated and will be removed in diffusers v1. One should use `variant=\'{revision}\'` instead. However, it appears that {pretrained_model_name_or_path} currently does not have a {_add_variant(UpperCAmelCase_					,				UpperCAmelCase_		)} file in the \'main\' branch of {pretrained_model_name_or_path}. \n The Diffusers team and community would be very grateful if you could open an issue: https://github.com/huggingface/diffusers/issues/new with the title \'{pretrained_model_name_or_path} is missing {_add_variant(UpperCAmelCase_					,				UpperCAmelCase_		)}\' so that the correct variant file can be added.'''					,				UpperCAmelCase_					,				)
  try:
   # 2. Load model file as usual
   lowerCamelCase_      =  hf_hub_download(
       UpperCAmelCase_					,				filename=UpperCAmelCase_					,				cache_dir=UpperCAmelCase_					,				force_download=UpperCAmelCase_					,				proxies=UpperCAmelCase_					,				resume_download=UpperCAmelCase_					,				local_files_only=UpperCAmelCase_					,				use_auth_token=UpperCAmelCase_					,				user_agent=UpperCAmelCase_					,				subfolder=UpperCAmelCase_					,				revision=revision or commit_hash					,				)
   return model_file

  except RepositoryNotFoundError:
   raise EnvironmentError(
       F'''{pretrained_model_name_or_path} is not a local folder and is not a valid model identifier '''
       "listed on 'https://huggingface.co/models'\nIf this is a private repository, make sure to pass a "
       "token having permission to this repo with `use_auth_token` or log in with `huggingface-cli "
       "login`."		)
  except RevisionNotFoundError:
   raise EnvironmentError(
       F'''{revision} is not a valid git identifier (branch name, tag name or commit id) that exists for '''
       "this model name. Check the model page at "
       F'''\'https://huggingface.co/{pretrained_model_name_or_path}\' for available revisions.'''		)
  except EntryNotFoundError:
   raise EnvironmentError(
       F'''{pretrained_model_name_or_path} does not appear to have a file named {weights_name}.'''		)
  except HTTPError as err:
   raise EnvironmentError(
       F'''There was a specific connection error when trying to load {pretrained_model_name_or_path}:\n{err}'''		)
  except ValueError:
   raise EnvironmentError(
       F'''We couldn\'t connect to \'{HUGGINGFACE_CO_RESOLVE_ENDPOINT}\' to load this model, couldn\'t find it'''
       F''' in the cached files and it looks like {pretrained_model_name_or_path} is not the path to a'''
       F''' directory containing a file named {weights_name} or'''
       " \nCheckout your internet connection or see how to run the library in"
       " offline mode at 'https://huggingface.co/docs/diffusers/installation#offline-mode'."		)
  except EnvironmentError:
   raise EnvironmentError(
       F'''Can\'t load the model for \'{pretrained_model_name_or_path}\'. If you were trying to load it from '''
       "'https://huggingface.co/models', make sure you don't have a local directory with the same name. "
       F'''Otherwise, make sure \'{pretrained_model_name_or_path}\' is the correct path to a directory '''
       F'''containing a file named {weights_name}'''		)










import pytest

from datasets.utils.sharding import _distribute_shards, _number_of_shards_in_gen_kwargs, _split_gen_kwargs

@pytest.mark.parametrize(
    'kwargs, expected',	[
        ({'num_shards': 0, 'max_num_jobs': 1}, []),
        ({'num_shards': 10, 'max_num_jobs': 1}, [range(10	)]),
        ({'num_shards': 10, 'max_num_jobs': 10}, [range(lowerCAmelCase_,	i + 1	) for i in range(10	)]),
        ({'num_shards': 1, 'max_num_jobs': 10}, [range(1	)]),
        ({'num_shards': 10, 'max_num_jobs': 3}, [range(0,	4	), range(4,	7	), range(7,	10	)]),
        ({'num_shards': 3, 'max_num_jobs': 10}, [range(0,	1	), range(1,	2	), range(2,	3	)]),
    ],	)
def   a_    ( lowerCAmelCase_      :			Optional[Any],	lowerCAmelCase_      :			Any	):
   __lowerCAmelCase								=	_distribute_shards(**lowerCAmelCase_	)
   assert out == expected

@pytest.mark.parametrize(
    'gen_kwargs, max_num_jobs, expected',	[
        ({'foo': 0}, 10, [{'foo': 0}]),
        ({'shards': [0, 1, 2, 3]}, 1, [{'shards': [0, 1, 2, 3]}]),
        ({'shards': [0, 1, 2, 3]}, 4, [{'shards': [0]}, {'shards': [1]}, {'shards': [2]}, {'shards': [3]}]),
        ({'shards': [0, 1]}, 4, [{'shards': [0]}, {'shards': [1]}]),
        ({'shards': [0, 1, 2, 3]}, 2, [{'shards': [0, 1]}, {'shards': [2, 3]}]),
    ],	)
def   a_    ( lowerCAmelCase_      :			Optional[Any],	lowerCAmelCase_      :			List[str],	lowerCAmelCase_      :			Optional[int]	):
   __lowerCAmelCase								=	_split_gen_kwargs(lowerCAmelCase_,	lowerCAmelCase_	)
   assert out == expected




@pytest.mark.parametrize(
    'gen_kwargs, expected',	[
        ({'foo': 0}, 1),
        ({'shards': [0]}, 1),
        ({'shards': [0, 1, 2, 3]}, 4),
        ({'shards': [0, 1, 2, 3], 'foo': 0}, 4),
        ({'shards': [0, 1, 2, 3], 'other': (0, 1)}, 4),
        ({'shards': [0, 1, 2, 3], 'shards2': [0, 1]}, RuntimeError),
    ],	)
def   a_    ( lowerCAmelCase_      :			int,	lowerCAmelCase_      :			Any	):
   if expected is RuntimeError:
      with pytest.raises(lowerCAmelCase_	):
         _number_of_shards_in_gen_kwargs(lowerCAmelCase_	)
   else:
      __lowerCAmelCase								=	_number_of_shards_in_gen_kwargs(lowerCAmelCase_	)
      assert out == expected

