





import os
from pickle import UnpicklingError
from typing import Dict, Tuple

import jax
import jax.numpy as jnp
import numpy as np
from flax.serialization import from_bytes
from flax.traverse_util import flatten_dict, unflatten_dict

import transformers

from .utils import logging


_snake_case  :    Optional[int]						    =  logging.get_logger(__name__)

def   a_    ( lowerCAmelCase_      :			Optional[Any],	lowerCAmelCase_      :			Any,	lowerCAmelCase_      :			Any,	lowerCAmelCase_      :			Any=False	):
   try:
      import torch  # noqa: F401
   except ImportError:
      logger.error(
          'Loading a PyTorch model in Flax, requires both PyTorch and Flax to be installed. Please see'
          ' https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation'
          ' instructions.'	)
      raise

   if not is_sharded:
      __lowerCAmelCase								=	os.path.abspath(lowerCAmelCase_	)
      logger.info(F"""Loading PyTorch weights from {pt_path}"""	)

      __lowerCAmelCase								=	torch.load(lowerCAmelCase_,	map_location='cpu'	)
      logger.info(F"""PyTorch checkpoint contains {sum(t.numel() for t in pt_state_dict.values()	):,} parameters."""	)

      __lowerCAmelCase								=	convert_pytorch_state_dict_to_flax(lowerCAmelCase_,	lowerCAmelCase_	)
   else:
      # model is sharded and pytorch_checkpoint_path already contains the list of .pt shard files
      __lowerCAmelCase								=	convert_pytorch_sharded_state_dict_to_flax(lowerCAmelCase_,	lowerCAmelCase_	)
   return flax_state_dict

def   a_    ( lowerCAmelCase_      :			Tuple[str],	lowerCAmelCase_      :			np.ndarray,	lowerCAmelCase_      :			Dict[str, jnp.ndarray],	lowerCAmelCase_      :			str,	):

   def is_key_or_prefix_key_in_dict(lowerCAmelCase_      :			Tuple[str]	) -> bool:
      return len(set(lowerCAmelCase_	) & {key, (model_prefix,) + key}	) > 0

   # layer norm
   __lowerCAmelCase								=	pt_tuple_key[:-1] + ('scale',)
   if pt_tuple_key[-1] in ["weight", "gamma"] and is_key_or_prefix_key_in_dict(lowerCAmelCase_	):
      return renamed_pt_tuple_key, pt_tensor

   # batch norm layer mean
   __lowerCAmelCase								=	pt_tuple_key[:-1] + ('mean',)
   if pt_tuple_key[-1] == "running_mean" and not is_key_or_prefix_key_in_dict(lowerCAmelCase_	):
      return renamed_pt_tuple_key, pt_tensor

   # batch norm layer var
   __lowerCAmelCase								=	pt_tuple_key[:-1] + ('var',)
   if pt_tuple_key[-1] == "running_var" and not is_key_or_prefix_key_in_dict(lowerCAmelCase_	):
      return renamed_pt_tuple_key, pt_tensor

   # embedding
   __lowerCAmelCase								=	pt_tuple_key[:-1] + ('embedding',)
   if pt_tuple_key[-1] == "weight" and is_key_or_prefix_key_in_dict(lowerCAmelCase_	):
      return renamed_pt_tuple_key, pt_tensor

   # conv layer
   __lowerCAmelCase								=	pt_tuple_key[:-1] + ('kernel',)
   if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4 and not is_key_or_prefix_key_in_dict(lowerCAmelCase_	):
      __lowerCAmelCase								=	pt_tensor.transpose(2,	3,	1,	0	)
      return renamed_pt_tuple_key, pt_tensor

   # linear layer
   __lowerCAmelCase								=	pt_tuple_key[:-1] + ('kernel',)
   if pt_tuple_key[-1] == "weight" and not is_key_or_prefix_key_in_dict(lowerCAmelCase_	):
      __lowerCAmelCase								=	pt_tensor.T
      return renamed_pt_tuple_key, pt_tensor

   # old PyTorch layer norm weight
   __lowerCAmelCase								=	pt_tuple_key[:-1] + ('weight',)
   if pt_tuple_key[-1] == "gamma":
      return renamed_pt_tuple_key, pt_tensor

   # old PyTorch layer norm bias
   __lowerCAmelCase								=	pt_tuple_key[:-1] + ('bias',)
   if pt_tuple_key[-1] == "beta":
      return renamed_pt_tuple_key, pt_tensor

   # New `weight_norm` from https://github.com/huggingface/transformers/pull/24030
   __lowerCAmelCase								=	None
   if pt_tuple_key[-3::2] == ("parametrizations", "original0"):
      __lowerCAmelCase								=	pt_tuple_key[-2] + '_g'
   elif pt_tuple_key[-3::2] == ("parametrizations", "original1"):
      __lowerCAmelCase								=	pt_tuple_key[-2] + '_v'
   if name is not None:
      __lowerCAmelCase								=	pt_tuple_key[:-3] + (name,)
      return renamed_pt_tuple_key, pt_tensor

   return pt_tuple_key, pt_tensor

def   a_    ( lowerCAmelCase_      :			Tuple,	lowerCAmelCase_      :			Tuple	):
   # convert pytorch tensor to numpy
   __lowerCAmelCase								=	{k: v.numpy() for k, v in pt_state_dict.items()}

   __lowerCAmelCase								=	flax_model.base_model_prefix

   # use params dict if the model contains batch norm layers
   if "params" in flax_model.params:
      __lowerCAmelCase								=	flax_model.params['params']
   else:
      __lowerCAmelCase								=	flax_model.params
   __lowerCAmelCase								=	flatten_dict(lowerCAmelCase_	)

   # add batch_stats keys,values to dict
   if "batch_stats" in flax_model.params:
      __lowerCAmelCase								=	flatten_dict(flax_model.params['batch_stats']	)
      random_flax_state_dict.update(lowerCAmelCase_	)

   __lowerCAmelCase								=	{}

   __lowerCAmelCase								=	(model_prefix not in flax_model_params) and (
       model_prefix in {k.split('.'	)[0] for k in pt_state_dict.keys()}
   )
   __lowerCAmelCase								=	(model_prefix in flax_model_params) and (
       model_prefix not in {k.split('.'	)[0] for k in pt_state_dict.keys()}
   )

   # Need to change some parameters name to match Flax names
   for pt_key, pt_tensor in pt_state_dict.items():
      __lowerCAmelCase								=	tuple(pt_key.split('.'	)	)

      # remove base model prefix if necessary
      __lowerCAmelCase								=	pt_tuple_key[0] == model_prefix
      if load_model_with_head_into_base_model and has_base_model_prefix:
         __lowerCAmelCase								=	pt_tuple_key[1:]

      # Correctly rename weight parameters
      __lowerCAmelCase	,	__lowerCAmelCase								=	rename_key_and_reshape_tensor(
          lowerCAmelCase_,	lowerCAmelCase_,	lowerCAmelCase_,	lowerCAmelCase_	)

      # add model prefix if necessary
      __lowerCAmelCase								=	(model_prefix,) + flax_key in random_flax_state_dict
      if load_base_model_into_model_with_head and require_base_model_prefix:
         __lowerCAmelCase								=	(model_prefix,) + flax_key

      if flax_key in random_flax_state_dict:
         if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
            raise ValueError(
                F"""PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape """
                F"""{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}."""	)

        # add batch stats if the model contains batchnorm layers
      if "batch_stats" in flax_model.params:
         if "mean" in flax_key[-1] or "var" in flax_key[-1]:
            __lowerCAmelCase								=	jnp.asarray(lowerCAmelCase_	)
            continue
         # remove num_batches_tracked key
         if "num_batches_tracked" in flax_key[-1]:
            flax_state_dict.pop(lowerCAmelCase_,	lowerCAmelCase_	)
            continue

         # also add unexpected weight so that warning is thrown
         __lowerCAmelCase								=	jnp.asarray(lowerCAmelCase_	)

      else:
         # also add unexpected weight so that warning is thrown
         __lowerCAmelCase								=	jnp.asarray(lowerCAmelCase_	)

   return unflatten_dict(lowerCAmelCase_	)

def   a_    ( lowerCAmelCase_      :			str,	lowerCAmelCase_      :			int	):
   import torch

   # Load the index
   __lowerCAmelCase								=	{}
   for shard_file in shard_filenames:
      # load using msgpack utils
      __lowerCAmelCase								=	torch.load(lowerCAmelCase_	)
      __lowerCAmelCase								=	{k: v.numpy() for k, v in pt_state_dict.items()}

      __lowerCAmelCase								=	flax_model.base_model_prefix

      # use params dict if the model contains batch norm layers and then add batch_stats keys,values to dict
      if "batch_stats" in flax_model.params:
         __lowerCAmelCase								=	flax_model.params['params']

         __lowerCAmelCase								=	flatten_dict(lowerCAmelCase_	)
         random_flax_state_dict.update(flatten_dict(flax_model.params['batch_stats']	)	)
      else:
         __lowerCAmelCase								=	flax_model.params
         __lowerCAmelCase								=	flatten_dict(lowerCAmelCase_	)

      __lowerCAmelCase								=	(model_prefix not in flax_model_params) and (
          model_prefix in {k.split('.'	)[0] for k in pt_state_dict.keys()}
      )
      __lowerCAmelCase								=	(model_prefix in flax_model_params) and (
          model_prefix not in {k.split('.'	)[0] for k in pt_state_dict.keys()}
      )
      # Need to change some parameters name to match Flax names
      for pt_key, pt_tensor in pt_state_dict.items():
         __lowerCAmelCase								=	tuple(pt_key.split('.'	)	)

         # remove base model prefix if necessary
         __lowerCAmelCase								=	pt_tuple_key[0] == model_prefix
         if load_model_with_head_into_base_model and has_base_model_prefix:
            __lowerCAmelCase								=	pt_tuple_key[1:]

         # Correctly rename weight parameters
         __lowerCAmelCase	,	__lowerCAmelCase								=	rename_key_and_reshape_tensor(
             lowerCAmelCase_,	lowerCAmelCase_,	lowerCAmelCase_,	lowerCAmelCase_	)
         # add model prefix if necessary
         __lowerCAmelCase								=	(model_prefix,) + flax_key in random_flax_state_dict
         if load_base_model_into_model_with_head and require_base_model_prefix:
            __lowerCAmelCase								=	(model_prefix,) + flax_key

         if flax_key in random_flax_state_dict:
            if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
               raise ValueError(
                   F"""PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape """
                   F"""{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}."""	)

            # add batch stats if the model contains batchnorm layers
         if "batch_stats" in flax_model.params:
            if "mean" in flax_key[-1]:
               __lowerCAmelCase								=	jnp.asarray(lowerCAmelCase_	)
               continue
            if "var" in flax_key[-1]:
               __lowerCAmelCase								=	jnp.asarray(lowerCAmelCase_	)
               continue
            # remove num_batches_tracked key
            if "num_batches_tracked" in flax_key[-1]:
               flax_state_dict.pop(lowerCAmelCase_,	lowerCAmelCase_	)
               continue

            # also add unexpected weight so that warning is thrown
            __lowerCAmelCase								=	jnp.asarray(lowerCAmelCase_	)

         else:
            # also add unexpected weight so that warning is thrown
            __lowerCAmelCase								=	jnp.asarray(lowerCAmelCase_	)
   return unflatten_dict(lowerCAmelCase_	)

def   a_    ( lowerCAmelCase_      :			Dict,	lowerCAmelCase_      :			List[Any]	):
   __lowerCAmelCase								=	os.path.abspath(lowerCAmelCase_	)
   logger.info(F"""Loading Flax weights from {flax_checkpoint_path}"""	)

   # import correct flax class
   __lowerCAmelCase								=	getattr(lowerCAmelCase_,	'Flax' + model.__class__.__name__	)

   # load flax weight dict
   with open(lowerCAmelCase_,	'rb'	) as state_f:
      try:
         __lowerCAmelCase								=	from_bytes(lowerCAmelCase_,	state_f.read()	)
      except UnpicklingError:
         raise EnvironmentError(F"""Unable to convert {flax_checkpoint_path} to Flax deserializable object. """	)

   return load_flax_weights_in_pytorch_model(lowerCAmelCase_,	lowerCAmelCase_	)




def   a_    ( lowerCAmelCase_      :			Optional[int],	lowerCAmelCase_      :			Optional[Any]	):

   try:
      import torch  # noqa: F401
   except ImportError:
      logger.error(
          'Loading a Flax weights in PyTorch, requires both PyTorch and Flax to be installed. Please see'
          ' https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation'
          ' instructions.'	)
      raise

   # check if we have bf16 weights
   __lowerCAmelCase								=	flatten_dict(jax.tree_util.tree_map(lambda lowerCAmelCase_	: x.dtype == jnp.bfloataa,	lowerCAmelCase_	)	).values()
   if any(lowerCAmelCase_	):
      # convert all weights to fp32 if the are bf16 since torch.from_numpy can-not handle bf16
      # and bf16 is not fully supported in PT yet.
      logger.warning(
          'Found ``bfloat16`` weights in Flax model. Casting all ``bfloat16`` weights to ``float32`` '
          'before loading those in PyTorch model.'	)
      __lowerCAmelCase								=	jax.tree_util.tree_map(
          lambda lowerCAmelCase_	: params.astype(np.floataa	) if params.dtype == jnp.bfloataa else params,	lowerCAmelCase_	)

   __lowerCAmelCase								=	flatten_dict(lowerCAmelCase_	)
   __lowerCAmelCase								=	pt_model.state_dict()

   __lowerCAmelCase								=	(pt_model.base_model_prefix in flax_state) and (
       pt_model.base_model_prefix not in {k.split('.'	)[0] for k in pt_model_dict.keys()}
   )
   __lowerCAmelCase								=	(pt_model.base_model_prefix not in flax_state) and (
       pt_model.base_model_prefix in {k.split('.'	)[0] for k in pt_model_dict.keys()}
   )

   # keep track of unexpected & missing keys
   __lowerCAmelCase								=	[]
   __lowerCAmelCase								=	set(pt_model_dict.keys()	)

   for flax_key_tuple, flax_tensor in flax_state_dict.items():
      __lowerCAmelCase								=	flax_key_tuple[0] == pt_model.base_model_prefix
      __lowerCAmelCase								=	'.'.join((pt_model.base_model_prefix,) + flax_key_tuple	) in pt_model_dict

      # adapt flax_key to prepare for loading from/to base model only
      if load_model_with_head_into_base_model and has_base_model_prefix:
         __lowerCAmelCase								=	flax_key_tuple[1:]
      elif load_base_model_into_model_with_head and require_base_model_prefix:
         __lowerCAmelCase								=	(pt_model.base_model_prefix,) + flax_key_tuple

      # rename flax weights to PyTorch format
      if flax_key_tuple[-1] == "kernel" and flax_tensor.ndim == 4 and ".".join(lowerCAmelCase_	) not in pt_model_dict:
         # conv layer
         __lowerCAmelCase								=	flax_key_tuple[:-1] + ('weight',)
         __lowerCAmelCase								=	jnp.transpose(lowerCAmelCase_,	(3, 2, 0, 1)	)
      elif flax_key_tuple[-1] == "kernel" and ".".join(lowerCAmelCase_	) not in pt_model_dict:
         # linear layer
         __lowerCAmelCase								=	flax_key_tuple[:-1] + ('weight',)
         __lowerCAmelCase								=	flax_tensor.T
      elif flax_key_tuple[-1] in ["scale", "embedding"]:
         __lowerCAmelCase								=	flax_key_tuple[:-1] + ('weight',)

      # adding batch stats from flax batch norm to pt
      elif "mean" in flax_key_tuple[-1]:
         __lowerCAmelCase								=	flax_key_tuple[:-1] + ('running_mean',)
      elif "var" in flax_key_tuple[-1]:
         __lowerCAmelCase								=	flax_key_tuple[:-1] + ('running_var',)

      if "batch_stats" in flax_state:
         __lowerCAmelCase								=	'.'.join(flax_key_tuple[1:]	)  # Remove the params/batch_stats header
      else:
         __lowerCAmelCase								=	'.'.join(lowerCAmelCase_	)

      # We also need to look at `pt_model_dict` and see if there are keys requiring further transformation.
      __lowerCAmelCase								=	{}
      # New `weight_norm` from https://github.com/huggingface/transformers/pull/24030
      for key in pt_model_dict:
         __lowerCAmelCase								=	key.split('.'	)
         __lowerCAmelCase								=	None
         if key_components[-3::2] == ["parametrizations", "original0"]:
            __lowerCAmelCase								=	key_components[-2] + '_g'
         elif key_components[-3::2] == ["parametrizations", "original1"]:
            __lowerCAmelCase								=	key_components[-2] + '_v'
         if name is not None:
            __lowerCAmelCase								=	key_components[:-3] + [name]
            __lowerCAmelCase								=	'.'.join(lowerCAmelCase_	)
            __lowerCAmelCase								=	key

      if flax_key in special_pt_names:
         __lowerCAmelCase								=	special_pt_names[flax_key]

      if flax_key in pt_model_dict:
         if flax_tensor.shape != pt_model_dict[flax_key].shape:
            raise ValueError(
                F"""Flax checkpoint seems to be incorrect. Weight {flax_key_tuple} was expected """
                F"""to be of shape {pt_model_dict[flax_key].shape}, but is {flax_tensor.shape}."""	)
         else:
            # add weight to pytorch dict
            __lowerCAmelCase								=	np.asarray(lowerCAmelCase_	) if not isinstance(lowerCAmelCase_,	np.ndarray	) else flax_tensor
            __lowerCAmelCase								=	torch.from_numpy(lowerCAmelCase_	)
            # remove from missing keys
            missing_keys.remove(lowerCAmelCase_	)
      else:
         # weight is not expected by PyTorch model
         unexpected_keys.append(lowerCAmelCase_	)

   pt_model.load_state_dict(lowerCAmelCase_	)

   # re-transform missing_keys to list
   __lowerCAmelCase								=	list(lowerCAmelCase_	)

   if len(lowerCAmelCase_	) > 0:
      logger.warning(
          'Some weights of the Flax model were not used when initializing the PyTorch model'
          F""" {pt_model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are initializing"""
          F""" {pt_model.__class__.__name__} from a Flax model trained on another task or with another architecture"""
          ' (e.g. initializing a BertForSequenceClassification model from a FlaxBertForPreTraining model).\n- This'
          F""" IS NOT expected if you are initializing {pt_model.__class__.__name__} from a Flax model that you expect"""
          ' to be exactly identical (e.g. initializing a BertForSequenceClassification model from a'
          ' FlaxBertForSequenceClassification model).'	)
   else:
      logger.warning(F"""All Flax model weights were used when initializing {pt_model.__class__.__name__}.\n"""	)
   if len(lowerCAmelCase_	) > 0:
      logger.warning(
          F"""Some weights of {pt_model.__class__.__name__} were not initialized from the Flax model and are newly"""
          F""" initialized: {missing_keys}\nYou should probably TRAIN this model on a down-stream task to be able to"""
          ' use it for predictions and inference.'	)
   else:
      logger.warning(
          F"""All the weights of {pt_model.__class__.__name__} were initialized from the Flax model.\n"""
          'If your task is similar to the task the model of the checkpoint was trained on, '
          F"""you can already use {pt_model.__class__.__name__} for predictions without further training."""	)

   return pt_model








from timeit import timeit

def   a_    ( lowerCAmelCase_      :			int	):
   if number < 0:
      raise ValueError('the value of input must not be negative'	)
   __lowerCAmelCase								=	0
   while number:
      number &= number - 1
      result += 1
   return result

def   a_    ( lowerCAmelCase_      :			int	):
   if number < 0:
      raise ValueError('the value of input must not be negative'	)
   __lowerCAmelCase								=	0
   while number:
      if number % 2 == 1:
         result += 1
      number >>= 1
   return result

def   a_    ( ):

   def do_benchmark(lowerCAmelCase_      :			int	) -> None:
      __lowerCAmelCase								=	'import __main__ as z'
      print(F"""Benchmark when {number = }:"""	)
      print(F"""{get_set_bits_count_using_modulo_operator(lowerCAmelCase_	) = }"""	)
      __lowerCAmelCase								=	timeit('z.get_set_bits_count_using_modulo_operator(25)',	setup=lowerCAmelCase_	)
      print(F"""timeit() runs in {timing} seconds"""	)
      print(F"""{get_set_bits_count_using_brian_kernighans_algorithm(lowerCAmelCase_	) = }"""	)
      __lowerCAmelCase								=	timeit(
          'z.get_set_bits_count_using_brian_kernighans_algorithm(25)',	setup=lowerCAmelCase_,	)
      print(F"""timeit() runs in {timing} seconds"""	)

   for number in (25, 37, 58, 0):
      do_benchmark(lowerCAmelCase_	)
      print()


if __name__ == "__main__":
     import doctest

     doctest.testmod()
     benchmark()

