



import os
from pickle import UnpicklingError
from typing import Dict, Tuple

import jax
import jax.numpy as jnp
import numpy as np
from flax.serialization import from_bytes
from flax.traverse_util import flatten_dict, unflatten_dict

import transformers

from .utils import logging


SCREAMING_SNAKE_CASE_:Optional[int]             =						logging.get_logger(__name__)




def 	__UpperCamelCase	(  _lowerCAmelCase      ,     _lowerCAmelCase      ,     _lowerCAmelCase      ,     _lowerCAmelCase=False		)       ->  List[Any]:



    """simple docstring"""






    try:
        import torch  # noqa: F401
    except ImportError:
        logger.error(
            """Loading a PyTorch model in Flax, requires both PyTorch and Flax to be installed. Please see"""
            """ https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation"""
            """ instructions."""		)
        raise

    if not is_sharded:
        A :       Dict          =   os.path.abspath(_lowerCAmelCase		)
        logger.info(f'''Loading PyTorch weights from {pt_path}'''		)

        A :       Optional[int]          =   torch.load(_lowerCAmelCase      ,     map_location="""cpu"""		)
        logger.info(f'''PyTorch checkpoint contains {sum(t.numel() for t in pt_state_dict.values()		):,} parameters.'''		)

        A :       List[Any]          =   convert_pytorch_state_dict_to_flax(_lowerCAmelCase      ,     _lowerCAmelCase		)
    else:
        # model is sharded and pytorch_checkpoint_path already contains the list of .pt shard files
        A :       List[Any]          =   convert_pytorch_sharded_state_dict_to_flax(_lowerCAmelCase      ,     _lowerCAmelCase		)
    return flax_state_dict




def 	__UpperCamelCase	(  _lowerCAmelCase      ,     _lowerCAmelCase      ,     _lowerCAmelCase      ,     _lowerCAmelCase      ,     )       ->  (Tuple[str], np.ndarray):



    """simple docstring"""






    def is_key_or_prefix_key_in_dict(_lowerCAmelCase		) -> bool:
        return len(set(_lowerCAmelCase		) & {key, (model_prefix,) + key}		) > 0

    # layer norm
    A :       Union[str, Any]          =   pt_tuple_key[:-1] + ("""scale""",)
    if pt_tuple_key[-1] in ["weight", "gamma"] and is_key_or_prefix_key_in_dict(_lowerCAmelCase		):
        return renamed_pt_tuple_key, pt_tensor

    # batch norm layer mean
    A :       Union[str, Any]          =   pt_tuple_key[:-1] + ("""mean""",)
    if pt_tuple_key[-1] == "running_mean" and not is_key_or_prefix_key_in_dict(_lowerCAmelCase		):
        return renamed_pt_tuple_key, pt_tensor

    # batch norm layer var
    A :       str          =   pt_tuple_key[:-1] + ("""var""",)
    if pt_tuple_key[-1] == "running_var" and not is_key_or_prefix_key_in_dict(_lowerCAmelCase		):
        return renamed_pt_tuple_key, pt_tensor

    # embedding
    A :       Optional[int]          =   pt_tuple_key[:-1] + ("""embedding""",)
    if pt_tuple_key[-1] == "weight" and is_key_or_prefix_key_in_dict(_lowerCAmelCase		):
        return renamed_pt_tuple_key, pt_tensor

    # conv layer
    A :       Union[str, Any]          =   pt_tuple_key[:-1] + ("""kernel""",)
    if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4 and not is_key_or_prefix_key_in_dict(_lowerCAmelCase		):
        A :       Optional[Any]          =   pt_tensor.transpose(2      ,     3      ,     1      ,     0		)
        return renamed_pt_tuple_key, pt_tensor

    # linear layer
    A :       Dict          =   pt_tuple_key[:-1] + ("""kernel""",)
    if pt_tuple_key[-1] == "weight" and not is_key_or_prefix_key_in_dict(_lowerCAmelCase		):
        A :       int          =   pt_tensor.T
        return renamed_pt_tuple_key, pt_tensor

    # old PyTorch layer norm weight
    A :       int          =   pt_tuple_key[:-1] + ("""weight""",)
    if pt_tuple_key[-1] == "gamma":
        return renamed_pt_tuple_key, pt_tensor

    # old PyTorch layer norm bias
    A :       List[str]          =   pt_tuple_key[:-1] + ("""bias""",)
    if pt_tuple_key[-1] == "beta":
        return renamed_pt_tuple_key, pt_tensor

    # New `weight_norm` from https://github.com/huggingface/transformers/pull/24030
    A :       Optional[Any]          =   None
    if pt_tuple_key[-3::2] == ("parametrizations", "original0"):
        A :       List[str]          =   pt_tuple_key[-2] + """_g"""
    elif pt_tuple_key[-3::2] == ("parametrizations", "original1"):
        A :       Any          =   pt_tuple_key[-2] + """_v"""
    if name is not None:
        A :       Any          =   pt_tuple_key[:-3] + (name,)
        return renamed_pt_tuple_key, pt_tensor

    return pt_tuple_key, pt_tensor




def 	__UpperCamelCase	(  _lowerCAmelCase      ,     _lowerCAmelCase		)       ->  Union[str, Any]:



    """simple docstring"""






    A :       Union[str, Any]          =   {k: v.numpy() for k, v in pt_state_dict.items()}

    A :       Optional[Any]          =   flax_model.base_model_prefix

    # use params dict if the model contains batch norm layers
    if "params" in flax_model.params:
        A :       Dict          =   flax_model.params["""params"""]
    else:
        A :       str          =   flax_model.params
    A :       Optional[int]          =   flatten_dict(_lowerCAmelCase		)

    # add batch_stats keys,values to dict
    if "batch_stats" in flax_model.params:
        A :       str          =   flatten_dict(flax_model.params["""batch_stats"""]		)
        random_flax_state_dict.update(_lowerCAmelCase		)

    A :       Any          =   {}

    A :       Optional[int]          =   (model_prefix not in flax_model_params) and (
        model_prefix in {k.split("""."""		)[0] for k in pt_state_dict.keys()}
    )
    A :       List[Any]          =   (model_prefix in flax_model_params) and (
        model_prefix not in {k.split("""."""		)[0] for k in pt_state_dict.keys()}
    )

    # Need to change some parameters name to match Flax names
    for pt_key, pt_tensor in pt_state_dict.items():
        A :       Optional[int]          =   tuple(pt_key.split("""."""		)		)

        # remove base model prefix if necessary
        A :       int          =   pt_tuple_key[0] == model_prefix
        if load_model_with_head_into_base_model and has_base_model_prefix:
            A :       Any          =   pt_tuple_key[1:]

        # Correctly rename weight parameters
        A     ,							A :       str          =   rename_key_and_reshape_tensor(
            _lowerCAmelCase      ,     _lowerCAmelCase      ,     _lowerCAmelCase      ,     _lowerCAmelCase		)

        # add model prefix if necessary
        A :       str          =   (model_prefix,) + flax_key in random_flax_state_dict
        if load_base_model_into_model_with_head and require_base_model_prefix:
            A :       List[Any]          =   (model_prefix,) + flax_key

        if flax_key in random_flax_state_dict:
            if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
                raise ValueError(
                    f'''PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape '''
                    f'''{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}.'''		)

        # add batch stats if the model contains batchnorm layers
        if "batch_stats" in flax_model.params:
            if "mean" in flax_key[-1] or "var" in flax_key[-1]:
                A :       Union[str, Any]          =   jnp.asarray(_lowerCAmelCase		)
                continue
            # remove num_batches_tracked key
            if "num_batches_tracked" in flax_key[-1]:
                flax_state_dict.pop(_lowerCAmelCase      ,     _lowerCAmelCase		)
                continue

            # also add unexpected weight so that warning is thrown
            A :       Optional[Any]          =   jnp.asarray(_lowerCAmelCase		)

        else:
            # also add unexpected weight so that warning is thrown
            A :       Optional[Any]          =   jnp.asarray(_lowerCAmelCase		)

    return unflatten_dict(_lowerCAmelCase		)




def 	__UpperCamelCase	(  _lowerCAmelCase      ,     _lowerCAmelCase		)       ->  Union[str, Any]:



    """simple docstring"""






    import torch

    # Load the index
    A :       Tuple          =   {}
    for shard_file in shard_filenames:
        # load using msgpack utils
        A :       Any          =   torch.load(_lowerCAmelCase		)
        A :       Optional[int]          =   {k: v.numpy() for k, v in pt_state_dict.items()}

        A :       Optional[int]          =   flax_model.base_model_prefix

        # use params dict if the model contains batch norm layers and then add batch_stats keys,values to dict
        if "batch_stats" in flax_model.params:
            A :       Optional[int]          =   flax_model.params["""params"""]

            A :       Optional[int]          =   flatten_dict(_lowerCAmelCase		)
            random_flax_state_dict.update(flatten_dict(flax_model.params["""batch_stats"""]		)		)
        else:
            A :       Dict          =   flax_model.params
            A :       Any          =   flatten_dict(_lowerCAmelCase		)

        A :       Optional[Any]          =   (model_prefix not in flax_model_params) and (
            model_prefix in {k.split("""."""		)[0] for k in pt_state_dict.keys()}
        )
        A :       Optional[Any]          =   (model_prefix in flax_model_params) and (
            model_prefix not in {k.split("""."""		)[0] for k in pt_state_dict.keys()}
        )
        # Need to change some parameters name to match Flax names
        for pt_key, pt_tensor in pt_state_dict.items():
            A :       Optional[Any]          =   tuple(pt_key.split("""."""		)		)

            # remove base model prefix if necessary
            A :       Optional[Any]          =   pt_tuple_key[0] == model_prefix
            if load_model_with_head_into_base_model and has_base_model_prefix:
                A :       Tuple          =   pt_tuple_key[1:]

            # Correctly rename weight parameters
            A     ,							A :       Union[str, Any]          =   rename_key_and_reshape_tensor(
                _lowerCAmelCase      ,     _lowerCAmelCase      ,     _lowerCAmelCase      ,     _lowerCAmelCase		)
            # add model prefix if necessary
            A :       int          =   (model_prefix,) + flax_key in random_flax_state_dict
            if load_base_model_into_model_with_head and require_base_model_prefix:
                A :       Union[str, Any]          =   (model_prefix,) + flax_key

            if flax_key in random_flax_state_dict:
                if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
                    raise ValueError(
                        f'''PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape '''
                        f'''{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}.'''		)

            # add batch stats if the model contains batchnorm layers
            if "batch_stats" in flax_model.params:
                if "mean" in flax_key[-1]:
                    A :       int          =   jnp.asarray(_lowerCAmelCase		)
                    continue
                if "var" in flax_key[-1]:
                    A :       Any          =   jnp.asarray(_lowerCAmelCase		)
                    continue
                # remove num_batches_tracked key
                if "num_batches_tracked" in flax_key[-1]:
                    flax_state_dict.pop(_lowerCAmelCase      ,     _lowerCAmelCase		)
                    continue

                # also add unexpected weight so that warning is thrown
                A :       str          =   jnp.asarray(_lowerCAmelCase		)

            else:
                # also add unexpected weight so that warning is thrown
                A :       Dict          =   jnp.asarray(_lowerCAmelCase		)
    return unflatten_dict(_lowerCAmelCase		)




def 	__UpperCamelCase	(  _lowerCAmelCase      ,     _lowerCAmelCase		)       ->  Union[str, Any]:



    """simple docstring"""






    A :       Tuple          =   os.path.abspath(_lowerCAmelCase		)
    logger.info(f'''Loading Flax weights from {flax_checkpoint_path}'''		)

    # import correct flax class
    A :       Tuple          =   getattr(_lowerCAmelCase      ,     """Flax""" + model.__class__.__name__		)

    # load flax weight dict
    with open(_lowerCAmelCase      ,     """rb"""		) as state_f:
        try:
            A :       Optional[Any]          =   from_bytes(_lowerCAmelCase      ,     state_f.read()		)
        except UnpicklingError:
            raise EnvironmentError(f'''Unable to convert {flax_checkpoint_path} to Flax deserializable object. '''		)

    return load_flax_weights_in_pytorch_model(_lowerCAmelCase      ,     _lowerCAmelCase		)




def 	__UpperCamelCase	(  _lowerCAmelCase      ,     _lowerCAmelCase		)       ->  int:



    """simple docstring"""






    try:
        import torch  # noqa: F401
    except ImportError:
        logger.error(
            """Loading a Flax weights in PyTorch, requires both PyTorch and Flax to be installed. Please see"""
            """ https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation"""
            """ instructions."""		)
        raise

    # check if we have bf16 weights
    A :       List[str]          =   flatten_dict(jax.tree_util.tree_map(lambda _lowerCAmelCase		: x.dtype == jnp.bfloataa      ,     _lowerCAmelCase		)		).values()
    if any(_lowerCAmelCase		):
        # convert all weights to fp32 if the are bf16 since torch.from_numpy can-not handle bf16
        # and bf16 is not fully supported in PT yet.
        logger.warning(
            """Found ``bfloat16`` weights in Flax model. Casting all ``bfloat16`` weights to ``float32`` """
            """before loading those in PyTorch model."""		)
        A :       List[str]          =   jax.tree_util.tree_map(
            lambda _lowerCAmelCase		: params.astype(np.floataa		) if params.dtype == jnp.bfloataa else params      ,     _lowerCAmelCase		)

    A :       int          =   flatten_dict(_lowerCAmelCase		)
    A :       List[Any]          =   pt_model.state_dict()

    A :       Optional[int]          =   (pt_model.base_model_prefix in flax_state) and (
        pt_model.base_model_prefix not in {k.split("""."""		)[0] for k in pt_model_dict.keys()}
    )
    A :       Optional[Any]          =   (pt_model.base_model_prefix not in flax_state) and (
        pt_model.base_model_prefix in {k.split("""."""		)[0] for k in pt_model_dict.keys()}
    )

    # keep track of unexpected & missing keys
    A :       Dict          =   []
    A :       Dict          =   set(pt_model_dict.keys()		)

    for flax_key_tuple, flax_tensor in flax_state_dict.items():
        A :       str          =   flax_key_tuple[0] == pt_model.base_model_prefix
        A :       List[str]          =   """.""".join((pt_model.base_model_prefix,) + flax_key_tuple		) in pt_model_dict

        # adapt flax_key to prepare for loading from/to base model only
        if load_model_with_head_into_base_model and has_base_model_prefix:
            A :       List[Any]          =   flax_key_tuple[1:]
        elif load_base_model_into_model_with_head and require_base_model_prefix:
            A :       Tuple          =   (pt_model.base_model_prefix,) + flax_key_tuple

        # rename flax weights to PyTorch format
        if flax_key_tuple[-1] == "kernel" and flax_tensor.ndim == 4 and ".".join(_lowerCAmelCase		) not in pt_model_dict:
            # conv layer
            A :       List[str]          =   flax_key_tuple[:-1] + ("""weight""",)
            A :       int          =   jnp.transpose(_lowerCAmelCase      ,     (3, 2, 0, 1)		)
        elif flax_key_tuple[-1] == "kernel" and ".".join(_lowerCAmelCase		) not in pt_model_dict:
            # linear layer
            A :       Any          =   flax_key_tuple[:-1] + ("""weight""",)
            A :       str          =   flax_tensor.T
        elif flax_key_tuple[-1] in ["scale", "embedding"]:
            A :       List[str]          =   flax_key_tuple[:-1] + ("""weight""",)

        # adding batch stats from flax batch norm to pt
        elif "mean" in flax_key_tuple[-1]:
            A :       List[str]          =   flax_key_tuple[:-1] + ("""running_mean""",)
        elif "var" in flax_key_tuple[-1]:
            A :       Any          =   flax_key_tuple[:-1] + ("""running_var""",)

        if "batch_stats" in flax_state:
            A :       Union[str, Any]          =   """.""".join(flax_key_tuple[1:]		)  # Remove the params/batch_stats header
        else:
            A :       List[Any]          =   """.""".join(_lowerCAmelCase		)

        # We also need to look at `pt_model_dict` and see if there are keys requiring further transformation.
        A :       Union[str, Any]          =   {}
        # New `weight_norm` from https://github.com/huggingface/transformers/pull/24030
        for key in pt_model_dict:
            A :       List[str]          =   key.split("""."""		)
            A :       Dict          =   None
            if key_components[-3::2] == ["parametrizations", "original0"]:
                A :       Union[str, Any]          =   key_components[-2] + """_g"""
            elif key_components[-3::2] == ["parametrizations", "original1"]:
                A :       List[Any]          =   key_components[-2] + """_v"""
            if name is not None:
                A :       Tuple          =   key_components[:-3] + [name]
                A :       Optional[int]          =   """.""".join(_lowerCAmelCase		)
                A :       Tuple          =   key

        if flax_key in special_pt_names:
            A :       Any          =   special_pt_names[flax_key]

        if flax_key in pt_model_dict:
            if flax_tensor.shape != pt_model_dict[flax_key].shape:
                raise ValueError(
                    f'''Flax checkpoint seems to be incorrect. Weight {flax_key_tuple} was expected '''
                    f'''to be of shape {pt_model_dict[flax_key].shape}, but is {flax_tensor.shape}.'''		)
            else:
                # add weight to pytorch dict
                A :       Optional[Any]          =   np.asarray(_lowerCAmelCase		) if not isinstance(_lowerCAmelCase      ,     np.ndarray		) else flax_tensor
                A :       Optional[int]          =   torch.from_numpy(_lowerCAmelCase		)
                # remove from missing keys
                missing_keys.remove(_lowerCAmelCase		)
        else:
            # weight is not expected by PyTorch model
            unexpected_keys.append(_lowerCAmelCase		)

    pt_model.load_state_dict(_lowerCAmelCase		)

    # re-transform missing_keys to list
    A :       Optional[int]          =   list(_lowerCAmelCase		)

    if len(_lowerCAmelCase		) > 0:
        logger.warning(
            """Some weights of the Flax model were not used when initializing the PyTorch model"""
            f''' {pt_model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are initializing'''
            f''' {pt_model.__class__.__name__} from a Flax model trained on another task or with another architecture'''
            """ (e.g. initializing a BertForSequenceClassification model from a FlaxBertForPreTraining model).\n- This"""
            f''' IS NOT expected if you are initializing {pt_model.__class__.__name__} from a Flax model that you expect'''
            """ to be exactly identical (e.g. initializing a BertForSequenceClassification model from a"""
            """ FlaxBertForSequenceClassification model)."""		)
    else:
        logger.warning(f'''All Flax model weights were used when initializing {pt_model.__class__.__name__}.\n'''		)
    if len(_lowerCAmelCase		) > 0:
        logger.warning(
            f'''Some weights of {pt_model.__class__.__name__} were not initialized from the Flax model and are newly'''
            f''' initialized: {missing_keys}\nYou should probably TRAIN this model on a down-stream task to be able to'''
            """ use it for predictions and inference."""		)
    else:
        logger.warning(
            f'''All the weights of {pt_model.__class__.__name__} were initialized from the Flax model.\n'''
            """If your task is similar to the task the model of the checkpoint was trained on, """
            f'''you can already use {pt_model.__class__.__name__} for predictions without further training.'''		)

    return pt_model







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


SCREAMING_SNAKE_CASE_:Dict             =						logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_:int             =						{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE_:Optional[Any]             =						[
    """small""",
    """small-base""",
    """medium""",
    """medium-base""",
    """intermediate""",
    """intermediate-base""",
    """large""",
    """large-base""",
    """xlarge""",
    """xlarge-base""",
]

SCREAMING_SNAKE_CASE_:int             =						{
    """vocab_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt""",
        """funnel-transformer/small-base""": """https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt""",
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt""",
        """funnel-transformer/large-base""": """https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """funnel-transformer/small""": """https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json""",
        """funnel-transformer/small-base""": (
            """https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/medium""": """https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json""",
        """funnel-transformer/medium-base""": (
            """https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate""": (
            """https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/intermediate-base""": (
            """https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/large""": """https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json""",
        """funnel-transformer/large-base""": (
            """https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"""
        ),
        """funnel-transformer/xlarge""": """https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json""",
        """funnel-transformer/xlarge-base""": (
            """https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"""
        ),
    },
}
SCREAMING_SNAKE_CASE_:int             =						{F"""funnel-transformer/{name}""": 512 for name in _model_names}
SCREAMING_SNAKE_CASE_:Dict             =						{F"""funnel-transformer/{name}""": {"""do_lower_case""": True} for name in _model_names}






class SCREAMING_SNAKE_CASE__     (		SCREAMING_SNAKE_CASE__    ):




    '''simple docstring'''






    __lowerCamelCase			: List[str]					=      VOCAB_FILES_NAMES
    __lowerCamelCase			: Union[str, Any]					=      PRETRAINED_VOCAB_FILES_MAP
    __lowerCamelCase			: str					=      PRETRAINED_INIT_CONFIGURATION
    __lowerCamelCase			: Tuple					=      FunnelTokenizer
    __lowerCamelCase			: str					=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __lowerCamelCase			: int					=      2






    def __init__(       self,		lowerCamelCase__=None,		lowerCamelCase__=None,		lowerCamelCase__=True,		lowerCamelCase__="<unk>",		lowerCamelCase__="<sep>",		lowerCamelCase__="<pad>",		lowerCamelCase__="<cls>",		lowerCamelCase__="<mask>",		lowerCamelCase__="<s>",		lowerCamelCase__="</s>",		lowerCamelCase__=True,		lowerCamelCase__=True,		lowerCamelCase__=None,		lowerCamelCase__="##",		**lowerCamelCase__,		):
        super().__init__(
            lowerCamelCase__,		tokenizer_file=lowerCamelCase__,		do_lower_case=lowerCamelCase__,		unk_token=lowerCamelCase__,		sep_token=lowerCamelCase__,		pad_token=lowerCamelCase__,		cls_token=lowerCamelCase__,		mask_token=lowerCamelCase__,		bos_token=lowerCamelCase__,		eos_token=lowerCamelCase__,		clean_text=lowerCamelCase__,		tokenize_chinese_chars=lowerCamelCase__,		strip_accents=lowerCamelCase__,		wordpieces_prefix=lowerCamelCase__,		**lowerCamelCase__,		)

        A :       List[str]          =   json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
        if (
            normalizer_state.get("""lowercase""",		lowerCamelCase__		) != do_lower_case
            or normalizer_state.get("""strip_accents""",		lowerCamelCase__		) != strip_accents
            or normalizer_state.get("""handle_chinese_chars""",		lowerCamelCase__		) != tokenize_chinese_chars
        ):
            A :       Optional[Any]          =   getattr(lowerCamelCase__,		normalizer_state.pop("""type"""		)		)
            A :       Union[str, Any]          =   do_lower_case
            A :       List[Any]          =   strip_accents
            A :       Optional[int]          =   tokenize_chinese_chars
            A :       Dict          =   normalizer_class(**lowerCamelCase__		)

        A :       List[Any]          =   do_lower_case






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__,		lowerCamelCase__=None		):
        A :       Union[str, Any]          =   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
            output += token_ids_a + [self.sep_token_id]

        return output






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__,		lowerCamelCase__ = None		):
        A :       Any          =   [self.sep_token_id]
        A :       Any          =   [self.cls_token_id]
        if token_ids_a is None:
            return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0]
        return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]






    def 			_lowerCAmelCase					(       self,		lowerCamelCase__,		lowerCamelCase__ = None		):
        A :       int          =   self._tokenizer.model.save(lowerCamelCase__,		name=lowerCamelCase__		)
        return tuple(lowerCamelCase__		)


