





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


a__:      str									=		logging.get_logger(__name__)

a__:      int									=		{'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

a__:      Dict									=		[
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

a__:      Optional[Any]									=		{
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
a__:      Optional[int]									=		{F"funnel-transformer/{name}": 512 for name in _model_names}
a__:      List[str]									=		{F"funnel-transformer/{name}": {'''do_lower_case''': True} for name in _model_names}




class     SCREAMING_SNAKE_CASE__ (						a__      ):
 __SCREAMING_SNAKE_CASE							       =	VOCAB_FILES_NAMES
 __SCREAMING_SNAKE_CASE							       =	PRETRAINED_VOCAB_FILES_MAP
 __SCREAMING_SNAKE_CASE							       =	PRETRAINED_INIT_CONFIGURATION
 __SCREAMING_SNAKE_CASE							       =	FunnelTokenizer
 __SCREAMING_SNAKE_CASE							       =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __SCREAMING_SNAKE_CASE							       =	2



 def __init__(							self,__lowerCamelCase=None,__lowerCamelCase=None,__lowerCamelCase=True,__lowerCamelCase="<unk>",__lowerCamelCase="<sep>",__lowerCamelCase="<pad>",__lowerCamelCase="<cls>",__lowerCamelCase="<mask>",__lowerCamelCase="<s>",__lowerCamelCase="</s>",__lowerCamelCase=True,__lowerCamelCase=True,__lowerCamelCase=None,__lowerCamelCase="##",**__lowerCamelCase,):
        super().__init__(
            lowerCAmelCase__,tokenizer_file=lowerCAmelCase__,do_lower_case=lowerCAmelCase__,unk_token=lowerCAmelCase__,sep_token=lowerCAmelCase__,pad_token=lowerCAmelCase__,cls_token=lowerCAmelCase__,mask_token=lowerCAmelCase__,bos_token=lowerCAmelCase__,eos_token=lowerCAmelCase__,clean_text=lowerCAmelCase__,tokenize_chinese_chars=lowerCAmelCase__,strip_accents=lowerCAmelCase__,wordpieces_prefix=lowerCAmelCase__,**lowerCAmelCase__,)

        A__											=						json.loads(self.backend_tokenizer.normalizer.__getstate__() )
        if (
            normalizer_state.get('''lowercase''',lowerCAmelCase__ ) != do_lower_case
            or normalizer_state.get('''strip_accents''',lowerCAmelCase__ ) != strip_accents
            or normalizer_state.get('''handle_chinese_chars''',lowerCAmelCase__ ) != tokenize_chinese_chars
        ):
               A__											=						getattr(lowerCAmelCase__,normalizer_state.pop('''type''' ) )
               A__											=						do_lower_case
               A__											=						strip_accents
               A__											=						tokenize_chinese_chars
               A__											=						normalizer_class(**lowerCAmelCase__ )

        A__											=						do_lower_case



 def 				UpperCamelCase			(							self,__lowerCamelCase,__lowerCamelCase=None ):
        A__											=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

        if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

        return output



 def 				UpperCamelCase			(							self,__lowerCamelCase,__lowerCamelCase = None ):
        A__											=						[self.sep_token_id]
        A__											=						[self.cls_token_id]
        if token_ids_a is None:
               return len(cls ) * [self.cls_token_type_id] + len(token_ids_a + sep ) * [0]
        return len(cls ) * [self.cls_token_type_id] + len(token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]




 def 				UpperCamelCase			(							self,__lowerCamelCase,__lowerCamelCase = None ):
        A__											=						self._tokenizer.model.save(lowerCAmelCase__,name=lowerCAmelCase__ )
        return tuple(lowerCAmelCase__ )












"""simple docstring"""




from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


a						: List[str]     	=			logging.get_logger(__name__)

a						: List[Any]     	=			{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

a						: str     	=			{
    '''vocab_file''': {
        '''allegro/herbert-base-cased''': '''https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json'''
    },
    '''merges_file''': {
        '''allegro/herbert-base-cased''': '''https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt'''
    },
}

a						: Tuple     	=			{'''allegro/herbert-base-cased''': 514}
a						: Optional[int]     	=			{}


class        __UpperCamelCase				(						a__						):
							lowerCamelCase   :					str      					=VOCAB_FILES_NAMES
							lowerCamelCase   :					Dict      					=PRETRAINED_VOCAB_FILES_MAP
							lowerCamelCase   :					Dict      					=PRETRAINED_INIT_CONFIGURATION
							lowerCamelCase   :					int      					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCamelCase   :					Optional[Any]      					=HerbertTokenizer






							def __init__(   self      ,					lowerCAmelCase__=None      ,					lowerCAmelCase__=None      ,					lowerCAmelCase__=None      ,					lowerCAmelCase__="<s>"      ,					lowerCAmelCase__="<unk>"      ,					lowerCAmelCase__="<pad>"      ,					lowerCAmelCase__="<mask>"      ,					lowerCAmelCase__="</s>"      ,					**lowerCAmelCase__      ,					)	-> Optional[int]:
								super().__init__(
								    lowerCAmelCase__      ,					lowerCAmelCase__      ,					tokenizer_file=lowerCAmelCase__      ,					cls_token=lowerCAmelCase__      ,					unk_token=lowerCAmelCase__      ,					pad_token=lowerCAmelCase__      ,					mask_token=lowerCAmelCase__      ,					sep_token=lowerCAmelCase__      ,					**lowerCAmelCase__      ,					)






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__ = None      )	-> List[int]:
								a      :		Optional[Any]	=				[self.cls_token_id]
								a      :		Any	=				[self.sep_token_id]
								if token_ids_a is None:
									return cls + token_ids_a + sep

								return cls + token_ids_a + sep + token_ids_a + sep






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__ = None      ,					lowerCAmelCase__ = False      )	-> List[int]:
								if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=lowerCAmelCase__      ,					token_ids_a=lowerCAmelCase__      ,					already_has_special_tokens=lowerCAmelCase__      )

								if token_ids_a is None:
									return [1] + ([0] * len(lowerCAmelCase__      )) + [1]
								return [1] + ([0] * len(lowerCAmelCase__      )) + [1] + ([0] * len(lowerCAmelCase__      )) + [1]






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__ = None      )	-> List[int]:
								a      :		Dict	=				[self.sep_token_id]
								a      :		Tuple	=				[self.cls_token_id]

								if token_ids_a is None:
									return len(cls + token_ids_a + sep      ) * [0]
								return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__ = None      )	-> Tuple[str]:
								a      :		List[str]	=				self._tokenizer.model.save(lowerCAmelCase__      ,					name=lowerCAmelCase__      )
								return tuple(lowerCAmelCase__      )


