





"""simple docstring"""




import re

from flax.core.frozen_dict import freeze
from flax.traverse_util import flatten_dict, unflatten_dict
from jax.experimental import PartitionSpec as P


# Sentinels
a						: int     	=			object()

# For specifying empty leaf dict `{}`
a						: Dict     	=			object()






def       _SCREAMING_SNAKE_CASE		(	_lowercase						:    List[str]    ,       _lowercase						:    Tuple					) ->int:






	'''simple docstring'''






	a      :		str	=				tuple((re.compile(x + "$"					) for x in qs)					)
	for i in range(len(_lowercase					) - len(_lowercase					) + 1					):
		a      :		int	=				[x.match(_lowercase					) for x, y in zip(_lowercase    ,       ks[i:]					)]
		if matches and all(_lowercase					):
			return True
	return False






def       _SCREAMING_SNAKE_CASE		(	_lowercase						:    Dict					) ->Union[str, Any]:






	'''simple docstring'''






	def replace(_lowercase						:    Tuple    ,       _lowercase						:    Optional[Any]					):
		for rule, replacement in rules:
			if _match(_lowercase    ,       _lowercase					):
				return replacement
		return val

	return replace






def       _SCREAMING_SNAKE_CASE		(	) ->Optional[Any]:






	'''simple docstring'''






	return [
	    # embeddings
	    (("transformer", "wpe", "embedding"), P("mp"    ,       _lowercase					)),
	    (("transformer", "wte", "embedding"), P("mp"    ,       _lowercase					)),
	    # atention
	    (("attention", "(q_proj|k_proj|v_proj)", "kernel"), P(_lowercase    ,       "mp"					)),
	    (("attention", "out_proj", "kernel"), P("mp"    ,       _lowercase					)),
	    (("attention", "out_proj", "bias"), None),
	    # mlp
	    (("mlp", "c_fc", "kernel"), P(_lowercase    ,       "mp"					)),
	    (("mlp", "c_fc", "bias"), P("mp"					)),
	    (("mlp", "c_proj", "kernel"), P("mp"    ,       _lowercase					)),
	    (("mlp", "c_proj", "bias"), None),
	    # layer norms
	    ((r"ln_\d+", "bias"), None),
	    ((r"\d+", r"ln_\d+", "scale"), None),
	    (("ln_f", "bias"), None),
	    (("ln_f", "scale"), None),
	]







def       _SCREAMING_SNAKE_CASE		(	_lowercase						:    Any					) ->Dict:






	'''simple docstring'''






	a      :		Tuple	=				_get_partition_rules()
	a      :		int	=				_replacement_rules(_lowercase					)
	a      :		str	=				{k: _unmatched for k in flatten_dict(_lowercase					)}
	a      :		List[Any]	=				{k: replace(_lowercase    ,       _lowercase					) for k, v in initd.items()}
	assert _unmatched not in result.values(), "Incomplete partition spec."
	return freeze(unflatten_dict(_lowercase					)					)









"""simple docstring"""




from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_herbert import HerbertTokenizer


a						: List[str]     	=			logging.get_logger(__name__)

a						: List[Any]     	=			{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

a						: str     	=			{
    '''vocab_file''': {
        '''allegro/herbert-base-cased''': '''https://huggingface.co/allegro/herbert-base-cased/resolve/main/vocab.json'''
    },
    '''merges_file''': {
        '''allegro/herbert-base-cased''': '''https://huggingface.co/allegro/herbert-base-cased/resolve/main/merges.txt'''
    },
}

a						: Tuple     	=			{'''allegro/herbert-base-cased''': 514}
a						: Optional[int]     	=			{}


class        __UpperCamelCase				(						a__						):
							lowerCamelCase   :					str      					=VOCAB_FILES_NAMES
							lowerCamelCase   :					Dict      					=PRETRAINED_VOCAB_FILES_MAP
							lowerCamelCase   :					Dict      					=PRETRAINED_INIT_CONFIGURATION
							lowerCamelCase   :					int      					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowerCamelCase   :					Optional[Any]      					=HerbertTokenizer






							def __init__(   self      ,					lowerCAmelCase__=None      ,					lowerCAmelCase__=None      ,					lowerCAmelCase__=None      ,					lowerCAmelCase__="<s>"      ,					lowerCAmelCase__="<unk>"      ,					lowerCAmelCase__="<pad>"      ,					lowerCAmelCase__="<mask>"      ,					lowerCAmelCase__="</s>"      ,					**lowerCAmelCase__      ,					)	-> Optional[int]:
								super().__init__(
								    lowerCAmelCase__      ,					lowerCAmelCase__      ,					tokenizer_file=lowerCAmelCase__      ,					cls_token=lowerCAmelCase__      ,					unk_token=lowerCAmelCase__      ,					pad_token=lowerCAmelCase__      ,					mask_token=lowerCAmelCase__      ,					sep_token=lowerCAmelCase__      ,					**lowerCAmelCase__      ,					)






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__ = None      )	-> List[int]:
								a      :		Optional[Any]	=				[self.cls_token_id]
								a      :		Any	=				[self.sep_token_id]
								if token_ids_a is None:
									return cls + token_ids_a + sep

								return cls + token_ids_a + sep + token_ids_a + sep






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__ = None      ,					lowerCAmelCase__ = False      )	-> List[int]:
								if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=lowerCAmelCase__      ,					token_ids_a=lowerCAmelCase__      ,					already_has_special_tokens=lowerCAmelCase__      )

								if token_ids_a is None:
									return [1] + ([0] * len(lowerCAmelCase__      )) + [1]
								return [1] + ([0] * len(lowerCAmelCase__      )) + [1] + ([0] * len(lowerCAmelCase__      )) + [1]






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__ = None      )	-> List[int]:
								a      :		Dict	=				[self.sep_token_id]
								a      :		Tuple	=				[self.cls_token_id]

								if token_ids_a is None:
									return len(cls + token_ids_a + sep      ) * [0]
								return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]






							def     __a   (   self      ,					lowerCAmelCase__      ,					lowerCAmelCase__ = None      )	-> Tuple[str]:
								a      :		List[str]	=				self._tokenizer.model.save(lowerCAmelCase__      ,					name=lowerCAmelCase__      )
								return tuple(lowerCAmelCase__      )


