






"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


lowerCamelCase_							      = logging.get_logger(__name__)

lowerCamelCase_							      = {'''vocab_file''': '''vocab.txt''', '''tokenizer_file''': '''tokenizer.json'''}

lowerCamelCase_							      = [
    '''small''',
    '''small-base''',
    '''medium''',
    '''medium-base''',
    '''intermediate''',
    '''intermediate-base''',
    '''large''',
    '''large-base''',
    '''xlarge''',
    '''xlarge-base''',
]

lowerCamelCase_							      = {
    '''vocab_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt''',
        '''funnel-transformer/small-base''': '''https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt''',
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt''',
        '''funnel-transformer/large-base''': '''https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt'''
        ),
    },
    '''tokenizer_file''': {
        '''funnel-transformer/small''': '''https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json''',
        '''funnel-transformer/small-base''': (
            '''https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/medium''': '''https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json''',
        '''funnel-transformer/medium-base''': (
            '''https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate''': (
            '''https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/intermediate-base''': (
            '''https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/large''': '''https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json''',
        '''funnel-transformer/large-base''': (
            '''https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json'''
        ),
        '''funnel-transformer/xlarge''': '''https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json''',
        '''funnel-transformer/xlarge-base''': (
            '''https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json'''
        ),
    },
}
lowerCamelCase_							      = {f'funnel-transformer/{name}': 512 for name in _model_names}
lowerCamelCase_							      = {f'funnel-transformer/{name}': {'''do_lower_case''': True} for name in _model_names}







class      UpperCamelCase_			(__A	):
	__magic_name__       =							VOCAB_FILES_NAMES
	__magic_name__       =							PRETRAINED_VOCAB_FILES_MAP
	__magic_name__       =							PRETRAINED_INIT_CONFIGURATION
	__magic_name__       =							FunnelTokenizer
	__magic_name__       =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	__magic_name__       =							2





	def __init__( self   :					Union[str, Any]      ,      lowerCAmelCase_   :					Dict=None      ,      lowerCAmelCase_   :					Optional[int]=None      ,      lowerCAmelCase_   :					Dict=True      ,      lowerCAmelCase_   :					Any="<unk>"      ,      lowerCAmelCase_   :					Tuple="<sep>"      ,      lowerCAmelCase_   :					Tuple="<pad>"      ,      lowerCAmelCase_   :					Optional[Any]="<cls>"      ,      lowerCAmelCase_   :					Tuple="<mask>"      ,      lowerCAmelCase_   :					int="<s>"      ,      lowerCAmelCase_   :					Dict="</s>"      ,      lowerCAmelCase_   :					Union[str, Any]=True      ,      lowerCAmelCase_   :					Dict=True      ,      lowerCAmelCase_   :					int=None      ,      lowerCAmelCase_   :					Union[str, Any]="##"      ,      **lowerCAmelCase_   :					str      ,      )		->     int:
				super().__init__(
				    lowerCAmelCase_      ,      tokenizer_file=lowerCAmelCase_      ,      do_lower_case=lowerCAmelCase_      ,      unk_token=lowerCAmelCase_      ,      sep_token=lowerCAmelCase_      ,      pad_token=lowerCAmelCase_      ,      cls_token=lowerCAmelCase_      ,      mask_token=lowerCAmelCase_      ,      bos_token=lowerCAmelCase_      ,      eos_token=lowerCAmelCase_      ,      clean_text=lowerCAmelCase_      ,      tokenize_chinese_chars=lowerCAmelCase_      ,      strip_accents=lowerCAmelCase_      ,      wordpieces_prefix=lowerCAmelCase_      ,      **lowerCAmelCase_      ,      )

				UpperCAmelCase_	:					Tuple     	=   json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
				if (
				    normalizer_state.get("lowercase"      ,      lowerCAmelCase_		) != do_lower_case
				    or normalizer_state.get("strip_accents"      ,      lowerCAmelCase_		) != strip_accents
				    or normalizer_state.get("handle_chinese_chars"      ,      lowerCAmelCase_		) != tokenize_chinese_chars
				):
							UpperCAmelCase_	:					str     	=   getattr(lowerCAmelCase_      ,      normalizer_state.pop("type"		)		)
							UpperCAmelCase_	:					List[Any]     	=   do_lower_case
							UpperCAmelCase_	:					List[Any]     	=   strip_accents
							UpperCAmelCase_	:					Dict     	=   tokenize_chinese_chars
							UpperCAmelCase_	:					Optional[Any]     	=   normalizer_class(**lowerCAmelCase_		)

				UpperCAmelCase_	:					Optional[Any]     	=   do_lower_case





	def 	_SCREAMING_SNAKE_CASE ( self   :					Optional[Any]      ,      lowerCAmelCase_   :					Tuple      ,      lowerCAmelCase_   :					Optional[int]=None		)		->     Optional[Any]:
				UpperCAmelCase_	:					Dict     	=   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

				if token_ids_a:
							output += token_ids_a + [self.sep_token_id]

				return output





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[Any]      ,      lowerCAmelCase_   :					List[int]      ,      lowerCAmelCase_   :					Optional[List[int]] = None		)		->     List[int]:
				UpperCAmelCase_	:					Tuple     	=   [self.sep_token_id]
				UpperCAmelCase_	:					List[Any]     	=   [self.cls_token_id]
				if token_ids_a is None:
							return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0]
				return len(cls		) * [self.cls_token_type_id] + len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]





	def 	_SCREAMING_SNAKE_CASE ( self   :					Any      ,      lowerCAmelCase_   :					str      ,      lowerCAmelCase_   :					Optional[str] = None		)		->     Tuple[str]:
				UpperCAmelCase_	:					Union[str, Any]     	=   self._tokenizer.model.save(lowerCAmelCase_      ,      name=lowerCAmelCase_		)
				return tuple(lowerCAmelCase_		)










"""simple docstring"""




from __future__ import annotations

import matplotlib.pyplot as plt  # type: ignore
import numpy

# initial triangle of Koch snowflake
a		:					int	      =     numpy.array([0, 0])
a		:					Optional[Any]	      =     numpy.array([0.5, 0.866_0254])
a		:					Tuple	      =     numpy.array([1, 0])
a		:					List[str]	      =     [VECTOR_1, VECTOR_2, VECTOR_3, VECTOR_1]






def  _SCREAMING_SNAKE_CASE						(      _lowercase		:       list[numpy.ndarray]       ,			_lowercase		:       int   )					->list[numpy.ndarray]:






	'''simple docstring'''






	a :						List[str]      =							initial_vectors
	for _ in range(_lowercase   ):
		a :						Optional[Any]      =							iteration_step(_lowercase   )
	return vectors






def  _SCREAMING_SNAKE_CASE						(      _lowercase		:       list[numpy.ndarray]   )					->list[numpy.ndarray]:






	'''simple docstring'''






	a :						Union[str, Any]      =							[]
	for i, start_vector in enumerate(vectors[:-1]   ):
		a :						str      =							vectors[i + 1]
		new_vectors.append(_lowercase   )
		a :						Optional[int]      =							end_vector - start_vector
		new_vectors.append(start_vector + difference_vector / 3   )
		new_vectors.append(
		    start_vector + difference_vector / 3 + rotate(difference_vector / 3       ,			60   )   )
		new_vectors.append(start_vector + difference_vector * 2 / 3   )
	new_vectors.append(vectors[-1]   )
	return new_vectors






def  _SCREAMING_SNAKE_CASE						(      _lowercase		:       numpy.ndarray       ,			_lowercase		:       float   )					->numpy.ndarray:






	'''simple docstring'''






	a :						int      =							numpy.radians(_lowercase   )
	a,			a :						Optional[int]      =							numpy.cos(_lowercase   ), numpy.sin(_lowercase   )
	a :						Dict      =							numpy.array(((c, -s), (s, c))   )
	return numpy.dot(_lowercase       ,			_lowercase   )






def  _SCREAMING_SNAKE_CASE						(      _lowercase		:       list[numpy.ndarray]   )					->None:






	'''simple docstring'''






	a :						Dict      =							plt.gca()
	axes.set_aspect("equal"   )

	# matplotlib.pyplot.plot takes a list of all x-coordinates and a list of all
	# y-coordinates as inputs, which are constructed from the vector-list using
	# zip()
	a,			a :						Any      =							zip(*_lowercase   )
	plt.plot(_lowercase       ,			_lowercase   )
	plt.show()


if __name__ == "__main__":
		import doctest

		doctest.testmod()

		a		:					Optional[int]	      =     iterate(INITIAL_VECTORS, 5)
		plot(processed_vectors)


