'''simple docstring'''






from collections.abc import Sequence
from queue import Queue







class lowercase_       :







   """simple docstring"""

   def __init__(      self      :			Tuple, UpperCamelCase__      :			Any, UpperCamelCase__      :			List[Any], UpperCamelCase__      :			Optional[int], UpperCamelCase__      :			Optional[int]=None, UpperCamelCase__      :			Tuple=None     )  ->			Any:
        _A    		=  start
        _A    		=  end
        _A    		=  val
        _A    		=  (start + end) // 2
        _A    		=  left
        _A    		=  right

   def __repr__(      self      :			str     )  ->			Optional[int]:
        return f'SegmentTreeNode(start={self.start}, end={self.end}, val={self.val})'







class lowercase_       :







   """simple docstring"""

   def __init__(      self      :			Dict, UpperCamelCase__      :			Sequence, UpperCamelCase__      :			Any     )  ->			Optional[Any]:
        _A    		=  collection
        _A    		=  function
        if self.collection:
             _A    		=  self._build_tree(0, len(UpperCamelCase__     ) - 1     )
   def 	__UpperCAmelCase   (      self      :			Any, UpperCamelCase__      :			str, UpperCamelCase__      :			Dict     )  ->			int:
        self._update_tree(self.root, UpperCamelCase__, UpperCamelCase__     )
   def 	__UpperCAmelCase   (      self      :			Union[str, Any], UpperCamelCase__      :			List[Any], UpperCamelCase__      :			Optional[Any]     )  ->			Union[str, Any]:
        return self._query_range(self.root, UpperCamelCase__, UpperCamelCase__     )
   def 	__UpperCAmelCase   (      self      :			Optional[Any], UpperCamelCase__      :			Any, UpperCamelCase__      :			List[Any]     )  ->			str:
        if start == end:
             return SegmentTreeNode(UpperCamelCase__, UpperCamelCase__, self.collection[start]     )
        _A    		=  (start + end) // 2
        _A    		=  self._build_tree(UpperCamelCase__, UpperCamelCase__     )
        _A    		=  self._build_tree(mid + 1, UpperCamelCase__     )
        return SegmentTreeNode(UpperCamelCase__, UpperCamelCase__, self.fn(left.val, right.val     ), UpperCamelCase__, UpperCamelCase__     )
   def 	__UpperCAmelCase   (      self      :			List[Any], UpperCamelCase__      :			Any, UpperCamelCase__      :			Union[str, Any], UpperCamelCase__      :			Optional[int]     )  ->			Tuple:
        if node.start == i and node.end == i:
             _A    		=  val
             return
        if i <= node.mid:
             self._update_tree(node.left, UpperCamelCase__, UpperCamelCase__     )
        else:
             self._update_tree(node.right, UpperCamelCase__, UpperCamelCase__     )
        _A    		=  self.fn(node.left.val, node.right.val     )
   def 	__UpperCAmelCase   (      self      :			Tuple, UpperCamelCase__      :			List[str], UpperCamelCase__      :			Tuple, UpperCamelCase__      :			int     )  ->			Tuple:
        if node.start == i and node.end == j:
             return node.val

        if i <= node.mid:
             if j <= node.mid:
                  # range in left child tree
                  return self._query_range(node.left, UpperCamelCase__, UpperCamelCase__     )
             else:
                  # range in left child tree and right child tree
                  return self.fn(
                      self._query_range(node.left, UpperCamelCase__, node.mid     ), self._query_range(node.right, node.mid + 1, UpperCamelCase__     ), )
        else:
             # range in right child tree
             return self._query_range(node.right, UpperCamelCase__, UpperCamelCase__     )

   def 	__UpperCAmelCase   (      self      :			List[Any]     )  ->			List[Any]:
        if self.root is not None:
             _A    		=  Queue()
             queue.put(self.root     )
             while not queue.empty():
                  _A    		=  queue.get()
                  yield node

                  if node.left is not None:
                       queue.put(node.left     )

                  if node.right is not None:
                       queue.put(node.right     )


if __name__ == "__main__":
   import operator

   for fn in [operator.add, max, min]:
      print('''*''' * 50)
      _UpperCAmelCase    :     Tuple         =       SegmentTree([2, 1, 5, 3, 4], fn)
      for node in arr.traverse():
         print(node)
      print()

      arr.update(1, 5)
      for node in arr.traverse():
         print(node)
      print()

      print(arr.query_range(3, 4))  # 7
      print(arr.query_range(2, 2))  # 5
      print(arr.query_range(1, 3))  # 13
      print()






"""simple docstring"""


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_funnel import FunnelTokenizer


UpperCAmelCase													=logging.get_logger(__name__)

UpperCAmelCase													={"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase													=[
    "small",
    "small-base",
    "medium",
    "medium-base",
    "intermediate",
    "intermediate-base",
    "large",
    "large-base",
    "xlarge",
    "xlarge-base",
]

UpperCAmelCase													={
    "vocab_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/vocab.txt",
        "funnel-transformer/small-base": "https://huggingface.co/funnel-transformer/small-base/resolve/main/vocab.txt",
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/vocab.txt",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/vocab.txt"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/vocab.txt"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/vocab.txt",
        "funnel-transformer/large-base": "https://huggingface.co/funnel-transformer/large-base/resolve/main/vocab.txt",
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/vocab.txt",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "funnel-transformer/small": "https://huggingface.co/funnel-transformer/small/resolve/main/tokenizer.json",
        "funnel-transformer/small-base": (
            "https://huggingface.co/funnel-transformer/small-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/medium": "https://huggingface.co/funnel-transformer/medium/resolve/main/tokenizer.json",
        "funnel-transformer/medium-base": (
            "https://huggingface.co/funnel-transformer/medium-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate": (
            "https://huggingface.co/funnel-transformer/intermediate/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/intermediate-base": (
            "https://huggingface.co/funnel-transformer/intermediate-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/large": "https://huggingface.co/funnel-transformer/large/resolve/main/tokenizer.json",
        "funnel-transformer/large-base": (
            "https://huggingface.co/funnel-transformer/large-base/resolve/main/tokenizer.json"
        ),
        "funnel-transformer/xlarge": "https://huggingface.co/funnel-transformer/xlarge/resolve/main/tokenizer.json",
        "funnel-transformer/xlarge-base": (
            "https://huggingface.co/funnel-transformer/xlarge-base/resolve/main/tokenizer.json"
        ),
    },
}
UpperCAmelCase													={f"""funnel-transformer/{name}""": 512 for name in _model_names}
UpperCAmelCase													={f"""funnel-transformer/{name}""": {"do_lower_case": True} for name in _model_names}





class lowerCamelCase__   (				SCREAMING_SNAKE_CASE    ):



    '''simple docstring'''




    _lowerCamelCase					=   VOCAB_FILES_NAMES
    _lowerCamelCase					=   PRETRAINED_VOCAB_FILES_MAP
    _lowerCamelCase					=   PRETRAINED_INIT_CONFIGURATION
    _lowerCamelCase					=   FunnelTokenizer
    _lowerCamelCase					=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    _lowerCamelCase					=   2




    def __init__( self ,lowerCamelCase_=None ,lowerCamelCase_=None ,lowerCamelCase_=True ,lowerCamelCase_="<unk>" ,lowerCamelCase_="<sep>" ,lowerCamelCase_="<pad>" ,lowerCamelCase_="<cls>" ,lowerCamelCase_="<mask>" ,lowerCamelCase_="<s>" ,lowerCamelCase_="</s>" ,lowerCamelCase_=True ,lowerCamelCase_=True ,lowerCamelCase_=None ,lowerCamelCase_="##" ,**lowerCamelCase_ ,)    ->      List[Any]:
          super().__init__(
              lowerCamelCase_ ,tokenizer_file=lowerCamelCase_ ,do_lower_case=lowerCamelCase_ ,unk_token=lowerCamelCase_ ,sep_token=lowerCamelCase_ ,pad_token=lowerCamelCase_ ,cls_token=lowerCamelCase_ ,mask_token=lowerCamelCase_ ,bos_token=lowerCamelCase_ ,eos_token=lowerCamelCase_ ,clean_text=lowerCamelCase_ ,tokenize_chinese_chars=lowerCamelCase_ ,strip_accents=lowerCamelCase_ ,wordpieces_prefix=lowerCamelCase_ ,**lowerCamelCase_ ,)

          A				  =		json.loads(self.backend_tokenizer.normalizer.__getstate__()    )
          if (
              normalizer_state.get("""lowercase""" ,lowerCamelCase_    ) != do_lower_case
              or normalizer_state.get("""strip_accents""" ,lowerCamelCase_    ) != strip_accents
              or normalizer_state.get("""handle_chinese_chars""" ,lowerCamelCase_    ) != tokenize_chinese_chars
          ):
                A				  =		getattr(lowerCamelCase_ ,normalizer_state.pop("""type"""    )    )
                A				  =		do_lower_case
                A				  =		strip_accents
                A				  =		tokenize_chinese_chars
                A				  =		normalizer_class(**lowerCamelCase_    )

          A				  =		do_lower_case




    def UpperCamelCase__		( self ,lowerCamelCase_ ,lowerCamelCase_=None    )    ->      List[Any]:
          A				  =		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
                output += token_ids_a + [self.sep_token_id]

          return output




    def UpperCamelCase__		( self ,lowerCamelCase_ ,lowerCamelCase_ = None    )    ->      List[int]:
          A				  =		[self.sep_token_id]
          A				  =		[self.cls_token_id]
          if token_ids_a is None:
                return len(cls    ) * [self.cls_token_type_id] + len(token_ids_a + sep    ) * [0]
          return len(cls    ) * [self.cls_token_type_id] + len(token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]




    def UpperCamelCase__		( self ,lowerCamelCase_ ,lowerCamelCase_ = None    )    ->      Tuple[str]:
          A				  =		self._tokenizer.model.save(lowerCamelCase_ ,name=lowerCamelCase_    )
          return tuple(lowerCamelCase_    )




