





from typing import Any



def 						A_							(	_lowerCAmelCase	)		->					list[Any]:
 if not input_list:
  return []
 UpperCamelCase      :   List[str]			      =  [input_list.count(_lowerCAmelCase	) for value in input_list]
 UpperCamelCase      :   Dict			      =  max(_lowerCAmelCase	)  # Gets the maximum count in the input list.
 # Gets values of modes
 return sorted({input_list[i] for i, value in enumerate(_lowerCAmelCase	) if value == y}	)


if __name__ == "__main__":
    import doctest

    doctest.testmod()







import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


__lowerCamelCase			:	Tuple					    =	logging.get_logger(__name__)


__lowerCamelCase			:	int					    =	{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt"""}

# See all LED models at https://huggingface.co/models?filter=LED
__lowerCamelCase			:	str					    =	{
    """vocab_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """allenai/led-base-16384""": """https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json""",
    },
}

__lowerCamelCase			:	List[str]					    =	{
    """allenai/led-base-16384""": 1_6384,
}



@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def 						A_							(	)		->					List[str]:
 UpperCamelCase      :   Optional[int]			      =  (
     list(range(ord("!"	)				,  ord("~"	) + 1	)	) + list(range(ord("¡"	)				,  ord("¬"	) + 1	)	) + list(range(ord("®"	)				,  ord("ÿ"	) + 1	)	)
 )
 UpperCamelCase      :   List[Any]			      =  bs[:]
 UpperCamelCase      :   List[Any]			      =  0
 for b in range(2**8	):
  if b not in bs:
   bs.append(_lowerCAmelCase	)
   cs.append(2**8 + n	)
   n += 1
 UpperCamelCase      :   Union[str, Any]			      =  [chr(_lowerCAmelCase	) for n in cs]
 return dict(zip(_lowerCAmelCase				,  _lowerCAmelCase	)	)



def 						A_							(	_lowerCAmelCase	)		->					Optional[Any]:
 UpperCamelCase      :   int			      =  set()
 UpperCamelCase      :   List[str]			      =  word[0]
 for char in word[1:]:
  pairs.add((prev_char, char)	)
  UpperCamelCase      :   List[str]			      =  char
 return pairs



class    A__  (			__snake_case     ):
      _UpperCAmelCase      :Dict		       =  VOCAB_FILES_NAMES
      _UpperCAmelCase      :List[Any]		       =  PRETRAINED_VOCAB_FILES_MAP
      _UpperCAmelCase      :List[Any]		       =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _UpperCAmelCase      :Union[str, Any]		       =  ['input_ids', 'attention_mask']





      def __init__(					self			,				A_			,				A_			,				A_="replace"			,				A_="<s>"			,				A_="</s>"			,				A_="</s>"			,				A_="<s>"			,				A_="<unk>"			,				A_="<pad>"			,				A_="<mask>"			,				A_=False			,				**A_			,				):
       '''simple docstring'''




       UpperCamelCase      :   int			      =  AddedToken(A_			,				lstrip=A_			,				rstrip=A_  ) if isinstance(A_			,				A_  ) else bos_token
       UpperCamelCase      :   Tuple			      =  AddedToken(A_			,				lstrip=A_			,				rstrip=A_  ) if isinstance(A_			,				A_  ) else eos_token
       UpperCamelCase      :   Optional[Any]			      =  AddedToken(A_			,				lstrip=A_			,				rstrip=A_  ) if isinstance(A_			,				A_  ) else sep_token
       UpperCamelCase      :   Union[str, Any]			      =  AddedToken(A_			,				lstrip=A_			,				rstrip=A_  ) if isinstance(A_			,				A_  ) else cls_token
       UpperCamelCase      :   Dict			      =  AddedToken(A_			,				lstrip=A_			,				rstrip=A_  ) if isinstance(A_			,				A_  ) else unk_token
       UpperCamelCase      :   int			      =  AddedToken(A_			,				lstrip=A_			,				rstrip=A_  ) if isinstance(A_			,				A_  ) else pad_token

       # Mask token behave like a normal word, i.e. include the space before it
       UpperCamelCase      :   Optional[Any]			      =  AddedToken(A_			,				lstrip=A_			,				rstrip=A_  ) if isinstance(A_			,				A_  ) else mask_token

       super().__init__(
           errors=A_			,				bos_token=A_			,				eos_token=A_			,				unk_token=A_			,				sep_token=A_			,				cls_token=A_			,				pad_token=A_			,				mask_token=A_			,				add_prefix_space=A_			,				**A_			,				)

       with open(A_			,				encoding="utf-8"  ) as vocab_handle:
        UpperCamelCase      :   int			      =  json.load(A_  )
       UpperCamelCase      :   Dict			      =  {v: k for k, v in self.encoder.items()}
       UpperCamelCase      :   List[str]			      =  errors  # how to handle errors in decoding
       UpperCamelCase      :   Optional[Any]			      =  bytes_to_unicode()
       UpperCamelCase      :   Tuple			      =  {v: k for k, v in self.byte_encoder.items()}
       with open(A_			,				encoding="utf-8"  ) as merges_handle:
        UpperCamelCase      :   str			      =  merges_handle.read().split("\n"  )[1:-1]
       UpperCamelCase      :   Optional[Any]			      =  [tuple(merge.split()  ) for merge in bpe_merges]
       UpperCamelCase      :   Any			      =  dict(zip(A_			,				range(len(A_  )  )  )  )
       UpperCamelCase      :   List[str]			      =  {}
       UpperCamelCase      :   Any			      =  add_prefix_space

       # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
       UpperCamelCase      :   Any			      =  re.compile(R"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"  )





      @property
      # Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
      def 			__UpperCamelCase(					self  ):
       '''simple docstring'''




       return len(self.encoder  )





      def 			__UpperCamelCase(					self  ):
       '''simple docstring'''




       return dict(self.encoder			,				**self.added_tokens_encoder  )





      def 			__UpperCamelCase(					self			,				A_  ):
       '''simple docstring'''




       if token in self.cache:
        return self.cache[token]
       UpperCamelCase      :   Union[str, Any]			      =  tuple(A_  )
       UpperCamelCase      :   str			      =  get_pairs(A_  )

       if not pairs:
        return token

       while True:
        UpperCamelCase      :   Any			      =  min(A_			,				key=lambda A_  : self.bpe_ranks.get(A_			,				float("inf"  )  )  )
        if bigram not in self.bpe_ranks:
         break
        UpperCamelCase      ,       UpperCamelCase      :   Union[str, Any]			      =  bigram
        UpperCamelCase      :   List[Any]			      =  []
        UpperCamelCase      :   Union[str, Any]			      =  0
        while i < len(A_  ):
         try:
          UpperCamelCase      :   Union[str, Any]			      =  word.index(A_			,				A_  )
         except ValueError:
          new_word.extend(word[i:]  )
          break
         else:
          new_word.extend(word[i:j]  )
          UpperCamelCase      :   Union[str, Any]			      =  j

         if word[i] == first and i < len(A_  ) - 1 and word[i + 1] == second:
          new_word.append(first + second  )
          i += 2
         else:
          new_word.append(word[i]  )
          i += 1
        UpperCamelCase      :   Tuple			      =  tuple(A_  )
        UpperCamelCase      :   Union[str, Any]			      =  new_word
        if len(A_  ) == 1:
         break
        else:
         UpperCamelCase      :   int			      =  get_pairs(A_  )
       UpperCamelCase      :   Union[str, Any]			      =  " ".join(A_  )
       UpperCamelCase      :   Any			      =  word
       return word





      def 			__UpperCamelCase(					self			,				A_  ):
       '''simple docstring'''




       UpperCamelCase      :   Union[str, Any]			      =  []
       for token in re.findall(self.pat			,				A_  ):
        UpperCamelCase      :   Optional[Any]			      =  "".join(
            self.byte_encoder[b] for b in token.encode("utf-8"  )  )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
        bpe_tokens.extend(bpe_token for bpe_token in self.bpe(A_  ).split(" "  )  )
       return bpe_tokens





      def 			__UpperCamelCase(					self			,				A_  ):
       '''simple docstring'''




       return self.encoder.get(A_			,				self.encoder.get(self.unk_token  )  )





      def 			__UpperCamelCase(					self			,				A_  ):
       '''simple docstring'''




       return self.decoder.get(A_  )





      def 			__UpperCamelCase(					self			,				A_  ):
       '''simple docstring'''




       UpperCamelCase      :   List[Any]			      =  "".join(A_  )
       UpperCamelCase      :   List[Any]			      =  bytearray([self.byte_decoder[c] for c in text]  ).decode("utf-8"			,				errors=self.errors  )
       return text





      def 			__UpperCamelCase(					self			,				A_			,				A_ = None  ):
       '''simple docstring'''




       if not os.path.isdir(A_  ):
        logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""  )
        return
       UpperCamelCase      :   List[str]			      =  os.path.join(
           A_			,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )
       UpperCamelCase      :   Any			      =  os.path.join(
           A_			,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]  )

       with open(A_			,				"w"			,				encoding="utf-8"  ) as f:
        f.write(json.dumps(self.encoder			,				indent=2			,				sort_keys=A_			,				ensure_ascii=A_  ) + "\n"  )

       UpperCamelCase      :   str			      =  0
       with open(A_			,				"w"			,				encoding="utf-8"  ) as writer:
        writer.write("#version: 0.2\n"  )
        for bpe_tokens, token_index in sorted(self.bpe_ranks.items()			,				key=lambda A_  : kv[1]  ):
         if index != token_index:
          logger.warning(
              F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
              " Please check that the tokenizer is not corrupted!"  )
          UpperCamelCase      :   Any			      =  token_index
         writer.write(" ".join(A_  ) + "\n"  )
         index += 1

       return vocab_file, merge_file





      def 			__UpperCamelCase(					self			,				A_			,				A_ = None  ):
       '''simple docstring'''




       if token_ids_a is None:
        return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       UpperCamelCase      :   List[str]			      =  [self.cls_token_id]
       UpperCamelCase      :   Optional[Any]			      =  [self.sep_token_id]
       return cls + token_ids_a + sep + sep + token_ids_a + sep





      def 			__UpperCamelCase(					self			,				A_			,				A_ = None			,				A_ = False  ):
       '''simple docstring'''




       if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=A_			,				token_ids_a=A_			,				already_has_special_tokens=A_  )

       if token_ids_a is None:
        return [1] + ([0] * len(A_  )) + [1]
       return [1] + ([0] * len(A_  )) + [1, 1] + ([0] * len(A_  )) + [1]





      def 			__UpperCamelCase(					self			,				A_			,				A_ = None  ):
       '''simple docstring'''




       UpperCamelCase      :   Any			      =  [self.sep_token_id]
       UpperCamelCase      :   Any			      =  [self.cls_token_id]

       if token_ids_a is None:
        return len(cls + token_ids_a + sep  ) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep  ) * [0]





      def 			__UpperCamelCase(					self			,				A_			,				A_=False			,				**A_  ):
       '''simple docstring'''




       UpperCamelCase      :   Tuple			      =  kwargs.pop("add_prefix_space"			,				self.add_prefix_space  )
       if (is_split_into_words or add_prefix_space) and (len(A_  ) > 0 and not text[0].isspace()):
        UpperCamelCase      :   List[str]			      =  " " + text
       return (text, kwargs)





      def 			__UpperCamelCase(					self			,				A_			,				A_ = None			,				A_ = PaddingStrategy.DO_NOT_PAD			,				A_ = None			,				A_ = None			,				):
       '''simple docstring'''




       UpperCamelCase      :   Optional[Any]			      =  super()._pad(
           encoded_inputs=A_			,				max_length=A_			,				padding_strategy=A_			,				pad_to_multiple_of=A_			,				return_attention_mask=A_			,				)

       # Load from model defaults
       if return_attention_mask is None:
        UpperCamelCase      :   Union[str, Any]			      =  "attention_mask" in self.model_input_names

       if return_attention_mask and "global_attention_mask" in encoded_inputs:
        UpperCamelCase      :   int			      =  encoded_inputs[self.model_input_names[0]]
        # `global_attention_mask` need to have the same length as other (sequential) inputs.
        UpperCamelCase      :   str			      =  len(encoded_inputs["global_attention_mask"]  ) != len(A_  )

        if needs_to_be_padded:
         UpperCamelCase      :   Optional[int]			      =  len(A_  ) - len(encoded_inputs["global_attention_mask"]  )

         if self.padding_side == "right":
          # Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
          UpperCamelCase      :   Union[str, Any]			      =  (
              encoded_inputs["global_attention_mask"] + [-1] * difference
          )
         elif self.padding_side == "left":
          UpperCamelCase      :   Tuple			      =  [-1] * difference + encoded_inputs[
              "global_attention_mask"
          ]
         else:
          raise ValueError("Invalid padding strategy:" + str(self.padding_side  )  )

       return encoded_inputs
