




import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing





class     SCREAMING_SNAKE_CASE__		(_a			):
     def __init__(						self	:		Optional[int]       ,						__lowerCamelCase	:		str = "▁"       ,						__lowerCamelCase	:		bool = True       ,						__lowerCamelCase	:		Union[str, AddedToken] = "<unk>"       ,						__lowerCamelCase	:		Union[str, AddedToken] = "</s>"       ,						__lowerCamelCase	:		Union[str, AddedToken] = "<pad>"       ,						):



       """simple docstring"""
       lowerCAmelCase__										=      {
           '''pad''': {'''id''': 0, '''token''': pad_token},
           '''eos''': {'''id''': 1, '''token''': eos_token},
           '''unk''': {'''id''': 2, '''token''': unk_token},
       }

       lowerCAmelCase__										=      [None] * len(self.special_tokens						)
       for token_dict in self.special_tokens.values():
         lowerCAmelCase__										=      token_dict['''token''']

       lowerCAmelCase__										=      Tokenizer(Unigram()						)

       lowerCAmelCase__										=      normalizers.Sequence(
           [
               normalizers.Nmt(),
               normalizers.NFKC(),
               normalizers.Replace(Regex(''' {2,}'''						)       ,						''' '''						),
               normalizers.Lowercase(),
           ]						)
       lowerCAmelCase__										=      pre_tokenizers.Sequence(
           [
               pre_tokenizers.Metaspace(replacement=__lowerCamelCase       ,						add_prefix_space=__lowerCamelCase						),
               pre_tokenizers.Digits(individual_digits=__lowerCamelCase						),
               pre_tokenizers.Punctuation(),
           ]						)
       lowerCAmelCase__										=      decoders.Metaspace(replacement=__lowerCamelCase       ,						add_prefix_space=__lowerCamelCase						)

       lowerCAmelCase__										=      TemplateProcessing(
           single=F"""$A {self.special_tokens['eos']['token']}"""       ,						special_tokens=[(self.special_tokens['''eos''']['''token'''], self.special_tokens['''eos''']['''id'''])]       ,						)

       lowerCAmelCase__										=      {
           '''model''': '''SentencePieceUnigram''',
           '''replacement''': replacement,
           '''add_prefix_space''': add_prefix_space,
       }

       super().__init__(__lowerCamelCase       ,						__lowerCamelCase						)
     def        A__      (						self	:		Tuple       ,						__lowerCamelCase	:		Union[str, List[str]]       ,						__lowerCamelCase	:		int = 80_00       ,						__lowerCamelCase	:		bool = True       ,						):



       """simple docstring"""
       lowerCAmelCase__										=      trainers.UnigramTrainer(
           vocab_size=__lowerCamelCase       ,						special_tokens=self.special_tokens_list       ,						show_progress=__lowerCamelCase       ,						)

       if isinstance(__lowerCamelCase       ,						__lowerCamelCase						):
         lowerCAmelCase__										=      [files]
       self._tokenizer.train(__lowerCamelCase       ,						trainer=__lowerCamelCase						)

       self.add_unk_id()
     def        A__      (						self	:		Optional[Any]       ,						__lowerCamelCase	:		Union[Iterator[str], Iterator[Iterator[str]]]       ,						__lowerCamelCase	:		int = 80_00       ,						__lowerCamelCase	:		bool = True       ,						):



       """simple docstring"""
       lowerCAmelCase__										=      trainers.UnigramTrainer(
           vocab_size=__lowerCamelCase       ,						special_tokens=self.special_tokens_list       ,						show_progress=__lowerCamelCase       ,						)

       self._tokenizer.train_from_iterator(__lowerCamelCase       ,						trainer=__lowerCamelCase						)

       self.add_unk_id()



     def        A__      (						self	:		List[Any]						):



       """simple docstring"""
       lowerCAmelCase__										=      json.loads(self._tokenizer.to_str()						)

       lowerCAmelCase__										=      self.special_tokens['''unk''']['''id''']

       lowerCAmelCase__										=      Tokenizer.from_str(json.dumps(__lowerCamelCase						)						)







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__magic_name__  :  Union[str, Any]					=       {
    """configuration_altclip""": [
        """ALTCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """AltCLIPConfig""",
        """AltCLIPTextConfig""",
        """AltCLIPVisionConfig""",
    ],
    """processing_altclip""": ["""AltCLIPProcessor"""],
}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __magic_name__  :  str					=       [
          """ALTCLIP_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """AltCLIPPreTrainedModel""",
          """AltCLIPModel""",
          """AltCLIPTextModel""",
          """AltCLIPVisionModel""",
      ]


if TYPE_CHECKING:
      from .configuration_altclip import (
          ALTCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
          AltCLIPConfig,
          AltCLIPTextConfig,
          AltCLIPVisionConfig,
      )
      from .processing_altclip import AltCLIPProcessor

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_altclip import (
                ALTCLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
                AltCLIPModel,
                AltCLIPPreTrainedModel,
                AltCLIPTextModel,
                AltCLIPVisionModel,
            )


else:
      import sys

      __magic_name__  :  str					=       _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

