




'''simple docstring'''


import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing





class  __lowercase    (	_a			):







   def __init__(						self   ,				UpperCamelCase = "▁"   ,				UpperCamelCase = True   ,				UpperCamelCase = "<unk>"   ,				UpperCamelCase = "</s>"   ,				UpperCamelCase = "<pad>"   ,				)					->     List[str]:
     __a								=       {
         'pad': {'id': 0, 'token': pad_token},
         'eos': {'id': 1, 'token': eos_token},
         'unk': {'id': 2, 'token': unk_token},
     }

     __a								=       [None] * len(self.special_tokens   )
     for token_dict in self.special_tokens.values():
       __a								=       token_dict['token']

     __a								=       Tokenizer(Unigram()   )

     __a								=       normalizers.Sequence(
         [
             normalizers.Nmt(),
             normalizers.NFKC(),
             normalizers.Replace(Regex(' {2,}'   )   ,				' '   ),
             normalizers.Lowercase(),
         ]   )
     __a								=       pre_tokenizers.Sequence(
         [
             pre_tokenizers.Metaspace(replacement=__lowerCamelCase   ,				add_prefix_space=__lowerCamelCase   ),
             pre_tokenizers.Digits(individual_digits=__lowerCamelCase   ),
             pre_tokenizers.Punctuation(),
         ]   )
     __a								=       decoders.Metaspace(replacement=__lowerCamelCase   ,				add_prefix_space=__lowerCamelCase   )

     __a								=       TemplateProcessing(
         single=f"$A {self.special_tokens['eos']['token']}"   ,				special_tokens=[(self.special_tokens['eos']['token'], self.special_tokens['eos']['id'])]   ,				)

     __a								=       {
         'model': 'SentencePieceUnigram',
         'replacement': replacement,
         'add_prefix_space': add_prefix_space,
     }

     super().__init__(__lowerCamelCase   ,				__lowerCamelCase   )







   def 		UpperCamelCase__      (						self   ,				UpperCamelCase   ,				UpperCamelCase = 8000   ,				UpperCamelCase = True   ,				)					->     Optional[int]:
     __a								=       trainers.UnigramTrainer(
         vocab_size=__lowerCamelCase   ,				special_tokens=self.special_tokens_list   ,				show_progress=__lowerCamelCase   ,				)

     if isinstance(__lowerCamelCase   ,				__lowerCamelCase   ):
       __a								=       [files]
     self._tokenizer.train(__lowerCamelCase   ,				trainer=__lowerCamelCase   )

     self.add_unk_id()







   def 		UpperCamelCase__      (						self   ,				UpperCamelCase   ,				UpperCamelCase = 8000   ,				UpperCamelCase = True   ,				)					->     Tuple:
     __a								=       trainers.UnigramTrainer(
         vocab_size=__lowerCamelCase   ,				special_tokens=self.special_tokens_list   ,				show_progress=__lowerCamelCase   ,				)

     self._tokenizer.train_from_iterator(__lowerCamelCase   ,				trainer=__lowerCamelCase   )

     self.add_unk_id()







   def 		UpperCamelCase__      (						self   )					->     str:
     __a								=       json.loads(self._tokenizer.to_str()   )

     __a								=       self.special_tokens['unk']['id']

     __a								=       Tokenizer.from_str(json.dumps(__lowerCamelCase   )   )









from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__magic_name__  :  Union[str, Any]					=       {
    """configuration_altclip""": [
        """ALTCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """AltCLIPConfig""",
        """AltCLIPTextConfig""",
        """AltCLIPVisionConfig""",
    ],
    """processing_altclip""": ["""AltCLIPProcessor"""],
}

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      __magic_name__  :  str					=       [
          """ALTCLIP_PRETRAINED_MODEL_ARCHIVE_LIST""",
          """AltCLIPPreTrainedModel""",
          """AltCLIPModel""",
          """AltCLIPTextModel""",
          """AltCLIPVisionModel""",
      ]


if TYPE_CHECKING:
      from .configuration_altclip import (
          ALTCLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
          AltCLIPConfig,
          AltCLIPTextConfig,
          AltCLIPVisionConfig,
      )
      from .processing_altclip import AltCLIPProcessor

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_altclip import (
                ALTCLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
                AltCLIPModel,
                AltCLIPPreTrainedModel,
                AltCLIPTextModel,
                AltCLIPVisionModel,
            )


else:
      import sys

      __magic_name__  :  str					=       _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

