


'''simple docstring'''







import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin







@require_jieba
class A__       (						UpperCAmelCase__ ,     unittest.TestCase      ):
      __UpperCamelCase       :  Dict	       =							CpmAntTokenizer
      __UpperCamelCase       :  Tuple	       =							False






      def __UpperCAmelCase       ( self			:Optional[int]	)       ->					str:




             '''simple docstring'''
             super().setUp()

             _a    : Dict     					=[
                 """<d>""",
                 """</d>""",
                 """<s>""",
                 """</s>""",
                 """</_>""",
                 """<unk>""",
                 """<pad>""",
                 """</n>""",
                 """我""",
                 """是""",
                 """C""",
                 """P""",
                 """M""",
                 """A""",
                 """n""",
                 """t""",
             ]
             _a    : Dict     					=os.path.join(self.tmpdirname     ,						VOCAB_FILES_NAMES["""vocab_file"""]	)
             with open(self.vocab_file     ,						"""w"""     ,						encoding="""utf-8"""	) as vocab_writer:
                    vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]	)	)






      @tooslow
      def __UpperCAmelCase       ( self			:Tuple	)       ->					int:




             '''simple docstring'''
             _a    : Optional[Any]     					=CpmAntTokenizer.from_pretrained("""openbmb/cpm-ant-10b"""	)
             _a    : Optional[int]     					="""今天天气真好！"""
             _a    : Union[str, Any]     					=["""今天""", """天气""", """真""", """好""", """！"""]
             _a    : List[Any]     					=tokenizer.tokenize(SCREAMING_SNAKE_CASE	)
             self.assertListEqual(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)
             _a    : Optional[int]     					="""今天天气真好！"""
             _a    : int     					=[tokenizer.bos_token] + tokens

             _a    : Dict     					=[6, 9_8_0_2, 1_4_9_6_2, 2_0_8_2, 8_3_1, 2_4_4]
             self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE	)     ,						SCREAMING_SNAKE_CASE	)

             _a    : List[str]     					=tokenizer.decode(SCREAMING_SNAKE_CASE	)
             self.assertEqual(SCREAMING_SNAKE_CASE     ,						SCREAMING_SNAKE_CASE	)






'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


A__:    List[str]					    =       {
    '''configuration_chinese_clip''': [
        '''CHINESE_CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''ChineseCLIPConfig''',
        '''ChineseCLIPOnnxConfig''',
        '''ChineseCLIPTextConfig''',
        '''ChineseCLIPVisionConfig''',
    ],
    '''processing_chinese_clip''': ['''ChineseCLIPProcessor'''],
}

try:
   if not is_vision_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A__:    Optional[int]					    =       ['''ChineseCLIPFeatureExtractor''']
   A__:    Any					    =       ['''ChineseCLIPImageProcessor''']

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   A__:    Dict					    =       [
       '''CHINESE_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''ChineseCLIPModel''',
       '''ChineseCLIPPreTrainedModel''',
       '''ChineseCLIPTextModel''',
       '''ChineseCLIPVisionModel''',
   ]

if TYPE_CHECKING:
   from .configuration_chinese_clip import (
       CHINESE_CLIP_PRETRAINED_CONFIG_ARCHIVE_MAP,
       ChineseCLIPConfig,
       ChineseCLIPOnnxConfig,
       ChineseCLIPTextConfig,
       ChineseCLIPVisionConfig,
   )
   from .processing_chinese_clip import ChineseCLIPProcessor

   try:
      if not is_vision_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .feature_extraction_chinese_clip import ChineseCLIPFeatureExtractor, ChineseCLIPImageProcessor

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_chinese_clip import (
          CHINESE_CLIP_PRETRAINED_MODEL_ARCHIVE_LIST,
          ChineseCLIPModel,
          ChineseCLIPPreTrainedModel,
          ChineseCLIPTextModel,
          ChineseCLIPVisionModel,
      )

else:
   import sys

   A__:    str					    =       _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)


