
'''simple docstring'''

import os
import unittest

from transformers.models.cpmant.tokenization_cpmant import VOCAB_FILES_NAMES, CpmAntTokenizer
from transformers.testing_utils import require_jieba, tooslow

from ...test_tokenization_common import TokenizerTesterMixin






@require_jieba
class  A  (						SCREAMING_SNAKE_CASE__	,   unittest.TestCase     ):
	snake_case__   :List[Any]   =  CpmAntTokenizer
	snake_case__   :str   =  False





	def 				__SCREAMING_SNAKE_CASE	(       self      :							Dict				):


				"""simple docstring"""

				super().setUp()

				lowerCAmelCase__										=				[
				    "<d>",
				    "</d>",
				    "<s>",
				    "</s>",
				    "</_>",
				    "<unk>",
				    "<pad>",
				    "</n>",
				    "我",
				    "是",
				    "C",
				    "P",
				    "M",
				    "A",
				    "n",
				    "t",
				]
				lowerCAmelCase__										=				os.path.join(self.tmpdirname   ,				VOCAB_FILES_NAMES["vocab_file"]				)
				with open(self.vocab_file   ,				"w"   ,				encoding="utf-8"				) as vocab_writer:
							vocab_writer.write("".join([x + "\n" for x in vocab_tokens]				)				)







	@tooslow
	def 				__SCREAMING_SNAKE_CASE	(       self      :							Optional[int]				):


				"""simple docstring"""

				lowerCAmelCase__										=				CpmAntTokenizer.from_pretrained("openbmb/cpm-ant-10b"				)
				lowerCAmelCase__										=				"今天天气真好！"
				lowerCAmelCase__										=				["今天", "天气", "真", "好", "！"]
				lowerCAmelCase__										=				tokenizer.tokenize(__magic_name__				)
				self.assertListEqual(__magic_name__   ,				__magic_name__				)
				lowerCAmelCase__										=				"今天天气真好！"
				lowerCAmelCase__										=				[tokenizer.bos_token] + tokens

				lowerCAmelCase__										=				[6, 9802, 14962, 2082, 831, 244]
				self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__				)   ,				__magic_name__				)

				lowerCAmelCase__										=				tokenizer.decode(__magic_name__				)
				self.assertEqual(__magic_name__   ,				__magic_name__				)








"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


UpperCAmelCase													={
    "configuration_mega": ["MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP", "MegaConfig", "MegaOnnxConfig"],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  UpperCAmelCase													=[
      "MEGA_PRETRAINED_MODEL_ARCHIVE_LIST",
      "MegaForCausalLM",
      "MegaForMaskedLM",
      "MegaForMultipleChoice",
      "MegaForQuestionAnswering",
      "MegaForSequenceClassification",
      "MegaForTokenClassification",
      "MegaModel",
      "MegaPreTrainedModel",
  ]

if TYPE_CHECKING:
  from .configuration_mega import MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP, MegaConfig, MegaOnnxConfig

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_mega import (
        MEGA_PRETRAINED_MODEL_ARCHIVE_LIST,
        MegaForCausalLM,
        MegaForMaskedLM,
        MegaForMultipleChoice,
        MegaForQuestionAnswering,
        MegaForSequenceClassification,
        MegaForTokenClassification,
        MegaModel,
        MegaPreTrainedModel,
    )

else:
  import sys

  UpperCAmelCase													=_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)




