





import json
import os
import unittest

from transformers.models.ctrl.tokenization_ctrl import VOCAB_FILES_NAMES, CTRLTokenizer

from ...test_tokenization_common import TokenizerTesterMixin

class 	A_       (a_     ,							unittest.TestCase			):
	UpperCAmelCase__					=			CTRLTokenizer
	UpperCAmelCase__					=			False
	UpperCAmelCase__					=			False
	def      _lowercase				(    self		):






				'''simple docstring'''







				super().setUp()

				# Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
				UpperCAmelCase   		=				['''adapt''', '''re@@''', '''a@@''', '''apt''', '''c@@''', '''t''', '''<unk>''']
				UpperCAmelCase   		=				dict(zip(_A							,					range(len(_A		)		)		)		)
				UpperCAmelCase   		=				['''#version: 0.2''', '''a p''', '''ap t</w>''', '''r e''', '''a d''', '''ad apt</w>''', '''''']
				UpperCAmelCase   		=				{'''unk_token''': '''<unk>'''}

				UpperCAmelCase   		=				os.path.join(self.tmpdirname							,					VOCAB_FILES_NAMES['''vocab_file''']		)
				UpperCAmelCase   		=				os.path.join(self.tmpdirname							,					VOCAB_FILES_NAMES['''merges_file''']		)
				with open(self.vocab_file							,					'''w'''							,					encoding='''utf-8'''		) as fp:
							fp.write(json.dumps(_A		) + '''\n'''		)
				with open(self.merges_file							,					'''w'''							,					encoding='''utf-8'''		) as fp:
							fp.write('''\n'''.join(_A		)		)
	def      _lowercase				(    self							,					**_A		):






				'''simple docstring'''







				kwargs.update(self.special_tokens_map		)
				return CTRLTokenizer.from_pretrained(self.tmpdirname							,					**_A		)
	def      _lowercase				(    self							,					_A		):






				'''simple docstring'''







				UpperCAmelCase   		=				'''adapt react readapt apt'''
				UpperCAmelCase   		=				'''adapt react readapt apt'''
				return input_text, output_text







	def      _lowercase				(    self		):






				'''simple docstring'''







				UpperCAmelCase   		=				CTRLTokenizer(self.vocab_file							,					self.merges_file							,					**self.special_tokens_map		)
				UpperCAmelCase   		=				'''adapt react readapt apt'''
				UpperCAmelCase   		=				'''adapt re@@ a@@ c@@ t re@@ adapt apt'''.split()
				UpperCAmelCase   		=				tokenizer.tokenize(_A		)
				self.assertListEqual(_A							,					_A		)

				UpperCAmelCase   		=				tokens + [tokenizer.unk_token]

				UpperCAmelCase   		=				[0, 1, 2, 4, 5, 1, 0, 3, 6]
				self.assertListEqual(tokenizer.convert_tokens_to_ids(_A		)							,					_A		)










from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_speech_available,
    is_tf_available,
    is_torch_available,
)


__A				:    Union[str, Any]       =   {
    "configuration_speech_to_text": ["SPEECH_TO_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP", "Speech2TextConfig"],
    "processing_speech_to_text": ["Speech2TextProcessor"],
}

try:
				if not is_sentencepiece_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				__A				:    List[Any]       =   ["Speech2TextTokenizer"]

try:
				if not is_speech_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				__A				:    List[Any]       =   ["Speech2TextFeatureExtractor"]

try:
				if not is_tf_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				__A				:    int       =   [
				    "TF_SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST",
				    "TFSpeech2TextForConditionalGeneration",
				    "TFSpeech2TextModel",
				    "TFSpeech2TextPreTrainedModel",
				]

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				__A				:    Tuple       =   [
				    "SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST",
				    "Speech2TextForConditionalGeneration",
				    "Speech2TextModel",
				    "Speech2TextPreTrainedModel",
				]


if TYPE_CHECKING:
				from .configuration_speech_to_text import SPEECH_TO_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP, SpeechaTextConfig
				from .processing_speech_to_text import SpeechaTextProcessor

				try:
								if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_speech_to_text import SpeechaTextTokenizer

				try:
								if not is_speech_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .feature_extraction_speech_to_text import SpeechaTextFeatureExtractor

				try:
								if not is_tf_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_tf_speech_to_text import (
								    TF_SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    TFSpeechaTextForConditionalGeneration,
								    TFSpeechaTextModel,
								    TFSpeechaTextPreTrainedModel,
								)

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_speech_to_text import (
								    SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    SpeechaTextForConditionalGeneration,
								    SpeechaTextModel,
								    SpeechaTextPreTrainedModel,
								)

else:
				import sys

				__A				:    Dict       =   _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



