


import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin
class 	UpperCAmelCase_					(					UpperCamelCase		, unittest.TestCase    ):

   '''simple docstring'''




   __A :      Dict     		=		XLMTokenizer
   __A :      Dict     		=		False






   def _snake_case    (		self ):






     """simple docstring"""





     super().setUp()

     # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
     lowerCamelCase :					str    =				[
         "l",
         "o",
         "w",
         "e",
         "r",
         "s",
         "t",
         "i",
         "d",
         "n",
         "w</w>",
         "r</w>",
         "t</w>",
         "lo",
         "low",
         "er</w>",
         "low</w>",
         "lowest</w>",
         "newer</w>",
         "wider</w>",
         "<unk>",
     ]
     lowerCamelCase :					Optional[Any]    =				dict(zip(__A						,				range(len(__A ) ) ) )
     lowerCamelCase :					str    =				["l o 123", "lo w 1456", "e r</w> 1789", ""]

     lowerCamelCase :					Dict    =				os.path.join(self.tmpdirname						,				VOCAB_FILES_NAMES["vocab_file"] )
     lowerCamelCase :					Optional[int]    =				os.path.join(self.tmpdirname						,				VOCAB_FILES_NAMES["merges_file"] )
     with open(self.vocab_file						,				"w" ) as fp:
       fp.write(json.dumps(__A ) )
     with open(self.merges_file						,				"w" ) as fp:
       fp.write("\n".join(__A ) )






   def _snake_case    (		self						,				__A ):






     """simple docstring"""





     lowerCamelCase :					Dict    =				"lower newer"
     lowerCamelCase :					int    =				"lower newer"
     return input_text, output_text






   def _snake_case    (		self ):






     """simple docstring"""





     lowerCamelCase :					List[Any]    =				XLMTokenizer(self.vocab_file						,				self.merges_file )

     lowerCamelCase :					Optional[Any]    =				"lower"
     lowerCamelCase :					List[str]    =				["low", "er</w>"]
     lowerCamelCase :					str    =				tokenizer.tokenize(__A )
     self.assertListEqual(__A						,				__A )

     lowerCamelCase :					Dict    =				tokens + ["<unk>"]
     lowerCamelCase :					Optional[int]    =				[14, 15, 20]
     self.assertListEqual(tokenizer.convert_tokens_to_ids(__A )						,				__A )






   @slow
   def _snake_case    (		self ):






     """simple docstring"""





     lowerCamelCase :					List[Any]    =				XLMTokenizer.from_pretrained("xlm-mlm-en-2048" )

     lowerCamelCase :					Tuple    =				tokenizer.encode("sequence builders"						,				add_special_tokens=__A )
     lowerCamelCase :					Any    =				tokenizer.encode("multi-sequence build"						,				add_special_tokens=__A )

     lowerCamelCase :					Tuple    =				tokenizer.build_inputs_with_special_tokens(__A )
     lowerCamelCase :					List[str]    =				tokenizer.build_inputs_with_special_tokens(__A						,				__A )

     assert encoded_sentence == [0] + text + [1]
     assert encoded_pair == [0] + text + [1] + text_a + [1]







# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ..models.whisper import WhisperForConditionalGeneration, WhisperProcessor
from .base import PipelineTool
class 	UpperCAmelCase_					(					UpperCamelCase    ):

   '''simple docstring'''




   __A :      Dict     		=		"openai/whisper-base"
   __A :      str     		=		(
       "This is a tool that transcribes an audio into text. It takes an input named `audio` and returns the "
       "transcribed text."
   )
   __A :      Any     		=		"transcriber"
   __A :      Any     		=		WhisperProcessor
   __A :      int     		=		WhisperForConditionalGeneration

   __A :      Any     		=		["audio"]
   __A :      List[str]     		=		["text"]






   def _snake_case    (		self						,				__A ):






     """simple docstring"""





     return self.pre_processor(__A						,				return_tensors="pt" ).input_features






   def _snake_case    (		self						,				__A ):






     """simple docstring"""





     return self.model.generate(inputs=__A )






   def _snake_case    (		self						,				__A ):






     """simple docstring"""





     return self.pre_processor.batch_decode(__A						,				skip_special_tokens=__A )[0]



