


"""simple docstring"""




import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class 					lowerCAmelCase	(					lowerCAmelCase_	,    unittest.TestCase    ):




		'''simple docstring'''







		SCREAMING_SNAKE_CASE_						:     Any								=     BertTokenizer
		SCREAMING_SNAKE_CASE_						:     Optional[int]								=     BertTokenizerFast
		SCREAMING_SNAKE_CASE_						:     Optional[Any]								=     True
		SCREAMING_SNAKE_CASE_						:     Tuple								=     True
		SCREAMING_SNAKE_CASE_						:     int								=     filter_non_english







		def 							__A						(	self				)     ->							Any:
								super().setUp()

								SCREAMING_SNAKE_CASE													=					[
								    '[UNK]',
								    '[CLS]',
								    '[SEP]',
								    '[PAD]',
								    '[MASK]',
								    'want',
								    '##want',
								    '##ed',
								    'wa',
								    'un',
								    'runn',
								    '##ing',
								    ',',
								    'low',
								    'lowest',
								]
								SCREAMING_SNAKE_CASE													=					os.path.join(self.tmpdirname						,       VOCAB_FILES_NAMES['vocab_file']				)
								with open(self.vocab_file						,       'w'						,       encoding='utf-8'				) as vocab_writer:
														vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]				)				)







		def 							__A						(	self						,       lowerCAmelCase__				)     ->							str:
								SCREAMING_SNAKE_CASE													=					'UNwant\u00E9d,running'
								SCREAMING_SNAKE_CASE													=					'unwanted, running'
								return input_text, output_text







		def 							__A						(	self				)     ->							Dict:
								SCREAMING_SNAKE_CASE													=					self.tokenizer_class(self.vocab_file				)

								SCREAMING_SNAKE_CASE													=					tokenizer.tokenize('UNwant\u00E9d,running'				)
								self.assertListEqual(_snake_case						,       ['un', '##want', '##ed', ',', 'runn', '##ing']				)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_snake_case				)						,       [9, 6, 7, 12, 10, 11]				)







		def 							__A						(	self				)     ->							List[str]:
								if not self.test_rust_tokenizer:
														return

								SCREAMING_SNAKE_CASE													=					self.get_tokenizer()
								SCREAMING_SNAKE_CASE													=					self.get_rust_tokenizer()

								SCREAMING_SNAKE_CASE													=					'UNwant\u00E9d,running'

								SCREAMING_SNAKE_CASE													=					tokenizer.tokenize(_snake_case				)
								SCREAMING_SNAKE_CASE													=					rust_tokenizer.tokenize(_snake_case				)
								self.assertListEqual(_snake_case						,       _snake_case				)

								SCREAMING_SNAKE_CASE													=					tokenizer.encode(_snake_case						,       add_special_tokens=_snake_case				)
								SCREAMING_SNAKE_CASE													=					rust_tokenizer.encode(_snake_case						,       add_special_tokens=_snake_case				)
								self.assertListEqual(_snake_case						,       _snake_case				)

								SCREAMING_SNAKE_CASE													=					self.get_rust_tokenizer()
								SCREAMING_SNAKE_CASE													=					tokenizer.encode(_snake_case				)
								SCREAMING_SNAKE_CASE													=					rust_tokenizer.encode(_snake_case				)
								self.assertListEqual(_snake_case						,       _snake_case				)

								# With lower casing
								SCREAMING_SNAKE_CASE													=					self.get_tokenizer(do_lower_case=_snake_case				)
								SCREAMING_SNAKE_CASE													=					self.get_rust_tokenizer(do_lower_case=_snake_case				)

								SCREAMING_SNAKE_CASE													=					'UNwant\u00E9d,running'

								SCREAMING_SNAKE_CASE													=					tokenizer.tokenize(_snake_case				)
								SCREAMING_SNAKE_CASE													=					rust_tokenizer.tokenize(_snake_case				)
								self.assertListEqual(_snake_case						,       _snake_case				)

								SCREAMING_SNAKE_CASE													=					tokenizer.encode(_snake_case						,       add_special_tokens=_snake_case				)
								SCREAMING_SNAKE_CASE													=					rust_tokenizer.encode(_snake_case						,       add_special_tokens=_snake_case				)
								self.assertListEqual(_snake_case						,       _snake_case				)

								SCREAMING_SNAKE_CASE													=					self.get_rust_tokenizer()
								SCREAMING_SNAKE_CASE													=					tokenizer.encode(_snake_case				)
								SCREAMING_SNAKE_CASE													=					rust_tokenizer.encode(_snake_case				)
								self.assertListEqual(_snake_case						,       _snake_case				)







		def 							__A						(	self				)     ->							Any:
								SCREAMING_SNAKE_CASE													=					BasicTokenizer()

								self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz'				)						,       ['ah', '\u535A', '\u63A8', 'zz']				)







		def 							__A						(	self				)     ->							Any:
								SCREAMING_SNAKE_CASE													=					BasicTokenizer(do_lower_case=_snake_case				)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '				)						,       ['hello', '!', 'how', 'are', 'you', '?']				)
								self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				)						,       ['hello']				)







		def 							__A						(	self				)     ->							Any:
								SCREAMING_SNAKE_CASE													=					BasicTokenizer(do_lower_case=_snake_case						,       strip_accents=_snake_case				)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				)						,       ['hällo', '!', 'how', 'are', 'you', '?']				)
								self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				)						,       ['h\u00E9llo']				)







		def 							__A						(	self				)     ->							str:
								SCREAMING_SNAKE_CASE													=					BasicTokenizer(do_lower_case=_snake_case						,       strip_accents=_snake_case				)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				)						,       ['hallo', '!', 'how', 'are', 'you', '?']				)
								self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				)						,       ['hello']				)







		def 							__A						(	self				)     ->							List[str]:
								SCREAMING_SNAKE_CASE													=					BasicTokenizer(do_lower_case=_snake_case				)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				)						,       ['hallo', '!', 'how', 'are', 'you', '?']				)
								self.assertListEqual(tokenizer.tokenize('H\u00E9llo'				)						,       ['hello']				)







		def 							__A						(	self				)     ->							Optional[Any]:
								SCREAMING_SNAKE_CASE													=					BasicTokenizer(do_lower_case=_snake_case				)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  '				)						,       ['HeLLo', '!', 'how', 'Are', 'yoU', '?']				)







		def 							__A						(	self				)     ->							Optional[int]:
								SCREAMING_SNAKE_CASE													=					BasicTokenizer(do_lower_case=_snake_case						,       strip_accents=_snake_case				)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				)						,       ['HäLLo', '!', 'how', 'Are', 'yoU', '?']				)







		def 							__A						(	self				)     ->							List[Any]:
								SCREAMING_SNAKE_CASE													=					BasicTokenizer(do_lower_case=_snake_case						,       strip_accents=_snake_case				)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  '				)						,       ['HaLLo', '!', 'how', 'Are', 'yoU', '?']				)







		def 							__A						(	self				)     ->							Optional[Any]:
								SCREAMING_SNAKE_CASE													=					BasicTokenizer(do_lower_case=_snake_case						,       never_split=['[UNK]']				)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]'				)						,       ['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]']				)







		def 							__A						(	self				)     ->							Union[str, Any]:
								SCREAMING_SNAKE_CASE													=					BasicTokenizer()
								SCREAMING_SNAKE_CASE													=					'a\n\'ll !!to?\'d of, can\'t.'
								SCREAMING_SNAKE_CASE													=					['a', '\'', 'll', '!', '!', 'to', '?', '\'', 'd', 'of', ',', 'can', '\'', 't', '.']
								self.assertListEqual(tokenizer.tokenize(_snake_case				)						,       _snake_case				)







		def 							__A						(	self				)     ->							List[Any]:
								SCREAMING_SNAKE_CASE													=					['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

								SCREAMING_SNAKE_CASE													=					{}
								for i, token in enumerate(_snake_case				):
														SCREAMING_SNAKE_CASE													=					i
								SCREAMING_SNAKE_CASE													=					WordpieceTokenizer(vocab=_snake_case						,       unk_token='[UNK]'				)

								self.assertListEqual(tokenizer.tokenize(''				)						,       []				)

								self.assertListEqual(tokenizer.tokenize('unwanted running'				)						,       ['un', '##want', '##ed', 'runn', '##ing']				)

								self.assertListEqual(tokenizer.tokenize('unwantedX running'				)						,       ['[UNK]', 'runn', '##ing']				)







		def 							__A						(	self				)     ->							Any:
								self.assertTrue(_is_whitespace(' '				)				)
								self.assertTrue(_is_whitespace('\t'				)				)
								self.assertTrue(_is_whitespace('\r'				)				)
								self.assertTrue(_is_whitespace('\n'				)				)
								self.assertTrue(_is_whitespace('\u00A0'				)				)

								self.assertFalse(_is_whitespace('A'				)				)
								self.assertFalse(_is_whitespace('-'				)				)







		def 							__A						(	self				)     ->							Optional[int]:
								self.assertTrue(_is_control('\u0005'				)				)

								self.assertFalse(_is_control('A'				)				)
								self.assertFalse(_is_control(' '				)				)
								self.assertFalse(_is_control('\t'				)				)
								self.assertFalse(_is_control('\r'				)				)







		def 							__A						(	self				)     ->							Tuple:
								self.assertTrue(_is_punctuation('-'				)				)
								self.assertTrue(_is_punctuation('$'				)				)
								self.assertTrue(_is_punctuation('`'				)				)
								self.assertTrue(_is_punctuation('.'				)				)

								self.assertFalse(_is_punctuation('A'				)				)
								self.assertFalse(_is_punctuation(' '				)				)







		def 							__A						(	self				)     ->							Any:
								SCREAMING_SNAKE_CASE													=					self.get_tokenizer()
								SCREAMING_SNAKE_CASE													=					self.get_rust_tokenizer()

								# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
								self.assertListEqual([tokenizer.tokenize(_snake_case				) for t in ['Test', '\xad', 'test']]						,       [['[UNK]'], [], ['[UNK]']]				)

								self.assertListEqual(
								    [rust_tokenizer.tokenize(_snake_case				) for t in ['Test', '\xad', 'test']]						,       [['[UNK]'], [], ['[UNK]']]				)







		@slow
		def 							__A						(	self				)     ->							Any:
								SCREAMING_SNAKE_CASE													=					self.tokenizer_class.from_pretrained('bert-base-uncased'				)

								SCREAMING_SNAKE_CASE													=					tokenizer.encode('sequence builders'						,       add_special_tokens=_snake_case				)
								SCREAMING_SNAKE_CASE													=					tokenizer.encode('multi-sequence build'						,       add_special_tokens=_snake_case				)

								SCREAMING_SNAKE_CASE													=					tokenizer.build_inputs_with_special_tokens(_snake_case				)
								SCREAMING_SNAKE_CASE													=					tokenizer.build_inputs_with_special_tokens(_snake_case						,       _snake_case				)

								assert encoded_sentence == [101] + text + [102]
								assert encoded_pair == [101] + text + [102] + text_a + [102]







		def 							__A						(	self				)     ->							List[Any]:
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'				):
																				SCREAMING_SNAKE_CASE													=					self.rust_tokenizer_class.from_pretrained(_snake_case						,       **_snake_case				)

																				SCREAMING_SNAKE_CASE													=					F'A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'
																				SCREAMING_SNAKE_CASE													=					tokenizer_r.encode_plus(
																				    _snake_case						,       return_attention_mask=_snake_case						,       return_token_type_ids=_snake_case						,       return_offsets_mapping=_snake_case						,       add_special_tokens=_snake_case						,       )

																				SCREAMING_SNAKE_CASE													=					tokenizer_r.do_lower_case if hasattr(_snake_case						,       'do_lower_case'				) else False
																				SCREAMING_SNAKE_CASE													=					(
																				    [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), 'A'),
																				        ((1, 2), ','),
																				        ((3, 5), 'na'),
																				        ((5, 6), '##ï'),
																				        ((6, 8), '##ve'),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), 'Allen'),
																				        ((21, 23), '##NL'),
																				        ((23, 24), '##P'),
																				        ((25, 33), 'sentence'),
																				        ((33, 34), '.'),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				    if not do_lower_case
																				    else [
																				        ((0, 0), tokenizer_r.cls_token),
																				        ((0, 1), 'a'),
																				        ((1, 2), ','),
																				        ((3, 8), 'naive'),
																				        ((9, 15), tokenizer_r.mask_token),
																				        ((16, 21), 'allen'),
																				        ((21, 23), '##nl'),
																				        ((23, 24), '##p'),
																				        ((25, 33), 'sentence'),
																				        ((33, 34), '.'),
																				        ((0, 0), tokenizer_r.sep_token),
																				    ]
																				)

																				self.assertEqual(
																				    [e[1] for e in expected_results]						,       tokenizer_r.convert_ids_to_tokens(tokens['input_ids']				)				)
																				self.assertEqual([e[0] for e in expected_results]						,       tokens['offset_mapping']				)







		def 							__A						(	self				)     ->							Optional[Any]:
								SCREAMING_SNAKE_CASE													=					['的', '人', '有']
								SCREAMING_SNAKE_CASE													=					''.join(_snake_case				)
								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
														with self.subTest(F'{tokenizer.__class__.__name__} ({pretrained_name})'				):
																				SCREAMING_SNAKE_CASE													=					True
																				SCREAMING_SNAKE_CASE													=					self.tokenizer_class.from_pretrained(_snake_case						,       **_snake_case				)
																				SCREAMING_SNAKE_CASE													=					self.rust_tokenizer_class.from_pretrained(_snake_case						,       **_snake_case				)

																				SCREAMING_SNAKE_CASE													=					tokenizer_p.encode(_snake_case						,       add_special_tokens=_snake_case				)
																				SCREAMING_SNAKE_CASE													=					tokenizer_r.encode(_snake_case						,       add_special_tokens=_snake_case				)

																				SCREAMING_SNAKE_CASE													=					tokenizer_r.convert_ids_to_tokens(_snake_case				)
																				SCREAMING_SNAKE_CASE													=					tokenizer_p.convert_ids_to_tokens(_snake_case				)

																				# it is expected that each Chinese character is not preceded by "##"
																				self.assertListEqual(_snake_case						,       _snake_case				)
																				self.assertListEqual(_snake_case						,       _snake_case				)

																				SCREAMING_SNAKE_CASE													=					False
																				SCREAMING_SNAKE_CASE													=					self.rust_tokenizer_class.from_pretrained(_snake_case						,       **_snake_case				)
																				SCREAMING_SNAKE_CASE													=					self.tokenizer_class.from_pretrained(_snake_case						,       **_snake_case				)

																				SCREAMING_SNAKE_CASE													=					tokenizer_r.encode(_snake_case						,       add_special_tokens=_snake_case				)
																				SCREAMING_SNAKE_CASE													=					tokenizer_p.encode(_snake_case						,       add_special_tokens=_snake_case				)

																				SCREAMING_SNAKE_CASE													=					tokenizer_r.convert_ids_to_tokens(_snake_case				)
																				SCREAMING_SNAKE_CASE													=					tokenizer_p.convert_ids_to_tokens(_snake_case				)

																				# it is expected that only the first Chinese character is not preceded by "##".
																				SCREAMING_SNAKE_CASE													=					[
																				    F'##{token}' if idx != 0 else token for idx, token in enumerate(_snake_case				)
																				]
																				self.assertListEqual(_snake_case						,       _snake_case				)
																				self.assertListEqual(_snake_case						,       _snake_case				)







from collections import UserDict
from typing import Union

import numpy as np
import requests

from ..utils import (
    add_end_docstrings,
    logging,
)
from .audio_classification import ffmpeg_read
from .base import PIPELINE_INIT_ARGS, Pipeline


a_   :List[Any]        =      logging.get_logger(__name__)




@add_end_docstrings(lowerCAmelCase_     )
class 					snake_case__						(       lowerCAmelCase_     ):






     """simple docstring"""


     def __init__(       self  :  Optional[Any], **_snake_case  :  str					)   ->Dict:
            super().__init__(**_snake_case					)

            if self.framework != "pt":
                   raise ValueError(F'''The {self.__class__} is only available in PyTorch.'''					)
            # No specific FOR_XXX available yet






     def __call__(       self  :  Union[str, Any], _snake_case  :  Union[np.ndarray, bytes, str], **_snake_case  :  Tuple					)   ->Dict:
            return super().__call__(_snake_case, **_snake_case					)






     def    lowercase_   (       self  :  Tuple, **_snake_case  :  Any					)   ->Union[str, Any]:
            snake_case__							:	str   	=	{}
            if "candidate_labels" in kwargs:
                   snake_case__							:	str   	=	kwargs['candidate_labels']
            if "hypothesis_template" in kwargs:
                   snake_case__							:	str   	=	kwargs['hypothesis_template']

            return preprocess_params, {}, {}






     def    lowercase_   (       self  :  Dict, _snake_case  :  str, _snake_case  :  Optional[int]=None, _snake_case  :  List[str]="This is a sound of {}."					)   ->int:
            if isinstance(_snake_case, _snake_case					):
                   if audio.startswith('http://'					) or audio.startswith('https://'					):
                          # We need to actually check for a real protocol, otherwise it's impossible to use a local file
                          # like http_huggingface_co.png
                          snake_case__							:	List[Any]   	=	requests.get(_snake_case					).content
                   else:
                          with open(_snake_case, 'rb'					) as f:
                                 snake_case__							:	Union[str, Any]   	=	f.read()

            if isinstance(_snake_case, _snake_case					):
                   snake_case__							:	List[Any]   	=	ffmpeg_read(_snake_case, self.feature_extractor.sampling_rate					)

            if not isinstance(_snake_case, np.ndarray					):
                   raise ValueError('We expect a numpy ndarray as input'					)
            if len(audio.shape					) != 1:
                   raise ValueError('We expect a single channel audio input for ZeroShotAudioClassificationPipeline'					)

            snake_case__							:	Tuple   	=	self.feature_extractor(
                [audio], sampling_rate=self.feature_extractor.sampling_rate, return_tensors='pt'					)
            snake_case__							:	int   	=	candidate_labels
            snake_case__							:	int   	=	[hypothesis_template.format(_snake_case					) for x in candidate_labels]
            snake_case__							:	Optional[int]   	=	self.tokenizer(_snake_case, return_tensors=self.framework, padding=_snake_case					)
            snake_case__							:	List[Any]   	=	[text_inputs]
            return inputs






     def    lowercase_   (       self  :  Optional[int], _snake_case  :  Optional[Any]					)   ->int:
            snake_case__							:	Optional[int]   	=	model_inputs.pop('candidate_labels'					)
            snake_case__							:	str   	=	model_inputs.pop('text_inputs'					)
            if isinstance(text_inputs[0], _snake_case					):
                   snake_case__							:	Optional[Any]   	=	text_inputs[0]
            else:
                   # Batching case.
                   snake_case__							:	int   	=	text_inputs[0][0]

            snake_case__							:	Any   	=	self.model(**_snake_case, **_snake_case					)

            snake_case__							:	List[Any]   	=	{
                'candidate_labels': candidate_labels,
                'logits': outputs.logits_per_audio,
            }
            return model_outputs






     def    lowercase_   (       self  :  Union[str, Any], _snake_case  :  str					)   ->List[str]:
            snake_case__							:	int   	=	model_outputs.pop('candidate_labels'					)
            snake_case__							:	List[Any]   	=	model_outputs['logits'][0]

            if self.framework == "pt":
                   snake_case__							:	Tuple   	=	logits.softmax(dim=0					)
                   snake_case__							:	Union[str, Any]   	=	probs.tolist()
            else:
                   raise ValueError('`tf` framework not supported.'					)

            snake_case__							:	Union[str, Any]   	=	[
                {'score': score, 'label': candidate_label}
                for score, candidate_label in sorted(zip(_snake_case, _snake_case					), key=lambda _snake_case					: -x[0]					)
            ]
            return result






