






import os
import unittest

from transformers import FunnelTokenizer, FunnelTokenizerFast
from transformers.models.funnel.tokenization_funnel import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class     a    ( lowercase__							,       unittest.TestCase   ):



      """simple docstring"""


      a  : List[str]							       =	FunnelTokenizer
      a  : Optional[int]							       =	FunnelTokenizerFast
      a  : str							       =	True
      a  : Dict							       =	True
      def 	UpperCAmelCase      (      self					:			Any					)							->      int:
        super().setUp()

        __UpperCAmelCase							:	Any           =  [
            """<unk>""",
            """<cls>""",
            """<sep>""",
            """want""",
            """##want""",
            """##ed""",
            """wa""",
            """un""",
            """runn""",
            """##ing""",
            """,""",
            """low""",
            """lowest""",
        ]
        __UpperCAmelCase							:	Tuple           =  os.path.join(self.tmpdirname		,      VOCAB_FILES_NAMES["""vocab_file"""]					)
        with open(self.vocab_file		,      """w"""		,      encoding="""utf-8"""					) as vocab_writer:
          vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]					)					)
      def 	UpperCAmelCase      (      self					:			int		,      **__lowercase					:			str					)							->      Dict:
        return FunnelTokenizer.from_pretrained(self.tmpdirname		,      **__lowercase					)
      def 	UpperCAmelCase      (      self					:			Any		,      **__lowercase					:			Dict					)							->      Union[str, Any]:
        return FunnelTokenizerFast.from_pretrained(self.tmpdirname		,      **__lowercase					)
      def 	UpperCAmelCase      (      self					:			int		,      __lowercase					:			Tuple					)							->      Tuple:
        __UpperCAmelCase							:	List[Any]           =  """UNwant\u00E9d,running"""
        __UpperCAmelCase							:	List[str]           =  """unwanted, running"""
        return input_text, output_text
      def 	UpperCAmelCase      (      self					:			Any					)							->      List[Any]:
        __UpperCAmelCase							:	Optional[int]           =  self.tokenizer_class(self.vocab_file					)

        __UpperCAmelCase							:	Union[str, Any]           =  tokenizer.tokenize("""UNwant\u00E9d,running"""					)
        self.assertListEqual(__lowercase		,      ["""un""", """##want""", """##ed""", """,""", """runn""", """##ing"""]					)
        self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowercase					)		,      [7, 4, 5, 10, 8, 9]					)




      def 	UpperCAmelCase      (      self					:			Optional[int]					)							->      Tuple:
        __UpperCAmelCase							:	List[Any]           =  self.get_tokenizers(do_lower_case=__lowercase					)
        for tokenizer in tokenizers:
          __UpperCAmelCase							:	int           =  tokenizer("""UNwant\u00E9d,running"""					)
          __UpperCAmelCase							:	str           =  len(inputs["""input_ids"""]					) - 1
          self.assertListEqual(inputs["""token_type_ids"""]		,      [2] + [0] * sentence_len					)

          __UpperCAmelCase							:	Any           =  tokenizer("""UNwant\u00E9d,running"""		,      """UNwant\u00E9d,running"""					)
          self.assertListEqual(inputs["""token_type_ids"""]		,      [2] + [0] * sentence_len + [1] * sentence_len					)










import warnings
from contextlib import contextmanager

from ....processing_utils import ProcessorMixin
class     a    ( lowercase__   ):



      """simple docstring"""


      a  : List[str]							       =	'MCTCTFeatureExtractor'
      a  : str							       =	'AutoTokenizer'
      def __init__(      self					:			Tuple		,      __lowercase					:			int		,      __lowercase					:			Dict					)							->      Any:
        super().__init__(__lowercase		,      __lowercase					)
        __UpperCAmelCase							:	Optional[Any]           =  self.feature_extractor
        __UpperCAmelCase							:	Optional[int]           =  False
      def __call__(      self					:			int		,      *__lowercase					:			Tuple		,      **__lowercase					:			Optional[int]					)							->      Union[str, Any]:
        # For backward compatibility
        if self._in_target_context_manager:
          return self.current_processor(*__lowercase		,      **__lowercase					)

        if "raw_speech" in kwargs:
          warnings.warn("""Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead."""					)
          __UpperCAmelCase							:	Dict           =  kwargs.pop("""raw_speech"""					)
        else:
          __UpperCAmelCase							:	Dict           =  kwargs.pop("""audio"""		,      __lowercase					)
        __UpperCAmelCase							:	List[str]           =  kwargs.pop("""sampling_rate"""		,      __lowercase					)
        __UpperCAmelCase							:	Tuple           =  kwargs.pop("""text"""		,      __lowercase					)
        if len(__lowercase					) > 0:
          __UpperCAmelCase							:	Tuple           =  args[0]
          __UpperCAmelCase							:	str           =  args[1:]

        if audio is None and text is None:
          raise ValueError("""You need to specify either an `audio` or `text` input to process."""					)

        if audio is not None:
          __UpperCAmelCase							:	Tuple           =  self.feature_extractor(__lowercase		,      *__lowercase		,      sampling_rate=__lowercase		,      **__lowercase					)
        if text is not None:
          __UpperCAmelCase							:	str           =  self.tokenizer(__lowercase		,      **__lowercase					)

        if text is None:
          return inputs
        elif audio is None:
          return encodings
        else:
          __UpperCAmelCase							:	Dict           =  encodings["""input_ids"""]
          return inputs
      def 	UpperCAmelCase      (      self					:			Optional[Any]		,      *__lowercase					:			List[Any]		,      **__lowercase					:			int					)							->      List[Any]:
        return self.tokenizer.batch_decode(*__lowercase		,      **__lowercase					)
      def 	UpperCAmelCase      (      self					:			Optional[int]		,      *__lowercase					:			Optional[Any]		,      **__lowercase					:			List[str]					)							->      int:
        # For backward compatibility
        if self._in_target_context_manager:
          return self.current_processor.pad(*__lowercase		,      **__lowercase					)

        __UpperCAmelCase							:	Optional[int]           =  kwargs.pop("""input_features"""		,      __lowercase					)
        __UpperCAmelCase							:	Optional[Any]           =  kwargs.pop("""labels"""		,      __lowercase					)
        if len(__lowercase					) > 0:
          __UpperCAmelCase							:	Union[str, Any]           =  args[0]
          __UpperCAmelCase							:	str           =  args[1:]

        if input_features is not None:
          __UpperCAmelCase							:	Any           =  self.feature_extractor.pad(__lowercase		,      *__lowercase		,      **__lowercase					)
        if labels is not None:
          __UpperCAmelCase							:	Union[str, Any]           =  self.tokenizer.pad(__lowercase		,      **__lowercase					)

        if labels is None:
          return input_features
        elif input_features is None:
          return labels
        else:
          __UpperCAmelCase							:	Any           =  labels["""input_ids"""]
          return input_features
      def 	UpperCAmelCase      (      self					:			Any		,      *__lowercase					:			Union[str, Any]		,      **__lowercase					:			Dict					)							->      List[Any]:
        return self.tokenizer.decode(*__lowercase		,      **__lowercase					)




      @contextmanager
      def 	UpperCAmelCase      (      self					:			Optional[Any]					)							->      Tuple:
        warnings.warn(
            """`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your """
            """labels by using the argument `text` of the regular `__call__` method (either in the same call as """
            """your audio inputs, or in a separate call."""					)
        __UpperCAmelCase							:	Any           =  True
        __UpperCAmelCase							:	Optional[int]           =  self.tokenizer
        yield
        __UpperCAmelCase							:	List[Any]           =  self.feature_extractor
        __UpperCAmelCase							:	int           =  False


