# Lint as: python3
# pylint: enable=line-too-long
# pylint: disable=g-import-not-at-top,g-bad-import-order,wrong-import-position

lowerCAmelCase__     :   Any		=   "2.13.1"

import platform

import pyarrow
from packaging import version


if version.parse(platform.python_version()) < version.parse('''3.7'''):
				raise ImportWarning(
				    '''To use `datasets`, Python>=3.7 is required, and the current version of Python doesn\'t match this condition.'''
				)

if version.parse(pyarrow.__version__).major < 8:
				raise ImportWarning(
				    '''To use `datasets`, the module `pyarrow>=8.0.0` is required, and the current version of `pyarrow` doesn\'t match this condition.\n'''
				    '''If you are running this in a Google Colab, you should probably just restart the runtime to use the right version of `pyarrow`.'''
				)

del platform
del pyarrow
del version

from .arrow_dataset import Dataset
from .arrow_reader import ReadInstruction
from .builder import ArrowBasedBuilder, BeamBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder
from .combine import concatenate_datasets, interleave_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .download import *
from .features import *
from .fingerprint import disable_caching, enable_caching, is_caching_enabled, set_caching_enabled
from .info import DatasetInfo, MetricInfo
from .inspect import (
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_infos,
    get_dataset_split_names,
    inspect_dataset,
    inspect_metric,
    list_datasets,
    list_metrics,
)
from .iterable_dataset import IterableDataset
from .load import load_dataset, load_dataset_builder, load_from_disk, load_metric
from .metric import Metric
from .splits import (
    NamedSplit,
    NamedSplitAll,
    Split,
    SplitBase,
    SplitDict,
    SplitGenerator,
    SplitInfo,
    SubSplitInfo,
    percent,
)
from .tasks import *
from .utils import *
from .utils import logging


# deprecated modules
from datasets import arrow_dataset as _arrow_dataset  # isort:skip
from datasets import utils as _utils  # isort:skip
from datasets.utils import download_manager as _deprecated_download_manager  # isort:skip

lowerCAmelCase__     :   List[Any]		=   concatenate_datasets
lowerCAmelCase__     :   List[str]		=   DownloadConfig
lowerCAmelCase__     :   Union[str, Any]		=   DownloadManager
lowerCAmelCase__     :   List[Any]		=   DownloadMode
lowerCAmelCase__     :   List[str]		=   DownloadConfig
lowerCAmelCase__     :   str		=   DownloadMode
lowerCAmelCase__     :   Optional[int]		=   DownloadManager

del _arrow_dataset, _utils, _deprecated_download_manager



import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin


class      lowerCamelCase    (SCREAMING_SNAKE_CASE__     ,					unittest.TestCase    ):
  """simple docstring"""







  lowerCamelCase__   =  DebertaTokenizer
  lowerCamelCase__   =  True
  lowerCamelCase__   =  DebertaTokenizerFast







  def       __A   (							self       :				List[Any]      )     ->		Dict:
        super().setUp()

        # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
        SCREAMING_SNAKE_CASE_      =     [
            "l",
            "o",
            "w",
            "e",
            "r",
            "s",
            "t",
            "i",
            "d",
            "n",
            "\u0120",
            "\u0120l",
            "\u0120n",
            "\u0120lo",
            "\u0120low",
            "er",
            "\u0120lowest",
            "\u0120newer",
            "\u0120wider",
            "[UNK]",
        ]
        SCREAMING_SNAKE_CASE_      =     dict(zip(__magic_name__     ,   range(len(__magic_name__      )      )      )      )
        SCREAMING_SNAKE_CASE_      =     ["#version: 0.2", "\u0120 l", "\u0120l o", "\u0120lo w", "e r", ""]
        SCREAMING_SNAKE_CASE_      =     {"unk_token": "[UNK]"}

        SCREAMING_SNAKE_CASE_      =     os.path.join(self.tmpdirname     ,   VOCAB_FILES_NAMES["vocab_file"]      )
        SCREAMING_SNAKE_CASE_      =     os.path.join(self.tmpdirname     ,   VOCAB_FILES_NAMES["merges_file"]      )
        with open(self.vocab_file     ,   "w"     ,   encoding="utf-8"      ) as fp:
              fp.write(json.dumps(__magic_name__      ) + "\n"      )
        with open(self.merges_file     ,   "w"     ,   encoding="utf-8"      ) as fp:
              fp.write("\n".join(__magic_name__      )      )







  def       __A   (							self       :				str     ,   **__magic_name__       :				int      )     ->		Union[str, Any]:
        kwargs.update(self.special_tokens_map      )
        return self.tokenizer_class.from_pretrained(self.tmpdirname     ,   **__magic_name__      )







  def       __A   (							self       :				str     ,   __magic_name__       :				List[Any]      )     ->		Dict:
        SCREAMING_SNAKE_CASE_      =     "lower newer"
        SCREAMING_SNAKE_CASE_      =     "lower newer"
        return input_text, output_text







  def       __A   (							self       :				Union[str, Any]      )     ->		str:
        SCREAMING_SNAKE_CASE_      =     self.get_tokenizer()
        SCREAMING_SNAKE_CASE_      =     "lower newer"
        SCREAMING_SNAKE_CASE_      =     ["l", "o", "w", "er", "\u0120", "n", "e", "w", "er"]
        SCREAMING_SNAKE_CASE_      =     tokenizer.tokenize(__magic_name__      )
        self.assertListEqual(__magic_name__     ,   __magic_name__      )

        SCREAMING_SNAKE_CASE_      =     tokens + [tokenizer.unk_token]
        SCREAMING_SNAKE_CASE_      =     [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
        self.assertListEqual(tokenizer.convert_tokens_to_ids(__magic_name__      )     ,   __magic_name__      )







  def       __A   (							self       :				Optional[int]      )     ->		Optional[Any]:
        SCREAMING_SNAKE_CASE_      =     self.get_tokenizer()
        SCREAMING_SNAKE_CASE_      =     tokenizer("Hello"     ,   "World"      )
        SCREAMING_SNAKE_CASE_      =     [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
        self.assertListEqual(tokd["token_type_ids"]     ,   __magic_name__      )







  @slow
  def       __A   (							self       :				Any      )     ->		Any:
        SCREAMING_SNAKE_CASE_      =     self.tokenizer_class.from_pretrained("microsoft/deberta-base"      )

        SCREAMING_SNAKE_CASE_      =     tokenizer.encode("sequence builders"     ,   add_special_tokens=__magic_name__      )
        SCREAMING_SNAKE_CASE_      =     tokenizer.encode("multi-sequence build"     ,   add_special_tokens=__magic_name__      )

        SCREAMING_SNAKE_CASE_      =     tokenizer.encode(
            "sequence builders"     ,   add_special_tokens=__magic_name__     ,   add_prefix_space=__magic_name__      )
        SCREAMING_SNAKE_CASE_      =     tokenizer.encode(
            "sequence builders"     ,   "multi-sequence build"     ,   add_special_tokens=__magic_name__     ,   add_prefix_space=__magic_name__      )

        SCREAMING_SNAKE_CASE_      =     tokenizer.build_inputs_with_special_tokens(__magic_name__      )
        SCREAMING_SNAKE_CASE_      =     tokenizer.build_inputs_with_special_tokens(__magic_name__     ,   __magic_name__      )

        assert encoded_sentence == encoded_text_from_decode
        assert encoded_pair == encoded_pair_from_decode







  @slow
  def       __A   (							self       :				Tuple      )     ->		str:
        SCREAMING_SNAKE_CASE_      =     [self.tokenizer_class]
        if self.test_rust_tokenizer:
              tokenizer_classes.append(self.rust_tokenizer_class      )

        for tokenizer_class in tokenizer_classes:
              SCREAMING_SNAKE_CASE_      =     tokenizer_class.from_pretrained("microsoft/deberta-base"      )

              SCREAMING_SNAKE_CASE_      =     [
                  "ALBERT: A Lite BERT for Self-supervised Learning of Language Representations",
                  "ALBERT incorporates two parameter reduction techniques",
                  "The first one is a factorized embedding parameterization. By decomposing the large vocabulary"
                  " embedding matrix into two small matrices, we separate the size of the hidden layers from the size of"
                  " vocabulary embedding.",
              ]

              SCREAMING_SNAKE_CASE_      =     tokenizer(__magic_name__     ,   padding=__magic_name__      )
              SCREAMING_SNAKE_CASE_      =     [tokenizer.decode(__magic_name__     ,   skip_special_tokens=__magic_name__      ) for seq in encoding["input_ids"]]

              # fmt: off
              SCREAMING_SNAKE_CASE_      =     {
                  "input_ids": [
                      [1, 2_118, 11_126, 565, 35, 83, 25_191, 163, 18_854, 13, 12_156, 12, 16_101, 25_376, 13_807, 9, 22_205, 27_893, 1_635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                      [1, 2_118, 11_126, 565, 24_536, 80, 43_797, 4_878, 7_373, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                      [1, 133, 78, 65, 16, 10, 3_724, 1_538, 33_183, 11_303, 43_797, 1_938, 4, 870, 24_165, 29_105, 5, 739, 32_644, 33_183, 11_303, 36_173, 88, 80, 650, 7_821, 45_940, 6, 52, 2_559, 5, 1_836, 9, 5, 7_397, 13_171, 31, 5, 1_836, 9, 32_644, 33_183, 11_303, 4, 2]
                  ],
                  "token_type_ids": [
                      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                  ],
                  "attention_mask": [
                      [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                      [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                      [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
                  ]
              }
              # fmt: on

              SCREAMING_SNAKE_CASE_      =     [
                  "ALBERT: A Lite BERT for Self-supervised Learning of Language Representations",
                  "ALBERT incorporates two parameter reduction techniques",
                  "The first one is a factorized embedding parameterization. By decomposing the large vocabulary"
                  " embedding matrix into two small matrices, we separate the size of the hidden layers from the size of"
                  " vocabulary embedding.",
              ]

              self.assertDictEqual(encoding.data     ,   __magic_name__      )

              for expected, decoded in zip(__magic_name__     ,   __magic_name__      ):
                    self.assertEqual(__magic_name__     ,   __magic_name__      )


