

import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bart import BartTokenizer


_UpperCamelCase					     =						logging.get_logger(__name__)


_UpperCamelCase					     =						{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

# See all BART models at https://huggingface.co/models?filter=bart
_UpperCamelCase					     =						{
    "vocab_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/vocab.json",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/vocab.json",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/vocab.json",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/vocab.json",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/vocab.json",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/vocab.json",
    },
    "merges_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/merges.txt",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/merges.txt",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/merges.txt",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/merges.txt",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/merges.txt",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "facebook/bart-base": "https://huggingface.co/facebook/bart-base/resolve/main/tokenizer.json",
        "facebook/bart-large": "https://huggingface.co/facebook/bart-large/resolve/main/tokenizer.json",
        "facebook/bart-large-mnli": "https://huggingface.co/facebook/bart-large-mnli/resolve/main/tokenizer.json",
        "facebook/bart-large-cnn": "https://huggingface.co/facebook/bart-large-cnn/resolve/main/tokenizer.json",
        "facebook/bart-large-xsum": "https://huggingface.co/facebook/bart-large-xsum/resolve/main/tokenizer.json",
        "yjernite/bart_eli5": "https://huggingface.co/yjernite/bart_eli5/resolve/main/tokenizer.json",
    },
}

_UpperCamelCase					     =						{
    "facebook/bart-base": 1024,
    "facebook/bart-large": 1024,
    "facebook/bart-large-mnli": 1024,
    "facebook/bart-large-cnn": 1024,
    "facebook/bart-large-xsum": 1024,
    "yjernite/bart_eli5": 1024,
}







class     __lowercase (A__					):
 _UpperCamelCase         =    VOCAB_FILES_NAMES
 _UpperCamelCase         =    PRETRAINED_VOCAB_FILES_MAP
 _UpperCamelCase         =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _UpperCamelCase         =    ["""input_ids""", """attention_mask"""]
 _UpperCamelCase         =    BartTokenizer
 def __init__(				self ,				A_=None ,				A_=None ,				A_=None ,				A_="replace" ,				A_="<s>" ,				A_="</s>" ,				A_="</s>" ,				A_="<s>" ,				A_="<unk>" ,				A_="<pad>" ,				A_="<mask>" ,				A_=False ,				A_=True ,				**A_ ,				)   ->int:



       '''simple docstring'''





       super().__init__(
           a__ ,				a__ ,				tokenizer_file=a__ ,				errors=a__ ,				bos_token=a__ ,				eos_token=a__ ,				sep_token=a__ ,				cls_token=a__ ,				unk_token=a__ ,				pad_token=a__ ,				mask_token=a__ ,				add_prefix_space=a__ ,				trim_offsets=a__ ,				**a__ ,				)

       __lowerCAmelCase   :		Any   =   json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()   )
       if pre_tok_state.get('''add_prefix_space''' ,				a__   ) != add_prefix_space:
             __lowerCAmelCase   :		Tuple   =   getattr(a__ ,				pre_tok_state.pop('''type'''   )   )
             __lowerCAmelCase   :		Any   =   add_prefix_space
             __lowerCAmelCase   :		Any   =   pre_tok_class(**a__   )

       __lowerCAmelCase   :		List[Any]   =   add_prefix_space

       # the pre_tokenizer is already updated in the GPT2TokenizerFast `__init__`
       __lowerCAmelCase   :		Union[str, Any]   =   '''post_processor'''
       __lowerCAmelCase   :		Any   =   getattr(self.backend_tokenizer ,				a__ ,				a__   )
       if tokenizer_component_instance:
             __lowerCAmelCase   :		List[Any]   =   json.loads(tokenizer_component_instance.__getstate__()   )

             # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
             if "sep" in state:
                   __lowerCAmelCase   :		List[str]   =   tuple(state['''sep''']   )
             if "cls" in state:
                   __lowerCAmelCase   :		Tuple   =   tuple(state['''cls''']   )

             __lowerCAmelCase   :		Optional[int]   =   False

             if state.get('''add_prefix_space''' ,				a__   ) != add_prefix_space:
                   __lowerCAmelCase   :		Union[str, Any]   =   add_prefix_space
                   __lowerCAmelCase   :		Dict   =   True

             if state.get('''trim_offsets''' ,				a__   ) != trim_offsets:
                   __lowerCAmelCase   :		Optional[Any]   =   trim_offsets
                   __lowerCAmelCase   :		List[str]   =   True

             if changes_to_apply:
                   __lowerCAmelCase   :		Any   =   getattr(a__ ,				state.pop('''type'''   )   )
                   __lowerCAmelCase   :		str   =   component_class(**a__   )
                   setattr(self.backend_tokenizer ,				a__ ,				a__   )
 @property
 def       UpperCamelCase__    (				self   )   ->Tuple:



       '''simple docstring'''





       if self._mask_token is None:
             if self.verbose:
                   logger.error('''Using mask_token, but it is not set yet.'''   )
             return None
       return str(self._mask_token   )
 @mask_token.setter
 def       UpperCamelCase__    (				self ,				A_   )   ->Optional[Any]:



       '''simple docstring'''





       __lowerCAmelCase   :		List[Any]   =   AddedToken(a__ ,				lstrip=a__ ,				rstrip=a__   ) if isinstance(a__ ,				a__   ) else value
       __lowerCAmelCase   :		Union[str, Any]   =   value
 def       UpperCamelCase__    (				self ,				*A_ ,				**A_   )   ->str:



       '''simple docstring'''





       __lowerCAmelCase   :		Optional[Any]   =   kwargs.get('''is_split_into_words''' ,				a__   )

       if is_split_into_words and not self.add_prefix_space:
             raise ValueError(
                 f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
                 '''to use it with pretokenized inputs.'''   )

       return super()._batch_encode_plus(*a__ ,				**a__   )
 def       UpperCamelCase__    (				self ,				*A_ ,				**A_   )   ->Optional[int]:



       '''simple docstring'''





       __lowerCAmelCase   :		Optional[int]   =   kwargs.get('''is_split_into_words''' ,				a__   )

       if is_split_into_words and not self.add_prefix_space:
             raise ValueError(
                 f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
                 '''to use it with pretokenized inputs.'''   )

       return super()._encode_plus(*a__ ,				**a__   )
 def       UpperCamelCase__    (				self ,				A_ ,				A_ = None   )   ->Tuple:



       '''simple docstring'''





       __lowerCAmelCase   :		Optional[int]   =   self._tokenizer.model.save(a__ ,				name=a__   )
       return tuple(a__   )
 def       UpperCamelCase__    (				self ,				A_ ,				A_=None   )   ->List[str]:



       '''simple docstring'''





       __lowerCAmelCase   :		Any   =   [self.bos_token_id] + token_ids_a + [self.eos_token_id]
       if token_ids_a is None:
             return output

       return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]





 def       UpperCamelCase__    (				self ,				A_ ,				A_ = None   )   ->List[str]:



       '''simple docstring'''





       __lowerCAmelCase   :		Any   =   [self.sep_token_id]
       __lowerCAmelCase   :		Tuple   =   [self.cls_token_id]

       if token_ids_a is None:
             return len(cls + token_ids_a + sep   ) * [0]
       return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]







# Lint as: python3
# pylint: enable=line-too-long
# pylint: disable=g-import-not-at-top,g-bad-import-order,wrong-import-position

_lowercase    	=		"2.13.1"

import platform

import pyarrow
from packaging import version


if version.parse(platform.python_version()) < version.parse("3.7"):
   raise ImportWarning(
       "To use `datasets`, Python>=3.7 is required, and the current version of Python doesn't match this condition."
   )

if version.parse(pyarrow.__version__).major < 8:
   raise ImportWarning(
       "To use `datasets`, the module `pyarrow>=8.0.0` is required, and the current version of `pyarrow` doesn't match this condition.\n"
       "If you are running this in a Google Colab, you should probably just restart the runtime to use the right version of `pyarrow`."
   )

del platform
del pyarrow
del version

from .arrow_dataset import Dataset
from .arrow_reader import ReadInstruction
from .builder import ArrowBasedBuilder, BeamBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder
from .combine import concatenate_datasets, interleave_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .download import *
from .features import *
from .fingerprint import disable_caching, enable_caching, is_caching_enabled, set_caching_enabled
from .info import DatasetInfo, MetricInfo
from .inspect import (
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_infos,
    get_dataset_split_names,
    inspect_dataset,
    inspect_metric,
    list_datasets,
    list_metrics,
)
from .iterable_dataset import IterableDataset
from .load import load_dataset, load_dataset_builder, load_from_disk, load_metric
from .metric import Metric
from .splits import (
    NamedSplit,
    NamedSplitAll,
    Split,
    SplitBase,
    SplitDict,
    SplitGenerator,
    SplitInfo,
    SubSplitInfo,
    percent,
)
from .tasks import *
from .utils import *
from .utils import logging


# deprecated modules
from datasets import arrow_dataset as _arrow_dataset  # isort:skip
from datasets import utils as _utils  # isort:skip
from datasets.utils import download_manager as _deprecated_download_manager  # isort:skip

_lowercase    	=		concatenate_datasets
_lowercase    	=		DownloadConfig
_lowercase    	=		DownloadManager
_lowercase    	=		DownloadMode
_lowercase    	=		DownloadConfig
_lowercase    	=		DownloadMode
_lowercase    	=		DownloadManager

del _arrow_dataset, _utils, _deprecated_download_manager
