




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


_lowerCAmelCase  :      Optional[Any]						=      {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

_lowerCAmelCase  :      Union[str, Any]						=      {
    """vocab_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-generator""": """https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt""",
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-generator""": (
            """https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"""
        ),
    },
}

_lowerCAmelCase  :      List[str]						=      {
    """google/electra-small-generator""": 5_1_2,
    """google/electra-base-generator""": 5_1_2,
    """google/electra-large-generator""": 5_1_2,
    """google/electra-small-discriminator""": 5_1_2,
    """google/electra-base-discriminator""": 5_1_2,
    """google/electra-large-discriminator""": 5_1_2,
}

_lowerCAmelCase  :      Tuple						=      {
    """google/electra-small-generator""": {"""do_lower_case""": True},
    """google/electra-base-generator""": {"""do_lower_case""": True},
    """google/electra-large-generator""": {"""do_lower_case""": True},
    """google/electra-small-discriminator""": {"""do_lower_case""": True},
    """google/electra-base-discriminator""": {"""do_lower_case""": True},
    """google/electra-large-discriminator""": {"""do_lower_case""": True},
}
class 	_UpperCamelCase					( UpperCamelCase__						):
      UpperCAmelCase_						=						VOCAB_FILES_NAMES
      UpperCAmelCase_						=						PRETRAINED_VOCAB_FILES_MAP
      UpperCAmelCase_						=						PRETRAINED_INIT_CONFIGURATION
      UpperCAmelCase_						=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      UpperCAmelCase_						=						ElectraTokenizer
      def __init__(	self      :Tuple       ,					lowerCamelCase      :int=None       ,					lowerCamelCase      :List[str]=None       ,					lowerCamelCase      :List[str]=True       ,					lowerCamelCase      :str="[UNK]"       ,					lowerCamelCase      :Any="[SEP]"       ,					lowerCamelCase      :Any="[PAD]"       ,					lowerCamelCase      :Optional[Any]="[CLS]"       ,					lowerCamelCase      :List[Any]="[MASK]"       ,					lowerCamelCase      :Tuple=True       ,					lowerCamelCase      :Dict=None       ,					**lowerCamelCase      :Dict       ,					)   ->	List[str]:
       super().__init__(
           lowerCAmelCase__       ,					tokenizer_file=lowerCAmelCase__       ,					do_lower_case=lowerCAmelCase__       ,					unk_token=lowerCAmelCase__       ,					sep_token=lowerCAmelCase__       ,					pad_token=lowerCAmelCase__       ,					cls_token=lowerCAmelCase__       ,					mask_token=lowerCAmelCase__       ,					tokenize_chinese_chars=lowerCAmelCase__       ,					strip_accents=lowerCAmelCase__       ,					**lowerCAmelCase__       ,					)

       UpperCAmelCase__							       =  json.loads(self.backend_tokenizer.normalizer.__getstate__()					)
       if (
           normalizer_state.get("lowercase"       ,					lowerCAmelCase__					) != do_lower_case
           or normalizer_state.get("strip_accents"       ,					lowerCAmelCase__					) != strip_accents
           or normalizer_state.get("handle_chinese_chars"       ,					lowerCAmelCase__					) != tokenize_chinese_chars
       ):
        UpperCAmelCase__							       =  getattr(lowerCAmelCase__       ,					normalizer_state.pop("type"					)					)
        UpperCAmelCase__							       =  do_lower_case
        UpperCAmelCase__							       =  strip_accents
        UpperCAmelCase__							       =  tokenize_chinese_chars
        UpperCAmelCase__							       =  normalizer_class(**lowerCAmelCase__					)

       UpperCAmelCase__							       =  do_lower_case
      def 		UpperCAmelCase_  (	self      :Any       ,					lowerCamelCase      :int       ,					lowerCamelCase      :int=None					)   ->	Tuple:
       UpperCAmelCase__							       =  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
        output += token_ids_a + [self.sep_token_id]

       return output
      def 		UpperCAmelCase_  (	self      :Any       ,					lowerCamelCase      :List[Any]       ,					lowerCamelCase      :Tuple = None					)   ->	List[int]:
       UpperCAmelCase__							       =  [self.sep_token_id]
       UpperCAmelCase__							       =  [self.cls_token_id]
       if token_ids_a is None:
        return len(cls + token_ids_a + sep					) * [0]
       return len(cls + token_ids_a + sep					) * [0] + len(token_ids_a + sep					) * [1]







      def 		UpperCAmelCase_  (	self      :List[Any]       ,					lowerCamelCase      :int       ,					lowerCamelCase      :List[str] = None					)   ->	Tuple[str]:
       UpperCAmelCase__							       =  self._tokenizer.model.save(lowerCAmelCase__       ,					name=lowerCAmelCase__					)
       return tuple(lowerCAmelCase__					)












from typing import Optional

import pyspark

from .. import Features, NamedSplit
from ..download import DownloadMode
from ..packaged_modules.spark.spark import Spark
from .abc import AbstractDatasetReader







class 				__lowerCAmelCase       (    UpperCamelCase__):







      def __init__(    self		,      lowerCAmelCase__		,      lowerCAmelCase__ = None		,      lowerCAmelCase__ = None		,      lowerCAmelCase__ = True		,      lowerCAmelCase__ = None		,      lowerCAmelCase__ = False		,      lowerCAmelCase__ = None		,      lowerCAmelCase__ = True		,      lowerCAmelCase__ = "arrow"		,      **lowerCAmelCase__		,      )   ->						Optional[int]:



        '''simple docstring'''




        super().__init__(
            split=lowerCAmelCase__		,      features=lowerCAmelCase__		,      cache_dir=lowerCAmelCase__		,      keep_in_memory=lowerCAmelCase__		,      streaming=lowerCAmelCase__		,      **lowerCAmelCase__		,      )
        a__	:					int										=load_from_cache_file
        a__	:					Tuple										=file_format
        a__	:					List[Any]										=Spark(
            df=lowerCAmelCase__		,      features=lowerCAmelCase__		,      cache_dir=lowerCAmelCase__		,      working_dir=lowerCAmelCase__		,      **lowerCAmelCase__		,      )







      def    _lowercase  (    self							)   ->						str:



        '''simple docstring'''




        if self.streaming:
          return self.builder.as_streaming_dataset(split=self.split							)
        a__	:					str										=None if self._load_from_cache_file else DownloadMode.FORCE_REDOWNLOAD
        self.builder.download_and_prepare(
            download_mode=lowerCAmelCase__		,      file_format=self._file_format		,      )
        return self.builder.as_dataset(split=self.split							)






