





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


__UpperCAmelCase       :   List[Any]				       =							{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

__UpperCAmelCase       :   Optional[Any]				       =							{
    "vocab_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"
        ),
        "google/electra-base-generator": "https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt",
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-generator": (
            "https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"
        ),
    },
}

__UpperCAmelCase       :   Tuple				       =							{
    "google/electra-small-generator": 512,
    "google/electra-base-generator": 512,
    "google/electra-large-generator": 512,
    "google/electra-small-discriminator": 512,
    "google/electra-base-discriminator": 512,
    "google/electra-large-discriminator": 512,
}

__UpperCAmelCase       :   int				       =							{
    "google/electra-small-generator": {"do_lower_case": True},
    "google/electra-base-generator": {"do_lower_case": True},
    "google/electra-large-generator": {"do_lower_case": True},
    "google/electra-small-discriminator": {"do_lower_case": True},
    "google/electra-base-discriminator": {"do_lower_case": True},
    "google/electra-large-discriminator": {"do_lower_case": True},
}







class     __snake_case       (							__lowerCamelCase			):





       '''simple docstring'''







       lowerCAmelCase__      		=     VOCAB_FILES_NAMES
       lowerCAmelCase__      		=     PRETRAINED_VOCAB_FILES_MAP
       lowerCAmelCase__      		=     PRETRAINED_INIT_CONFIGURATION
       lowerCAmelCase__      		=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       lowerCAmelCase__      		=     ElectraTokenizer




       def __init__(  self :    Tuple							,				A :    Union[str, Any]=None							,				A :    List[Any]=None							,				A :    List[Any]=True							,				A :    str="[UNK]"							,				A :    Dict="[SEP]"							,				A :    Tuple="[PAD]"							,				A :    str="[CLS]"							,				A :    Tuple="[MASK]"							,				A :    Union[str, Any]=True							,				A :    List[str]=None							,				**A :    Optional[Any]							,				):
         super().__init__(
             A							,				tokenizer_file=A							,				do_lower_case=A							,				unk_token=A							,				sep_token=A							,				pad_token=A							,				cls_token=A							,				mask_token=A							,				tokenize_chinese_chars=A							,				strip_accents=A							,				**A							,				)

         __snake_case:						List[str]   =						json.loads(self.backend_tokenizer.normalizer.__getstate__()		)
         if (
             normalizer_state.get("""lowercase"""							,				A		) != do_lower_case
             or normalizer_state.get("""strip_accents"""							,				A		) != strip_accents
             or normalizer_state.get("""handle_chinese_chars"""							,				A		) != tokenize_chinese_chars
         ):
           __snake_case:						List[Any]   =						getattr(A							,				normalizer_state.pop("""type"""		)		)
           __snake_case:						Dict   =						do_lower_case
           __snake_case:						Optional[Any]   =						strip_accents
           __snake_case:						Dict   =						tokenize_chinese_chars
           __snake_case:						str   =						normalizer_class(**A		)

         __snake_case:						List[str]   =						do_lower_case




       def 							UpperCAmelCase__ (  self :    Union[str, Any]							,				A :    Optional[Any]							,				A :    Optional[Any]=None		):
         __snake_case:						Optional[Any]   =						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
           output += token_ids_a + [self.sep_token_id]

         return output




       def 							UpperCAmelCase__ (  self :    Union[str, Any]							,				A :    List[int]							,				A :    Optional[List[int]] = None		):
         __snake_case:						List[str]   =						[self.sep_token_id]
         __snake_case:						str   =						[self.cls_token_id]
         if token_ids_a is None:
           return len(cls + token_ids_a + sep		) * [0]
         return len(cls + token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1]






       def 							UpperCAmelCase__ (  self :    Optional[int]							,				A :    str							,				A :    Optional[str] = None		):
         __snake_case:						Optional[int]   =						self._tokenizer.model.save(A							,				name=A		)
         return tuple(A		)













import pytest


__UpperCAmelCase       :   Optional[Any]				       =							"__dummy_dataset1__"

__UpperCAmelCase       :   List[str]				       =							"\nimport json\nimport os\n\nimport datasets\n\n\nREPO_URL = \"https://huggingface.co/datasets/albertvillanova/tests-raw-jsonl/resolve/main/\"\nURLS = {\"train\": REPO_URL + \"wikiann-bn-train.jsonl\", \"validation\": REPO_URL + \"wikiann-bn-validation.jsonl\"}\n\n\nclass __DummyDataset1__(datasets.GeneratorBasedBuilder):\n\n    def _info(self):\n        features = datasets.Features(\n            {\n                \"tokens\": datasets.Sequence(datasets.Value(\"string\")),\n                \"ner_tags\": datasets.Sequence(\n                    datasets.features.ClassLabel(\n                        names=[\n                            \"O\",\n                            \"B-PER\",\n                            \"I-PER\",\n                            \"B-ORG\",\n                            \"I-ORG\",\n                            \"B-LOC\",\n                            \"I-LOC\",\n                        ]\n                    )\n                ),\n                \"langs\": datasets.Sequence(datasets.Value(\"string\")),\n                \"spans\": datasets.Sequence(datasets.Value(\"string\")),\n            }\n        )\n        return datasets.DatasetInfo(features=features)\n\n    def _split_generators(self, dl_manager):\n        dl_path = dl_manager.download(URLS)\n        return [\n            datasets.SplitGenerator(datasets.Split.TRAIN, gen_kwargs={\"filepath\": dl_path[\"train\"]}),\n            datasets.SplitGenerator(datasets.Split.VALIDATION, gen_kwargs={\"filepath\": dl_path[\"validation\"]}),\n        ]\n\n    def _generate_examples(self, filepath):\n        with open(filepath, \"r\", encoding=\"utf-8\") as f:\n            for i, line in enumerate(f):\n                yield i, json.loads(line)\n"







@pytest.fixture
def 			A__							(    )   ->    Optional[int]:
  return DATASET_LOADING_SCRIPT_NAME







@pytest.fixture
def 			A__							(    )   ->    Tuple:
  return DATASET_LOADING_SCRIPT_CODE







@pytest.fixture
def 			A__							(    SCREAMING_SNAKE_CASE__					,					SCREAMING_SNAKE_CASE__					,					SCREAMING_SNAKE_CASE__)   ->    Tuple:
  __snake_case:						List[Any]   =						dataset_loading_script_name
  __snake_case:						Any   =						tmp_path / """datasets""" / script_name
  script_dir.mkdir(parents=SCREAMING_SNAKE_CASE__)
  __snake_case:						int   =						script_dir / F'''{script_name}.py'''
  with open(SCREAMING_SNAKE_CASE__					,					"""w""") as f:
    f.write(SCREAMING_SNAKE_CASE__)
  return str(SCREAMING_SNAKE_CASE__)






