
"""simple docstring"""





import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


__magic_name__       =			datasets.utils.logging.get_logger(__name__)
@dataclass
class 						SCREAMING_SNAKE_CASE_ (						datasets.BuilderConfig ):




		"""simple docstring"""





		__lowercase							:  int						   =		10000
		__lowercase							:  Optional[List[str]]						   =		None
		__lowercase							:  Optional[datasets.Features]						   =		None



class 						SCREAMING_SNAKE_CASE_ (						datasets.ArrowBasedBuilder ):




		"""simple docstring"""





		__lowercase							:  Tuple						   =		ParquetConfig




		def 	snake_case_     (     self):
				return datasets.DatasetInfo(features=self.config.features)




		def 	snake_case_     (     self							,  lowerCAmelCase__):
				if not self.config.data_files:
						raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
				__SCREAMING_SNAKE_CASE            = dl_manager.download_and_extract(self.config.data_files)
				if isinstance(A__							,  (str, list, tuple)):
						__SCREAMING_SNAKE_CASE            = data_files
						if isinstance(A__							,  A__):
								__SCREAMING_SNAKE_CASE            = [files]
						# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
						__SCREAMING_SNAKE_CASE            = [dl_manager.iter_files(A__) for file in files]
						return [datasets.SplitGenerator(name=datasets.Split.TRAIN							,  gen_kwargs={"""files""": files})]
				__SCREAMING_SNAKE_CASE            = []
				for split_name, files in data_files.items():
						if isinstance(A__							,  A__):
								__SCREAMING_SNAKE_CASE            = [files]
						# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
						__SCREAMING_SNAKE_CASE            = [dl_manager.iter_files(A__) for file in files]
						# Infer features is they are stoed in the arrow schema
						if self.info.features is None:
								for file in itertools.chain.from_iterable(A__):
										with open(A__							,  """rb""") as f:
												__SCREAMING_SNAKE_CASE            = datasets.Features.from_arrow_schema(pq.read_schema(A__))
										break
						splits.append(datasets.SplitGenerator(name=A__							,  gen_kwargs={"""files""": files}))
				return splits




		def 	snake_case_     (     self							,  lowerCAmelCase__):
				if self.info.features is not None:
						# more expensive cast to support nested features with keys in a different order
						# allows str <-> int/float or str to Audio for example
						__SCREAMING_SNAKE_CASE            = table_cast(A__							,  self.info.features.arrow_schema)
				return pa_table




		def 	snake_case_     (     self							,  lowerCAmelCase__):
				__SCREAMING_SNAKE_CASE            = self.info.features.arrow_schema if self.info.features is not None else None
				if self.info.features is not None and self.config.columns is not None:
						if sorted(field.name for field in schema) != sorted(self.config.columns):
								raise ValueError(
								    f"Tried to load parquet data with columns \'{self.config.columns}\' with mismatching features \'{self.info.features}\'")
				for file_idx, file in enumerate(itertools.chain.from_iterable(A__)):
						with open(A__							,  """rb""") as f:
								__SCREAMING_SNAKE_CASE            = pq.ParquetFile(A__)
								try:
										for batch_idx, record_batch in enumerate(
										    parquet_file.iter_batches(batch_size=self.config.batch_size							,  columns=self.config.columns)):
												__SCREAMING_SNAKE_CASE            = pa.Table.from_batches([record_batch])
												# Uncomment for debugging (will print the Arrow table size and elements)
												# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
												# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
												yield f"{file_idx}_{batch_idx}", self._cast_table(A__)
								except ValueError as e:
										logger.error(f"Failed to read file \'{file}\' with error {type(A__)}: {e}")
										raise






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


lowercase__		:List[Any]   =    logging.get_logger(__name__)

lowercase__		:Optional[Any]   =    {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

lowercase__		:Union[str, Any]   =    {
    "vocab_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"
        ),
        "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt",
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"
        ),
        "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt",
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json"
        ),
        "bert-base-multilingual-cased": (
            "https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-cased": (
            "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json"
        ),
    },
}

lowercase__		:str   =    {
    "bert-base-uncased": 512,
    "bert-large-uncased": 512,
    "bert-base-cased": 512,
    "bert-large-cased": 512,
    "bert-base-multilingual-uncased": 512,
    "bert-base-multilingual-cased": 512,
    "bert-base-chinese": 512,
    "bert-base-german-cased": 512,
    "bert-large-uncased-whole-word-masking": 512,
    "bert-large-cased-whole-word-masking": 512,
    "bert-large-uncased-whole-word-masking-finetuned-squad": 512,
    "bert-large-cased-whole-word-masking-finetuned-squad": 512,
    "bert-base-cased-finetuned-mrpc": 512,
    "bert-base-german-dbmdz-cased": 512,
    "bert-base-german-dbmdz-uncased": 512,
    "TurkuNLP/bert-base-finnish-cased-v1": 512,
    "TurkuNLP/bert-base-finnish-uncased-v1": 512,
    "wietsedv/bert-base-dutch-cased": 512,
}

lowercase__		:Dict   =    {
    "bert-base-uncased": {"do_lower_case": True},
    "bert-large-uncased": {"do_lower_case": True},
    "bert-base-cased": {"do_lower_case": False},
    "bert-large-cased": {"do_lower_case": False},
    "bert-base-multilingual-uncased": {"do_lower_case": True},
    "bert-base-multilingual-cased": {"do_lower_case": False},
    "bert-base-chinese": {"do_lower_case": False},
    "bert-base-german-cased": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking-finetuned-squad": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking-finetuned-squad": {"do_lower_case": False},
    "bert-base-cased-finetuned-mrpc": {"do_lower_case": False},
    "bert-base-german-dbmdz-cased": {"do_lower_case": False},
    "bert-base-german-dbmdz-uncased": {"do_lower_case": True},
    "TurkuNLP/bert-base-finnish-cased-v1": {"do_lower_case": False},
    "TurkuNLP/bert-base-finnish-uncased-v1": {"do_lower_case": True},
    "wietsedv/bert-base-dutch-cased": {"do_lower_case": False},
}







class 				lowercase		(					SCREAMING_SNAKE_CASE__		):
							lowercase_       :       str       					=VOCAB_FILES_NAMES
							lowercase_       :       Dict       					=PRETRAINED_VOCAB_FILES_MAP
							lowercase_       :       Optional[int]       					=PRETRAINED_INIT_CONFIGURATION
							lowercase_       :       Tuple       					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_       :       Any       					=BertTokenizer




							def __init__(				self   ,A__=None   ,A__=None   ,A__=True   ,A__="[UNK]"   ,A__="[SEP]"   ,A__="[PAD]"   ,A__="[CLS]"   ,A__="[MASK]"   ,A__=True   ,A__=None   ,**A__   ,):
												super().__init__(
												    A__   ,tokenizer_file=A__   ,do_lower_case=A__   ,unk_token=A__   ,sep_token=A__   ,pad_token=A__   ,cls_token=A__   ,mask_token=A__   ,tokenize_chinese_chars=A__   ,strip_accents=A__   ,**A__   ,)

												lowercase							=						json.loads(self.backend_tokenizer.normalizer.__getstate__())
												if (
												    normalizer_state.get('''lowercase'''   ,A__) != do_lower_case
												    or normalizer_state.get('''strip_accents'''   ,A__) != strip_accents
												    or normalizer_state.get('''handle_chinese_chars'''   ,A__) != tokenize_chinese_chars
												):
																	lowercase							=						getattr(A__   ,normalizer_state.pop('''type'''))
																	lowercase							=						do_lower_case
																	lowercase							=						strip_accents
																	lowercase							=						tokenize_chinese_chars
																	lowercase							=						normalizer_class(**A__)

												lowercase							=						do_lower_case




							def  A__      (				self   ,A__   ,A__=None):
												lowercase							=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

												if token_ids_a:
																	output += token_ids_a + [self.sep_token_id]

												return output




							def  A__      (				self   ,A__   ,A__ = None):
												lowercase							=						[self.sep_token_id]
												lowercase							=						[self.cls_token_id]
												if token_ids_a is None:
																	return len(cls + token_ids_a + sep) * [0]
												return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]





							def  A__      (				self   ,A__   ,A__ = None):
												lowercase							=						self._tokenizer.model.save(A__   ,name=A__)
												return tuple(A__)




