


import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


_lowercase							:   Union[str, Any]						=						datasets.utils.logging.get_logger(__name__)






@dataclass
class    _UpperCamelCase     (       datasets.BuilderConfig					):

				"""simple docstring"""







				lowerCAmelCase					=			None
				lowerCAmelCase					=			"utf-8"
				lowerCAmelCase					=			None
				lowerCAmelCase					=			None
				lowerCAmelCase					=			True  # deprecated
				lowerCAmelCase					=			None  # deprecated
				lowerCAmelCase					=			1_0 << 2_0  # 10MB
				lowerCAmelCase					=			None






class    _UpperCamelCase     (       datasets.ArrowBasedBuilder					):

				"""simple docstring"""







				lowerCAmelCase					=			JsonConfig

				def   _UpperCAmelCase       (    self )						->					List[str]:
					if self.config.block_size is not None:
						logger.warning("""The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead""" )
						A             =					self.config.block_size
					if self.config.use_threads is not True:
						logger.warning(
						    """The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore.""" )
					if self.config.newlines_in_values is not None:
						raise ValueError("""The JSON loader parameter `newlines_in_values` is no longer supported""" )
					return datasets.DatasetInfo(features=self.config.features )

				def   _UpperCAmelCase       (    self					, a__ )						->					Dict:
					if not self.config.data_files:
						raise ValueError(f'At least one data file must be specified, but got data_files={self.config.data_files}' )
					A             =					dl_manager.download_and_extract(self.config.data_files )
					if isinstance(a__					, (str, list, tuple) ):
						A             =					data_files
						if isinstance(a__					, a__ ):
							A             =					[files]
						A             =					[dl_manager.iter_files(a__ ) for file in files]
						return [datasets.SplitGenerator(name=datasets.Split.TRAIN					, gen_kwargs={"""files""": files} )]
					A             =					[]
					for split_name, files in data_files.items():
						if isinstance(a__					, a__ ):
							A             =					[files]
						A             =					[dl_manager.iter_files(a__ ) for file in files]
						splits.append(datasets.SplitGenerator(name=a__					, gen_kwargs={"""files""": files} ) )
					return splits

				def   _UpperCAmelCase       (    self					, a__ )						->					pa.Table:
					if self.config.features is not None:
						# adding missing columns
						for column_name in set(self.config.features ) - set(pa_table.column_names ):
							A             =					self.config.features.arrow_schema.field(a__ ).type
							A             =					pa_table.append_column(a__					, pa.array([None] * len(a__ )					, type=a__ ) )
						# more expensive cast to support nested structures with keys in a different order
						# allows str <-> int/float or str to Audio for example
						A             =					table_cast(a__					, self.config.features.arrow_schema )
					return pa_table

				def   _UpperCAmelCase       (    self					, a__ )						->					Tuple:
					for file_idx, file in enumerate(itertools.chain.from_iterable(a__ ) ):
						# If the file is one json object and if we need to look at the list of items in one specific field
						if self.config.field is not None:
							with open(a__					, encoding=self.config.encoding					, errors=self.config.encoding_errors ) as f:
								A             =					json.load(a__ )

							# We keep only the field we are interested in
							A             =					dataset[self.config.field]

							# We accept two format: a list of dicts or a dict of lists
							if isinstance(a__					, (list, tuple) ):
								A             =					set().union(*[row.keys() for row in dataset] )
								A             =					{col: [row.get(a__ ) for row in dataset] for col in keys}
							else:
								A             =					dataset
							A             =					pa.Table.from_pydict(a__ )
							yield file_idx, self._cast_table(a__ )

						# If the file has one json object per line
						else:
							with open(a__					, """rb""" ) as f:
								A             =					0
								# Use block_size equal to the chunk size divided by 32 to leverage multithreading
								# Set a default minimum value of 16kB if the chunk size is really small
								A             =					max(self.config.chunksize // 32					, 16 << 10 )
								A             =					(
								    self.config.encoding_errors if self.config.encoding_errors is not None else """strict"""
								)
								while True:
									A             =					f.read(self.config.chunksize )
									if not batch:
										break
									# Finish current line
									try:
										batch += f.readline()
									except (AttributeError, io.UnsupportedOperation):
										batch += readline(a__ )
									# PyArrow only accepts utf-8 encoded bytes
									if self.config.encoding != "utf-8":
										A             =					batch.decode(self.config.encoding					, errors=a__ ).encode("""utf-8""" )
									try:
										while True:
											try:
												A             =					paj.read_json(
												    io.BytesIO(a__ )					, read_options=paj.ReadOptions(block_size=a__ ) )
												break
											except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
												if (
												    isinstance(a__					, pa.ArrowInvalid )
												    and "straddling" not in str(a__ )
												    or block_size > len(a__ )
												):
													raise
												else:
													# Increase the block size in case it was too small.
													# The block size will be reset for the next file.
													logger.debug(
													    f'Batch of {len(a__ )} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.' )
													block_size *= 2
									except pa.ArrowInvalid as e:
										try:
											with open(
											    a__					, encoding=self.config.encoding					, errors=self.config.encoding_errors ) as f:
												A             =					json.load(a__ )
										except json.JSONDecodeError:
											logger.error(f'Failed to read file \'{file}\' with error {type(a__ )}: {e}' )
											raise e
										# If possible, parse the file as a list of json objects and exit the loop
										if isinstance(a__					, a__ ):  # list is the only sequence type supported in JSON
											try:
												A             =					set().union(*[row.keys() for row in dataset] )
												A             =					{col: [row.get(a__ ) for row in dataset] for col in keys}
												A             =					pa.Table.from_pydict(a__ )
											except (pa.ArrowInvalid, AttributeError) as e:
												logger.error(f'Failed to read file \'{file}\' with error {type(a__ )}: {e}' )
												raise ValueError(f'Not able to read records in the JSON file at {file}.' ) from None
											yield file_idx, self._cast_table(a__ )
											break
										else:
											logger.error(f'Failed to read file \'{file}\' with error {type(a__ )}: {e}' )
											raise ValueError(
											    f'Not able to read records in the JSON file at {file}. '
											    f'You should probably indicate the field of the JSON file containing your records. '
											    f'This JSON file contain the following fields: {str(list(dataset.keys() ) )}. '
											    f'Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. ' ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
									yield (file_idx, batch_idx), self._cast_table(a__ )
									batch_idx += 1







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
				from .tokenization_albert import AlbertTokenizer
else:
				_lowercase							:   List[str]						=						None

_lowercase							:   List[str]						=						logging.get_logger(__name__)
_lowercase							:   List[Any]						=						{"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

_lowercase							:   str						=						{
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    },
    "tokenizer_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json",
    },
}

_lowercase							:   List[str]						=						{
    "albert-base-v1": 512,
    "albert-large-v1": 512,
    "albert-xlarge-v1": 512,
    "albert-xxlarge-v1": 512,
    "albert-base-v2": 512,
    "albert-large-v2": 512,
    "albert-xlarge-v2": 512,
    "albert-xxlarge-v2": 512,
}

_lowercase							:   Dict						=						"▁"






class    _UpperCamelCase     (       __snake_case					):

				"""simple docstring"""







				lowerCAmelCase					=			VOCAB_FILES_NAMES
				lowerCAmelCase					=			PRETRAINED_VOCAB_FILES_MAP
				lowerCAmelCase					=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCAmelCase					=			AlbertTokenizer

				def __init__(    self					, a__=None					, a__=None					, a__=True					, a__=True					, a__=False					, a__="[CLS]"					, a__="[SEP]"					, a__="<unk>"					, a__="[SEP]"					, a__="<pad>"					, a__="[CLS]"					, a__="[MASK]"					, **a__					, )						->					Dict:
					# Mask token behave like a normal word, i.e. include the space before it and
					# is included in the raw text, there should be a match in a non-normalized sentence.
					A             =					(
					    AddedToken(a__					, lstrip=a__					, rstrip=a__					, normalized=a__ )
					    if isinstance(a__					, a__ )
					    else mask_token
					)

					super().__init__(
					    a__					, tokenizer_file=a__					, do_lower_case=a__					, remove_space=a__					, keep_accents=a__					, bos_token=a__					, eos_token=a__					, unk_token=a__					, sep_token=a__					, pad_token=a__					, cls_token=a__					, mask_token=a__					, **a__					, )

					A             =					do_lower_case
					A             =					remove_space
					A             =					keep_accents
					A             =					vocab_file
					A             =					False if not self.vocab_file else True

				def   _UpperCAmelCase       (    self					, a__					, a__ = None )						->					List[int]:
					A             =					[self.sep_token_id]
					A             =					[self.cls_token_id]
					if token_ids_a is None:
						return cls + token_ids_a + sep
					return cls + token_ids_a + sep + token_ids_a + sep

				def   _UpperCAmelCase       (    self					, a__					, a__ = None )						->					List[int]:
					A             =					[self.sep_token_id]
					A             =					[self.cls_token_id]

					if token_ids_a is None:
						return len(cls + token_ids_a + sep ) * [0]
					return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]

				def   _UpperCAmelCase       (    self					, a__					, a__ = None )						->					Tuple[str]:
					if not self.can_save_slow_tokenizer:
						raise ValueError(
						    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
						    """tokenizer.""" )

					if not os.path.isdir(a__ ):
						logger.error(f'Vocabulary path ({save_directory}) should be a directory' )
						return
					A             =					os.path.join(
					    a__					, (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )

					if os.path.abspath(self.vocab_file ) != os.path.abspath(a__ ):
						copyfile(self.vocab_file					, a__ )

					return (out_vocab_file,)



