


import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
						import sqlitea

						import sqlalchemy
class        _lowercase      (			A__       ):






				'''simple docstring'''



				def __init__(       self			:int			,			lowerCAmelCase__			:Union[str, "sqlalchemy.sql.Selectable"]			,			lowerCAmelCase__			:Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"]			,			lowerCAmelCase__			:Optional[Features] = None			,			lowerCAmelCase__			:str = None			,			lowerCAmelCase__			:bool = False			,			**lowerCAmelCase__			:Union[str, Any]			,			)		->			Optional[int]:
										super().__init__(features=lowerCAmelCase__			,			cache_dir=lowerCAmelCase__			,			keep_in_memory=lowerCAmelCase__			,			**lowerCAmelCase__	)
										__SCREAMING_SNAKE_CASE						:     Any					=		Sql(
										    cache_dir=lowerCAmelCase__			,			features=lowerCAmelCase__			,			sql=lowerCAmelCase__			,			con=lowerCAmelCase__			,			**lowerCAmelCase__			,			)



				def 		__magic_name__(       self			:List[Any]	)		->			Union[str, Any]:
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		None
										__SCREAMING_SNAKE_CASE						:     int					=		None
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		None
										__SCREAMING_SNAKE_CASE						:     Dict					=		None

										self.builder.download_and_prepare(
										    download_config=lowerCAmelCase__			,			download_mode=lowerCAmelCase__			,			verification_mode=lowerCAmelCase__			,			base_path=lowerCAmelCase__			,			)

										# Build dataset for splits
										__SCREAMING_SNAKE_CASE						:     List[Any]					=		self.builder.as_dataset(
										    split='''train'''			,			verification_mode=lowerCAmelCase__			,			in_memory=self.keep_in_memory	)
										return dataset




class        _lowercase      :






				'''simple docstring'''



				def __init__(       self			:int			,			lowerCAmelCase__			:Dataset			,			lowerCAmelCase__			:str			,			lowerCAmelCase__			:Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"]			,			lowerCAmelCase__			:Optional[int] = None			,			lowerCAmelCase__			:Optional[int] = None			,			**lowerCAmelCase__			:Union[str, Any]			,			)		->			int:
										if num_proc is not None and num_proc <= 0:
																raise ValueError(f'''num_proc {num_proc} must be an integer > 0.'''	)

										__SCREAMING_SNAKE_CASE						:     List[Any]					=		dataset
										__SCREAMING_SNAKE_CASE						:     Tuple					=		name
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		con
										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
										__SCREAMING_SNAKE_CASE						:     List[Any]					=		num_proc
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		to_sql_kwargs


				def 		__magic_name__(       self			:Tuple	)		->			int:
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		self.to_sql_kwargs.pop('''sql'''			,			lowerCAmelCase__	)
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		self.to_sql_kwargs.pop('''con'''			,			lowerCAmelCase__	)
										__SCREAMING_SNAKE_CASE						:     int					=		self.to_sql_kwargs.pop('''index'''			,			lowerCAmelCase__	)

										__SCREAMING_SNAKE_CASE						:     int					=		self._write(index=lowerCAmelCase__			,			**self.to_sql_kwargs	)
										return written


				def 		__magic_name__(       self			:Dict			,			lowerCAmelCase__			:List[Any]	)		->			Optional[Any]:
										__SCREAMING_SNAKE_CASE			,			__SCREAMING_SNAKE_CASE			,			__SCREAMING_SNAKE_CASE						:     List[str]					=		args
										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		{**to_sql_kwargs, '''if_exists''': '''append'''} if offset > 0 else to_sql_kwargs
										__SCREAMING_SNAKE_CASE						:     List[str]					=		query_table(
										    table=self.dataset.data			,			key=slice(lowerCAmelCase__			,			offset + self.batch_size	)			,			indices=self.dataset._indices			,			)
										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		batch.to_pandas()
										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		df.to_sql(self.name			,			self.con			,			index=lowerCAmelCase__			,			**lowerCAmelCase__	)
										return num_rows or len(lowerCAmelCase__	)



				def 		__magic_name__(       self			:str			,			lowerCAmelCase__			:str			,			**lowerCAmelCase__			:Tuple	)		->			int:
										__SCREAMING_SNAKE_CASE						:     str					=		0

										if self.num_proc is None or self.num_proc == 1:
																for offset in logging.tqdm(
																    range(0			,			len(self.dataset	)			,			self.batch_size	)			,			unit='''ba'''			,			disable=not logging.is_progress_bar_enabled()			,			desc='''Creating SQL from Arrow format'''			,			):
																						written += self._batch_sql((offset, index, to_sql_kwargs)	)
										else:
																__SCREAMING_SNAKE_CASE			,			__SCREAMING_SNAKE_CASE						:     Optional[int]					=		len(self.dataset	), self.batch_size
																with multiprocessing.Pool(self.num_proc	) as pool:
																						for num_rows in logging.tqdm(
																						    pool.imap(
																						        self._batch_sql			,			[(offset, index, to_sql_kwargs) for offset in range(0			,			lowerCAmelCase__			,			lowerCAmelCase__	)]			,			)			,			total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size			,			unit='''ba'''			,			disable=not logging.is_progress_bar_enabled()			,			desc='''Creating SQL from Arrow format'''			,			):
																												written += num_rows

										return written






from typing import Dict, List, Optional

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_A     =			logging.get_logger(__name__)


_A     =			{
    '''nielsr/canine-s''': 2_048,
}

# Unicode defines 1,114,112 total “codepoints”
_A     =			1_114_112

# Below: Constants defining canonical codepoints for special, pseudo-characters.
# Copied from https://github.com/google-research/language/blob/master/language/canine/special_codepoints.py
_A     =			0

_A     =			0xe0_00
_A     =			0xe0_01
_A     =			0xe0_02
_A     =			0xe0_03
_A     =			0xe0_04

# Maps special codepoints to human-readable names.
_A     =			{
    # Special symbols are represented using codepoints values that are valid,
    # but designated as "Private Use", meaning that they will never be assigned
    # characters by the Unicode Consortium, and are thus safe for use here.
    #
    # NOTE: Do *NOT* add any sort of [UNK_CHAR] here. They are explicitly
    # excluded and should fail with a hard error.
    CLS: "[CLS]",
    SEP: "[SEP]",
    BOS: "[BOS]",
    MASK: "[MASK]",
    PAD: "[PAD]",
    RESERVED: "[RESERVED]",
}

# Maps special codepoint human-readable names to their codepoint values.
_A     =			{name: codepoint for codepoint, name in SPECIAL_CODEPOINTS.items()}


class  A		(  __UpperCAmelCase     ):
		__snake_case									=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



		def __init__(     self,	UpperCamelCase__=chr(UpperCamelCase__   ),	UpperCamelCase__=chr(UpperCamelCase__   ),	UpperCamelCase__=chr(UpperCamelCase__   ),	UpperCamelCase__=chr(UpperCamelCase__   ),	UpperCamelCase__=chr(UpperCamelCase__   ),	UpperCamelCase__=chr(UpperCamelCase__   ),	UpperCamelCase__=False,	UpperCamelCase__=2048,	**UpperCamelCase__,	):







								"""simple docstring"""



								lowerCAmelCase_      			=  AddedToken(UpperCamelCase__,	lstrip=UpperCamelCase__,	rstrip=UpperCamelCase__   ) if isinstance(UpperCamelCase__,	UpperCamelCase__   ) else bos_token
								lowerCAmelCase_      			=  AddedToken(UpperCamelCase__,	lstrip=UpperCamelCase__,	rstrip=UpperCamelCase__   ) if isinstance(UpperCamelCase__,	UpperCamelCase__   ) else eos_token
								lowerCAmelCase_      			=  AddedToken(UpperCamelCase__,	lstrip=UpperCamelCase__,	rstrip=UpperCamelCase__   ) if isinstance(UpperCamelCase__,	UpperCamelCase__   ) else sep_token
								lowerCAmelCase_      			=  AddedToken(UpperCamelCase__,	lstrip=UpperCamelCase__,	rstrip=UpperCamelCase__   ) if isinstance(UpperCamelCase__,	UpperCamelCase__   ) else cls_token
								lowerCAmelCase_      			=  AddedToken(UpperCamelCase__,	lstrip=UpperCamelCase__,	rstrip=UpperCamelCase__   ) if isinstance(UpperCamelCase__,	UpperCamelCase__   ) else pad_token

								# Mask token behave like a normal word, i.e. include the space before it
								lowerCAmelCase_      			=  AddedToken(UpperCamelCase__,	lstrip=UpperCamelCase__,	rstrip=UpperCamelCase__   ) if isinstance(UpperCamelCase__,	UpperCamelCase__   ) else mask_token

								super().__init__(
								    bos_token=UpperCamelCase__,	eos_token=UpperCamelCase__,	sep_token=UpperCamelCase__,	cls_token=UpperCamelCase__,	pad_token=UpperCamelCase__,	mask_token=UpperCamelCase__,	add_prefix_space=UpperCamelCase__,	model_max_length=UpperCamelCase__,	**UpperCamelCase__,	)

								# Creates a mapping for looking up the IDs of special symbols.
								lowerCAmelCase_      			=  {}
								for codepoint, name in SPECIAL_CODEPOINTS.items():
														lowerCAmelCase_      			=  codepoint

								# Creates a mapping for looking up the string forms of special symbol IDs.
								lowerCAmelCase_      			=  {
								    codepoint: name for name, codepoint in self._special_codepoints.items()
								}

								lowerCAmelCase_      			=  UNICODE_VOCAB_SIZE
								lowerCAmelCase_      			=  len(self._special_codepoints   )



		@property
		def        SCREAMING_SNAKE_CASE__			(     self   ):







								"""simple docstring"""



								return self._unicode_vocab_size



		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__   ):







								"""simple docstring"""



								return list(UpperCamelCase__   )



		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__   ):







								"""simple docstring"""



								try:
														return ord(UpperCamelCase__   )
								except TypeError:
														raise ValueError(f"invalid token: '{token}'"   )



		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__   ):







								"""simple docstring"""



								try:
														if index in SPECIAL_CODEPOINTS:
																				return SPECIAL_CODEPOINTS[index]
														return chr(UpperCamelCase__   )
								except TypeError:
														raise ValueError(f"invalid id: {index}"   )



		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__   ):







								"""simple docstring"""



								return "".join(UpperCamelCase__   )



		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__ = None   ):







								"""simple docstring"""



								lowerCAmelCase_      			=  [self.sep_token_id]
								lowerCAmelCase_      			=  [self.cls_token_id]

								lowerCAmelCase_      			=  cls + token_ids_a + sep
								if token_ids_a is not None:
														result += token_ids_a + sep
								return result



		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__ = None,	UpperCamelCase__ = False   ):







								"""simple docstring"""



								if already_has_special_tokens:
														return super().get_special_tokens_mask(
														    token_ids_a=UpperCamelCase__,	token_ids_a=UpperCamelCase__,	already_has_special_tokens=UpperCamelCase__   )

								lowerCAmelCase_      			=  [1] + ([0] * len(UpperCamelCase__   )) + [1]
								if token_ids_a is not None:
														result += ([0] * len(UpperCamelCase__   )) + [1]
								return result



		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__ = None   ):







								"""simple docstring"""



								lowerCAmelCase_      			=  [self.sep_token_id]
								lowerCAmelCase_      			=  [self.cls_token_id]

								lowerCAmelCase_      			=  len(cls + token_ids_a + sep   ) * [0]
								if token_ids_a is not None:
														result += len(token_ids_a + sep   ) * [1]
								return result







		def        SCREAMING_SNAKE_CASE__			(     self,	UpperCamelCase__,	UpperCamelCase__ = None   ):







								"""simple docstring"""



								return ()




