


import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
						import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__lowerCAmelCase	: Dict  				=logging.get_logger(__name__)

__lowerCAmelCase	: Dict  				={'vocab_file': 'vocab.txt'}

__lowerCAmelCase	: List[Any]  				={
    'vocab_file': {
        'openbmb/cpm-ant-10b': 'https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt',
    },
}

__lowerCAmelCase	: Any  				={
    'openbmb/cpm-ant-10b': 1_0_2_4,
}


def        _UpperCamelCase							(      lowercase__      ):
						__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		collections.OrderedDict()
						with open(lowercase__    ,       '''r'''    ,       encoding='''utf-8'''      ) as reader:
												__SCREAMING_SNAKE_CASE						:     Optional[int]					=		reader.readlines()
						for index, token in enumerate(lowercase__      ):
												__SCREAMING_SNAKE_CASE						:     int					=		token.rstrip('''\n'''      )
												__SCREAMING_SNAKE_CASE						:     Tuple					=		index
						return vocab
class        _lowercase      (			A__       ):






				'''simple docstring'''



				def __init__(       self			:List[Any]			,			lowerCAmelCase__			:List[str]			,			lowerCAmelCase__			:List[Any]="<unk>"			,			lowerCAmelCase__			:str=200	)		->			Optional[int]:
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		vocab
										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		unk_token
										__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		max_input_chars_per_word



				def 		__magic_name__(       self			:str			,			lowerCAmelCase__			:Tuple	)		->			Dict:
										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		list(lowerCAmelCase__	)
										if len(lowerCAmelCase__	) > self.max_input_chars_per_word:
																return [self.unk_token]

										__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		0
										__SCREAMING_SNAKE_CASE						:     int					=		[]
										while start < len(lowerCAmelCase__	):
																__SCREAMING_SNAKE_CASE						:     Optional[int]					=		len(lowerCAmelCase__	)
																__SCREAMING_SNAKE_CASE						:     int					=		None
																while start < end:
																						__SCREAMING_SNAKE_CASE						:     List[str]					=		''''''.join(chars[start:end]	)
																						if substr in self.vocab:
																												__SCREAMING_SNAKE_CASE						:     Optional[int]					=		substr
																												break
																						end -= 1
																if cur_substr is None:
																						sub_tokens.append(self.unk_token	)
																						start += 1
																else:
																						sub_tokens.append(lowerCAmelCase__	)
																						__SCREAMING_SNAKE_CASE						:     Optional[int]					=		end

										return sub_tokens




class        _lowercase      (			A__       ):






				'''simple docstring'''



				SCREAMING_SNAKE_CASE__				:			Tuple						=		VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE__				:			Any						=		PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE__				:			Tuple						=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE__				:			List[Any]						=		['''input_ids''', '''attention_mask''']
				SCREAMING_SNAKE_CASE__				:			Any						=		False


				def __init__(       self			:Tuple			,			lowerCAmelCase__			:int			,			lowerCAmelCase__			:int="<d>"			,			lowerCAmelCase__			:int="</d>"			,			lowerCAmelCase__			:Optional[Any]="<s>"			,			lowerCAmelCase__			:Optional[Any]="</s>"			,			lowerCAmelCase__			:Optional[int]="<pad>"			,			lowerCAmelCase__			:Dict="<unk>"			,			lowerCAmelCase__			:Any="</n>"			,			lowerCAmelCase__			:List[str]="</_>"			,			lowerCAmelCase__			:Optional[Any]="left"			,			**lowerCAmelCase__			:Tuple			,			)		->			Optional[Any]:
										requires_backends(self			,			['''jieba''']	)
										super().__init__(
										    bod_token=lowerCAmelCase__			,			eod_token=lowerCAmelCase__			,			bos_token=lowerCAmelCase__			,			eos_token=lowerCAmelCase__			,			pad_token=lowerCAmelCase__			,			unk_token=lowerCAmelCase__			,			line_token=lowerCAmelCase__			,			space_token=lowerCAmelCase__			,			padding_side=lowerCAmelCase__			,			**lowerCAmelCase__			,			)
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		bod_token
										__SCREAMING_SNAKE_CASE						:     List[str]					=		eod_token
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		load_vocab(lowerCAmelCase__	)
										__SCREAMING_SNAKE_CASE						:     int					=		self.encoder[space_token]
										__SCREAMING_SNAKE_CASE						:     str					=		self.encoder[line_token]

										del self.encoder[space_token]
										del self.encoder[line_token]

										__SCREAMING_SNAKE_CASE						:     Dict					=		collections.OrderedDict(sorted(self.encoder.items()			,			key=lambda lowerCAmelCase__	: x[1]	)	)
										__SCREAMING_SNAKE_CASE						:     Tuple					=		{v: k for k, v in self.encoder.items()}

										__SCREAMING_SNAKE_CASE						:     Any					=		WordpieceTokenizer(vocab=self.encoder			,			unk_token=self.unk_token	)


				@property
				def 		__magic_name__(       self			:List[Any]	)		->			int:
										return self.encoder[self.bod_token]


				@property
				def 		__magic_name__(       self			:int	)		->			List[Any]:
										return self.encoder[self.eod_token]


				@property
				def 		__magic_name__(       self			:List[Any]	)		->			Union[str, Any]:
										return self.encoder["\n"]


				@property
				def 		__magic_name__(       self			:Optional[int]	)		->			int:
										return len(self.encoder	)


				def 		__magic_name__(       self			:int	)		->			Optional[int]:
										return dict(self.encoder			,			**self.added_tokens_encoder	)


				def 		__magic_name__(       self			:str			,			lowerCAmelCase__			:List[str]	)		->			Any:
										__SCREAMING_SNAKE_CASE						:     int					=		[]
										for x in jieba.cut(lowerCAmelCase__			,			cut_all=lowerCAmelCase__	):
																output_tokens.extend(self.wordpiece_tokenizer.tokenize(lowerCAmelCase__	)	)
										return output_tokens


				def 		__magic_name__(       self			:str			,			lowerCAmelCase__			:Tuple			,			**lowerCAmelCase__			:Optional[int]	)		->			List[str]:
										__SCREAMING_SNAKE_CASE						:     Any					=		[i for i in token_ids if i >= 0]
										__SCREAMING_SNAKE_CASE						:     int					=		[
										    x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
										]
										return super()._decode(lowerCAmelCase__			,			**lowerCAmelCase__	)


				def 		__magic_name__(       self			:int			,			lowerCAmelCase__			:Tuple	)		->			List[str]:
										return token in self.encoder


				def 		__magic_name__(       self			:Tuple			,			lowerCAmelCase__			:List[str]	)		->			str:
										return "".join(lowerCAmelCase__	)


				def 		__magic_name__(       self			:List[str]			,			lowerCAmelCase__			:List[Any]	)		->			List[Any]:
										return self.encoder.get(lowerCAmelCase__			,			self.encoder.get(self.unk_token	)	)


				def 		__magic_name__(       self			:Any			,			lowerCAmelCase__			:Any	)		->			Optional[Any]:
										return self.decoder.get(lowerCAmelCase__			,			self.unk_token	)


				def 		__magic_name__(       self			:List[str]			,			lowerCAmelCase__			:str			,			lowerCAmelCase__			:Optional[str] = None	)		->			Tuple[str]:
										if os.path.isdir(lowerCAmelCase__	):
																__SCREAMING_SNAKE_CASE						:     Any					=		os.path.join(
																    lowerCAmelCase__			,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)
										else:
																__SCREAMING_SNAKE_CASE						:     Union[str, Any]					=		(filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		0
										if " " in self.encoder:
																__SCREAMING_SNAKE_CASE						:     List[str]					=		self.encoder[''' ''']
																del self.encoder[" "]
										if "\n" in self.encoder:
																__SCREAMING_SNAKE_CASE						:     Tuple					=		self.encoder['''\n''']
																del self.encoder["\n"]
										__SCREAMING_SNAKE_CASE						:     Optional[int]					=		collections.OrderedDict(sorted(self.encoder.items()			,			key=lambda lowerCAmelCase__	: x[1]	)	)
										with open(lowerCAmelCase__			,			'''w'''			,			encoding='''utf-8'''	) as writer:
																for token, token_index in self.encoder.items():
																						if index != token_index:
																												logger.warning(
																												    f'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
																												    ''' Please check that the vocabulary is not corrupted!'''	)
																												__SCREAMING_SNAKE_CASE						:     Any					=		token_index
																						writer.write(token + '''\n'''	)
																						index += 1
										return (vocab_file,)


				def 		__magic_name__(       self			:Optional[Any]			,			lowerCAmelCase__			:List[int]			,			lowerCAmelCase__			:List[int] = None	)		->			List[int]:
										if token_ids_a is None:
																return [self.bos_token_id] + token_ids_a
										return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a



				def 		__magic_name__(       self			:str			,			lowerCAmelCase__			:List[int]			,			lowerCAmelCase__			:Optional[List[int]] = None			,			lowerCAmelCase__			:bool = False	)		->			List[int]:

										if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=lowerCAmelCase__			,			token_ids_a=lowerCAmelCase__			,			already_has_special_tokens=lowerCAmelCase__	)

										if token_ids_a is not None:
																return [1] + ([0] * len(lowerCAmelCase__	)) + [1] + ([0] * len(lowerCAmelCase__	))
										return [1] + ([0] * len(lowerCAmelCase__	))





'''simple docstring'''

import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
							import sqlitea

							import sqlalchemy

class 	__magic_name__						(    _UpperCamelCase   ):







							def __init__(							self					:       Optional[int]	,_UpperCAmelCase					:       Union[str, "sqlalchemy.sql.Selectable"]	,_UpperCAmelCase					:       Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"]	,_UpperCAmelCase					:       Optional[Features] = None	,_UpperCAmelCase					:       str = None	,_UpperCAmelCase					:       bool = False	,**_UpperCAmelCase					:       Dict	,):
									super().__init__(features=_UpperCAmelCase	,cache_dir=_UpperCAmelCase	,keep_in_memory=_UpperCAmelCase	,**_UpperCAmelCase		)
									_a     :      Tuple        =							Sql(
									    cache_dir=_UpperCAmelCase	,features=_UpperCAmelCase	,sql=_UpperCAmelCase	,con=_UpperCAmelCase	,**_UpperCAmelCase	,)







							def 							__lowercase			(							self					:       Dict		):
									_a     :      Optional[Any]        =							None
									_a     :      Dict        =							None
									_a     :      Dict        =							None
									_a     :      Optional[int]        =							None

									self.builder.download_and_prepare(
									    download_config=_UpperCAmelCase	,download_mode=_UpperCAmelCase	,verification_mode=_UpperCAmelCase	,base_path=_UpperCAmelCase	,)

									# Build dataset for splits
									_a     :      List[str]        =							self.builder.as_dataset(
									    split='train'	,verification_mode=_UpperCAmelCase	,in_memory=self.keep_in_memory		)
									return dataset

class 	__magic_name__						:







							def __init__(							self					:       Optional[int]	,_UpperCAmelCase					:       Dataset	,_UpperCAmelCase					:       str	,_UpperCAmelCase					:       Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"]	,_UpperCAmelCase					:       Optional[int] = None	,_UpperCAmelCase					:       Optional[int] = None	,**_UpperCAmelCase					:       Dict	,):
									if num_proc is not None and num_proc <= 0:
											raise ValueError(F"""num_proc {num_proc} must be an integer > 0."""		)

									_a     :      Dict        =							dataset
									_a     :      List[Any]        =							name
									_a     :      Tuple        =							con
									_a     :      Union[str, Any]        =							batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
									_a     :      List[Any]        =							num_proc
									_a     :      Tuple        =							to_sql_kwargs







							def 							__lowercase			(							self					:       List[Any]		):
									_a     :      Tuple        =							self.to_sql_kwargs.pop('sql'	,_UpperCAmelCase		)
									_a     :      str        =							self.to_sql_kwargs.pop('con'	,_UpperCAmelCase		)
									_a     :      Optional[Any]        =							self.to_sql_kwargs.pop('index'	,_UpperCAmelCase		)

									_a     :      Any        =							self._write(index=_UpperCAmelCase	,**self.to_sql_kwargs		)
									return written







							def 							__lowercase			(							self					:       Optional[int]	,_UpperCAmelCase					:       Dict		):
									_a       ,					_a       ,					_a     :      Any        =							args
									_a     :      Tuple        =							{**to_sql_kwargs, 'if_exists': 'append'} if offset > 0 else to_sql_kwargs
									_a     :      Dict        =							query_table(
									    table=self.dataset.data	,key=slice(_UpperCAmelCase	,offset + self.batch_size		)	,indices=self.dataset._indices	,)
									_a     :      Tuple        =							batch.to_pandas()
									_a     :      Dict        =							df.to_sql(self.name	,self.con	,index=_UpperCAmelCase	,**_UpperCAmelCase		)
									return num_rows or len(_UpperCAmelCase		)







							def 							__lowercase			(							self					:       int	,_UpperCAmelCase					:       Optional[int]	,**_UpperCAmelCase					:       List[Any]		):
									_a     :      Union[str, Any]        =							0

									if self.num_proc is None or self.num_proc == 1:
											for offset in logging.tqdm(
											    range(0	,len(self.dataset		)	,self.batch_size		)	,unit='ba'	,disable=not logging.is_progress_bar_enabled()	,desc='Creating SQL from Arrow format'	,):
													written += self._batch_sql((offset, index, to_sql_kwargs)		)
									else:
											_a       ,					_a     :      List[Any]        =							len(self.dataset		), self.batch_size
											with multiprocessing.Pool(self.num_proc		) as pool:
													for num_rows in logging.tqdm(
													    pool.imap(
													        self._batch_sql	,[(offset, index, to_sql_kwargs) for offset in range(0	,_UpperCAmelCase	,_UpperCAmelCase		)]	,)	,total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size	,unit='ba'	,disable=not logging.is_progress_bar_enabled()	,desc='Creating SQL from Arrow format'	,):
															written += num_rows

									return written




