




"""simple docstring"""




import collections
import os
from typing import List, Optional, Tuple

from transformers.utils import is_jieba_available, requires_backends


if is_jieba_available():
			import jieba

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


a_        =       logging.get_logger(__name__)

a_        =       {'vocab_file': 'vocab.txt'}

a_        =       {
    'vocab_file': {
        'openbmb/cpm-ant-10b': 'https://huggingface.co/openbmb/cpm-ant-10b/blob/main/vocab.txt',
    },
}

a_        =       {
    'openbmb/cpm-ant-10b': 1_0_2_4,
}







def      __UpperCAmelCase     ( __UpperCamelCase	):
			__lowercase			:  Any								=       collections.OrderedDict()
			with open(__UpperCamelCase	,				'''r'''	,				encoding='''utf-8'''	) as reader:
						__lowercase			:  List[Any]								=       reader.readlines()
			for index, token in enumerate(__UpperCamelCase	):
						__lowercase			:  str								=       token.rstrip('''\n'''	)
						__lowercase			:  Optional[Any]								=       index
			return vocab

class   UpperCAmelCase_    (			snake_case		):
		def __init__(						self				,						UpperCamelCase_				,						UpperCamelCase_="<unk>"				,						UpperCamelCase_=2_00       )						->       Any:
					__lowercase			:  Any								=       vocab
					__lowercase			:  Dict								=       unk_token
					__lowercase			:  List[str]								=       max_input_chars_per_word
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       Union[str, Any]:
					__lowercase			:  Union[str, Any]								=       list(UpperCamelCase_       )
					if len(UpperCamelCase_       ) > self.max_input_chars_per_word:
								return [self.unk_token]

					__lowercase			:  Any								=       0
					__lowercase			:  List[str]								=       []
					while start < len(UpperCamelCase_       ):
								__lowercase			:  List[str]								=       len(UpperCamelCase_       )
								__lowercase			:  Optional[Any]								=       None
								while start < end:
											__lowercase			:  Union[str, Any]								=       ''''''.join(chars[start:end]       )
											if substr in self.vocab:
														__lowercase			:  Union[str, Any]								=       substr
														break
											end -= 1
								if cur_substr is None:
											sub_tokens.append(self.unk_token       )
											start += 1
								else:
											sub_tokens.append(UpperCamelCase_       )
											__lowercase			:  int								=       end

					return sub_tokens


class   UpperCAmelCase_    (			snake_case		):
		UpperCamelCase							=VOCAB_FILES_NAMES
		UpperCamelCase							=PRETRAINED_VOCAB_FILES_MAP
		UpperCamelCase							=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		UpperCamelCase							=["input_ids", "attention_mask"]
		UpperCamelCase							=False
		def __init__(						self				,						UpperCamelCase_				,						UpperCamelCase_="<d>"				,						UpperCamelCase_="</d>"				,						UpperCamelCase_="<s>"				,						UpperCamelCase_="</s>"				,						UpperCamelCase_="<pad>"				,						UpperCamelCase_="<unk>"				,						UpperCamelCase_="</n>"				,						UpperCamelCase_="</_>"				,						UpperCamelCase_="left"				,						**UpperCamelCase_				,						)						->       int:
					requires_backends(self				,						['''jieba''']       )
					super().__init__(
					    bod_token=UpperCamelCase_				,						eod_token=UpperCamelCase_				,						bos_token=UpperCamelCase_				,						eos_token=UpperCamelCase_				,						pad_token=UpperCamelCase_				,						unk_token=UpperCamelCase_				,						line_token=UpperCamelCase_				,						space_token=UpperCamelCase_				,						padding_side=UpperCamelCase_				,						**UpperCamelCase_				,						)
					__lowercase			:  Optional[int]								=       bod_token
					__lowercase			:  Optional[int]								=       eod_token
					__lowercase			:  Union[str, Any]								=       load_vocab(UpperCamelCase_       )
					__lowercase			:  List[str]								=       self.encoder[space_token]
					__lowercase			:  Dict								=       self.encoder[line_token]

					del self.encoder[space_token]
					del self.encoder[line_token]

					__lowercase			:  int								=       collections.OrderedDict(sorted(self.encoder.items()				,						key=lambda UpperCamelCase_       : x[1]       )       )
					__lowercase			:  List[str]								=       {v: k for k, v in self.encoder.items()}

					__lowercase			:  Tuple								=       WordpieceTokenizer(vocab=self.encoder				,						unk_token=self.unk_token       )
		@property
		def 	_lowerCamelCase							(						self       )						->       List[str]:
					return self.encoder[self.bod_token]
		@property
		def 	_lowerCamelCase							(						self       )						->       Optional[Any]:
					return self.encoder[self.eod_token]
		@property
		def 	_lowerCamelCase							(						self       )						->       Optional[Any]:
					return self.encoder["\n"]
		@property
		def 	_lowerCamelCase							(						self       )						->       int:
					return len(self.encoder       )
		def 	_lowerCamelCase							(						self       )						->       List[str]:
					return dict(self.encoder				,						**self.added_tokens_encoder       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       List[Any]:
					__lowercase			:  List[str]								=       []
					for x in jieba.cut(UpperCamelCase_				,						cut_all=UpperCamelCase_       ):
								output_tokens.extend(self.wordpiece_tokenizer.tokenize(UpperCamelCase_       )       )
					return output_tokens
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						**UpperCamelCase_       )						->       str:
					__lowercase			:  Union[str, Any]								=       [i for i in token_ids if i >= 0]
					__lowercase			:  int								=       [
					    x for x in token_ids if x != self.pad_token_id and x != self.eos_token_id and x != self.bos_token_id
					]
					return super()._decode(UpperCamelCase_				,						**UpperCamelCase_       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       List[Any]:
					return token in self.encoder
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       str:
					return "".join(UpperCamelCase_       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       Optional[int]:
					return self.encoder.get(UpperCamelCase_				,						self.encoder.get(self.unk_token       )       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_       )						->       Optional[Any]:
					return self.decoder.get(UpperCamelCase_				,						self.unk_token       )
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_ = None       )						->       Tuple[str]:
					if os.path.isdir(UpperCamelCase_       ):
								__lowercase			:  Optional[int]								=       os.path.join(
								    UpperCamelCase_				,						(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )
					else:
								__lowercase			:  Optional[Any]								=       (filename_prefix + '''-''' if filename_prefix else '''''') + save_directory
					__lowercase			:  int								=       0
					if " " in self.encoder:
								__lowercase			:  List[Any]								=       self.encoder[''' ''']
								del self.encoder[" "]
					if "\n" in self.encoder:
								__lowercase			:  Dict								=       self.encoder['''\n''']
								del self.encoder["\n"]
					__lowercase			:  Optional[Any]								=       collections.OrderedDict(sorted(self.encoder.items()				,						key=lambda UpperCamelCase_       : x[1]       )       )
					with open(UpperCamelCase_				,						'''w'''				,						encoding='''utf-8'''       ) as writer:
								for token, token_index in self.encoder.items():
											if index != token_index:
														logger.warning(
														    F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
														    ''' Please check that the vocabulary is not corrupted!'''       )
														__lowercase			:  int								=       token_index
											writer.write(token + '''\n'''       )
											index += 1
					return (vocab_file,)
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_ = None       )						->       List[int]:
					if token_ids_a is None:
								return [self.bos_token_id] + token_ids_a
					return [self.bos_token_id] + token_ids_a + [self.bos_token_id] + token_ids_a
		def 	_lowerCamelCase							(						self				,						UpperCamelCase_				,						UpperCamelCase_ = None				,						UpperCamelCase_ = False       )						->       List[int]:

					if already_has_special_tokens:
								return super().get_special_tokens_mask(
								    token_ids_a=UpperCamelCase_				,						token_ids_a=UpperCamelCase_				,						already_has_special_tokens=UpperCamelCase_       )

					if token_ids_a is not None:
								return [1] + ([0] * len(UpperCamelCase_       )) + [1] + ([0] * len(UpperCamelCase_       ))
					return [1] + ([0] * len(UpperCamelCase_       ))







from typing import Optional

from .. import Features, NamedSplit
from ..packaged_modules.text.text import Text
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader
class 						lowerCamelCase_							(					lowerCAmelCase__					):



					'''simple docstring'''







					def __init__(					self			,							snake_case_			,							snake_case_ = None			,							snake_case_ = None			,							snake_case_ = None			,							snake_case_ = False			,							snake_case_ = False			,							snake_case_ = None			,							**snake_case_			,							) ->      Dict:







								'''simple docstring'''

								super().__init__(
								    snake_case_			,							split=snake_case_			,							features=snake_case_			,							cache_dir=snake_case_			,							keep_in_memory=snake_case_			,							streaming=snake_case_			,							num_proc=snake_case_			,							**snake_case_			,							)
								__lowercase			       = path_or_paths if isinstance(snake_case_			,							snake_case_    ) else {self.split: path_or_paths}
								__lowercase			       = Text(
								    cache_dir=snake_case_			,							data_files=snake_case_			,							features=snake_case_			,							**snake_case_			,							)






					def A   (					self    ) ->      int:







								'''simple docstring'''

								if self.streaming:
											__lowercase			       = self.builder.as_streaming_dataset(split=self.split    )
								# Build regular (map-style) dataset
								else:
											__lowercase			       = None
											__lowercase			       = None
											__lowercase			       = None
											__lowercase			       = None

											self.builder.download_and_prepare(
											    download_config=snake_case_			,							download_mode=snake_case_			,							verification_mode=snake_case_			,							base_path=snake_case_			,							num_proc=self.num_proc			,							)
											__lowercase			       = self.builder.as_dataset(
											    split=self.split			,							verification_mode=snake_case_			,							in_memory=self.keep_in_memory    )
								return dataset


