





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__A				:    int       =   logging.get_logger(__name__)

__A				:    Tuple       =   {"vocab_file": "sentencepiece.bpe.model"}

__A				:    Optional[Any]       =   {
    "vocab_file": {
        "camembert-base": "https://huggingface.co/camembert-base/resolve/main/sentencepiece.bpe.model",
    }
}

__A				:    Union[str, Any]       =   {
    "camembert-base": 512,
}

__A				:    Dict       =   "▁"

class 	A_       (a_			):
	UpperCAmelCase__					=			VOCAB_FILES_NAMES
	UpperCAmelCase__					=			PRETRAINED_VOCAB_FILES_MAP
	UpperCAmelCase__					=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	UpperCAmelCase__					=			['''input_ids''', '''attention_mask''']
	def __init__(    self							,					_A							,					_A="<s>"							,					_A="</s>"							,					_A="</s>"							,					_A="<s>"							,					_A="<unk>"							,					_A="<pad>"							,					_A="<mask>"							,					_A=["<s>NOTUSED", "</s>NOTUSED"]							,					_A = None							,					**_A							,					):






				'''simple docstring'''







				UpperCAmelCase   		=				AddedToken(_A							,					lstrip=_A							,					rstrip=_A		) if isinstance(_A							,					_A		) else mask_token

				UpperCAmelCase   		=				{} if sp_model_kwargs is None else sp_model_kwargs

				super().__init__(
				    bos_token=_A							,					eos_token=_A							,					unk_token=_A							,					sep_token=_A							,					cls_token=_A							,					pad_token=_A							,					mask_token=_A							,					additional_special_tokens=_A							,					sp_model_kwargs=self.sp_model_kwargs							,					**_A							,					)
				UpperCAmelCase   		=				spm.SentencePieceProcessor(**self.sp_model_kwargs		)
				self.sp_model.Load(str(_A		)		)
				UpperCAmelCase   		=				vocab_file
				# HACK: These tokens were added by fairseq but don't seem to be actually used when duplicated in the actual
				# sentencepiece vocabulary (this is the case for <s> and </s>
				UpperCAmelCase   		=				{'''<s>NOTUSED''': 0, '''<pad>''': 1, '''</s>NOTUSED''': 2, '''<unk>''': 3}
				UpperCAmelCase   		=				len(self.fairseq_tokens_to_ids		)
				UpperCAmelCase   		=				len(self.sp_model		) + len(self.fairseq_tokens_to_ids		)
				UpperCAmelCase   		=				{v: k for k, v in self.fairseq_tokens_to_ids.items()}
	def      _lowercase				(    self							,					_A							,					_A = None		):






				'''simple docstring'''







				if token_ids_a is None:
							return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
				UpperCAmelCase   		=				[self.cls_token_id]
				UpperCAmelCase   		=				[self.sep_token_id]
				return cls + token_ids_a + sep + sep + token_ids_a + sep
	def      _lowercase				(    self							,					_A							,					_A = None							,					_A = False		):






				'''simple docstring'''







				if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=_A							,					token_ids_a=_A							,					already_has_special_tokens=_A		)

				if token_ids_a is None:
							return [1] + ([0] * len(_A		)) + [1]
				return [1] + ([0] * len(_A		)) + [1, 1] + ([0] * len(_A		)) + [1]
	def      _lowercase				(    self							,					_A							,					_A = None		):






				'''simple docstring'''







				UpperCAmelCase   		=				[self.sep_token_id]
				UpperCAmelCase   		=				[self.cls_token_id]

				if token_ids_a is None:
							return len(cls + token_ids_a + sep		) * [0]
				return len(cls + token_ids_a + sep + sep + token_ids_a + sep		) * [0]
	@property
	def      _lowercase				(    self		):






				'''simple docstring'''







				return len(self.fairseq_tokens_to_ids		) + len(self.sp_model		)
	def      _lowercase				(    self		):






				'''simple docstring'''







				UpperCAmelCase   		=				{self.convert_ids_to_tokens(_A		): i for i in range(self.vocab_size		)}
				vocab.update(self.added_tokens_encoder		)
				return vocab
	def      _lowercase				(    self							,					_A		):






				'''simple docstring'''







				return self.sp_model.encode(_A							,					out_type=_A		)
	def      _lowercase				(    self							,					_A		):






				'''simple docstring'''







				if token in self.fairseq_tokens_to_ids:
							return self.fairseq_tokens_to_ids[token]
				elif self.sp_model.PieceToId(_A		) == 0:
							# Convert sentence piece unk token to fairseq unk token index
							return self.unk_token_id
				return self.fairseq_offset + self.sp_model.PieceToId(_A		)
	def      _lowercase				(    self							,					_A		):






				'''simple docstring'''







				if index in self.fairseq_ids_to_tokens:
							return self.fairseq_ids_to_tokens[index]
				return self.sp_model.IdToPiece(index - self.fairseq_offset		)
	def      _lowercase				(    self							,					_A		):






				'''simple docstring'''







				UpperCAmelCase   		=				[]
				UpperCAmelCase   		=				''''''
				UpperCAmelCase   		=				False
				for token in tokens:
							# make sure that special tokens are not decoded using sentencepiece model
							if token in self.all_special_tokens:
										if not prev_is_special:
													out_string += " "
										out_string += self.sp_model.decode(_A		) + token
										UpperCAmelCase   		=				True
										UpperCAmelCase   		=				[]
							else:
										current_sub_tokens.append(_A		)
										UpperCAmelCase   		=				False
				out_string += self.sp_model.decode(_A		)
				return out_string.strip()
	def __getstate__(    self		):






				'''simple docstring'''







				UpperCAmelCase   		=				self.__dict__.copy()
				UpperCAmelCase   		=				None
				return state
	def __setstate__(    self							,					_A		):






				'''simple docstring'''







				UpperCAmelCase   		=				d

				# for backward compatibility
				if not hasattr(self							,					'''sp_model_kwargs'''		):
							UpperCAmelCase   		=				{}

				UpperCAmelCase   		=				spm.SentencePieceProcessor(**self.sp_model_kwargs		)
				self.sp_model.Load(self.vocab_file		)







	def      _lowercase				(    self							,					_A							,					_A = None		):






				'''simple docstring'''







				if not os.path.isdir(_A		):
							logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""		)
							return
				UpperCAmelCase   		=				os.path.join(
				    _A							,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

				if os.path.abspath(self.vocab_file		) != os.path.abspath(_A		) and os.path.isfile(self.vocab_file		):
							copyfile(self.vocab_file							,					_A		)
				elif not os.path.isfile(self.vocab_file		):
							with open(_A							,					'''wb'''		) as fi:
										UpperCAmelCase   		=				self.sp_model.serialized_model_proto()
										fi.write(_A		)

				return (out_vocab_file,)










from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_speech_available,
    is_tf_available,
    is_torch_available,
)


__A				:    Union[str, Any]       =   {
    "configuration_speech_to_text": ["SPEECH_TO_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP", "Speech2TextConfig"],
    "processing_speech_to_text": ["Speech2TextProcessor"],
}

try:
				if not is_sentencepiece_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				__A				:    List[Any]       =   ["Speech2TextTokenizer"]

try:
				if not is_speech_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				__A				:    List[Any]       =   ["Speech2TextFeatureExtractor"]

try:
				if not is_tf_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				__A				:    int       =   [
				    "TF_SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST",
				    "TFSpeech2TextForConditionalGeneration",
				    "TFSpeech2TextModel",
				    "TFSpeech2TextPreTrainedModel",
				]

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				__A				:    Tuple       =   [
				    "SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST",
				    "Speech2TextForConditionalGeneration",
				    "Speech2TextModel",
				    "Speech2TextPreTrainedModel",
				]


if TYPE_CHECKING:
				from .configuration_speech_to_text import SPEECH_TO_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP, SpeechaTextConfig
				from .processing_speech_to_text import SpeechaTextProcessor

				try:
								if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_speech_to_text import SpeechaTextTokenizer

				try:
								if not is_speech_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .feature_extraction_speech_to_text import SpeechaTextFeatureExtractor

				try:
								if not is_tf_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_tf_speech_to_text import (
								    TF_SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    TFSpeechaTextForConditionalGeneration,
								    TFSpeechaTextModel,
								    TFSpeechaTextPreTrainedModel,
								)

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_speech_to_text import (
								    SPEECH_TO_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    SpeechaTextForConditionalGeneration,
								    SpeechaTextModel,
								    SpeechaTextPreTrainedModel,
								)

else:
				import sys

				__A				:    Dict       =   _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



