


'''simple docstring'''







import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A__:    Optional[Any]					    =       logging.get_logger(__name__)

A__:    Any					    =       {'''vocab_file''': '''spm_char.model'''}

A__:    str					    =       {
    '''vocab_file''': {
        '''microsoft/speecht5_asr''': '''https://huggingface.co/microsoft/speecht5_asr/resolve/main/spm_char.model''',
        '''microsoft/speecht5_tts''': '''https://huggingface.co/microsoft/speecht5_tts/resolve/main/spm_char.model''',
        '''microsoft/speecht5_vc''': '''https://huggingface.co/microsoft/speecht5_vc/resolve/main/spm_char.model''',
    }
}

A__:    Dict					    =       {
    '''microsoft/speecht5_asr''': 1024,
    '''microsoft/speecht5_tts''': 1024,
    '''microsoft/speecht5_vc''': 1024,
}







class A__       (						UpperCAmelCase__      ):
      __UpperCamelCase       :  List[str]	       =							VOCAB_FILES_NAMES
      __UpperCamelCase       :  Optional[Any]	       =							PRETRAINED_VOCAB_FILES_MAP
      __UpperCamelCase       :  Optional[int]	       =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __UpperCamelCase       :  Optional[Any]	       =							["input_ids", "attention_mask"]






      def __init__( self			:str     ,						SCREAMING_SNAKE_CASE			:int     ,						SCREAMING_SNAKE_CASE			:List[Any]="<s>"     ,						SCREAMING_SNAKE_CASE			:Optional[Any]="</s>"     ,						SCREAMING_SNAKE_CASE			:Optional[int]="<unk>"     ,						SCREAMING_SNAKE_CASE			:str="<pad>"     ,						SCREAMING_SNAKE_CASE			:Optional[Dict[str, Any]] = None     ,						**SCREAMING_SNAKE_CASE			:Tuple     ,						)       ->					None:




             '''simple docstring'''
             _a    : Optional[Any]     					={} if sp_model_kwargs is None else sp_model_kwargs

             super().__init__(
                 bos_token=SCREAMING_SNAKE_CASE     ,						eos_token=SCREAMING_SNAKE_CASE     ,						unk_token=SCREAMING_SNAKE_CASE     ,						pad_token=SCREAMING_SNAKE_CASE     ,						sp_model_kwargs=self.sp_model_kwargs     ,						**SCREAMING_SNAKE_CASE     ,						)

             _a    : str     					=vocab_file

             _a    : Dict     					=spm.SentencePieceProcessor(**self.sp_model_kwargs	)
             self.sp_model.Load(SCREAMING_SNAKE_CASE	)






      @property
      def __UpperCAmelCase       ( self			:Any	)       ->					Optional[int]:




             '''simple docstring'''
             return self.sp_model.get_piece_size()






      def __UpperCAmelCase       ( self			:List[Any]	)       ->					Optional[int]:




             '''simple docstring'''
             _a    : Optional[Any]     					={self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE	): i for i in range(self.vocab_size	)}
             vocab.update(self.added_tokens_encoder	)
             return vocab






      def __getstate__( self			:Optional[int]	)       ->					Dict:




             '''simple docstring'''
             _a    : Optional[Any]     					=self.__dict__.copy()
             _a    : Union[str, Any]     					=None
             return state






      def __setstate__( self			:str     ,						SCREAMING_SNAKE_CASE			:List[Any]	)       ->					Optional[int]:




             '''simple docstring'''
             _a    : Dict     					=d

             # for backward compatibility
             if not hasattr(self     ,						"""sp_model_kwargs"""	):
                    _a    : List[str]     					={}

             _a    : Optional[Any]     					=spm.SentencePieceProcessor(**self.sp_model_kwargs	)
             self.sp_model.Load(self.vocab_file	)






      def __UpperCAmelCase       ( self			:List[Any]     ,						SCREAMING_SNAKE_CASE			:str	)       ->					List[str]:




             '''simple docstring'''
             return self.sp_model.encode(SCREAMING_SNAKE_CASE     ,						out_type=SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:int     ,						SCREAMING_SNAKE_CASE			:Dict	)       ->					int:




             '''simple docstring'''
             return self.sp_model.piece_to_id(SCREAMING_SNAKE_CASE	)






      def __UpperCAmelCase       ( self			:Union[str, Any]     ,						SCREAMING_SNAKE_CASE			:Union[str, Any]	)       ->					Optional[Any]:




             '''simple docstring'''
             _a    : int     					=self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE	)
             return token






      def __UpperCAmelCase       ( self			:str     ,						SCREAMING_SNAKE_CASE			:int	)       ->					int:




             '''simple docstring'''
             _a    : int     					=[]
             _a    : List[Any]     					=""""""
             for token in tokens:
                    # make sure that special tokens are not decoded using sentencepiece model
                    if token in self.all_special_tokens:
                           out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE	) + token
                           _a    : Dict     					=[]
                    else:
                           current_sub_tokens.append(SCREAMING_SNAKE_CASE	)
             out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE	)
             return out_string.strip()






      def __UpperCAmelCase       ( self			:str     ,						SCREAMING_SNAKE_CASE			:Union[str, Any]     ,						SCREAMING_SNAKE_CASE			:Optional[Any]=None	)       ->					List[int]:




             '''simple docstring'''
             if token_ids_a is None:
                    return token_ids_a + [self.eos_token_id]
             # We don't expect to process pairs, but leave the pair logic for API consistency
             return token_ids_a + token_ids_a + [self.eos_token_id]






      def __UpperCAmelCase       ( self			:List[str]     ,						SCREAMING_SNAKE_CASE			:List[int]     ,						SCREAMING_SNAKE_CASE			:Optional[List[int]] = None     ,						SCREAMING_SNAKE_CASE			:bool = False	)       ->					List[int]:




             '''simple docstring'''
             if already_has_special_tokens:
                    return super().get_special_tokens_mask(
                        token_ids_a=SCREAMING_SNAKE_CASE     ,						token_ids_a=SCREAMING_SNAKE_CASE     ,						already_has_special_tokens=SCREAMING_SNAKE_CASE	)

             _a    : Dict     					=[1]
             if token_ids_a is None:
                    return ([0] * len(SCREAMING_SNAKE_CASE	)) + suffix_ones
             return ([0] * len(SCREAMING_SNAKE_CASE	)) + ([0] * len(SCREAMING_SNAKE_CASE	)) + suffix_ones






      def __UpperCAmelCase       ( self			:List[Any]     ,						SCREAMING_SNAKE_CASE			:str     ,						SCREAMING_SNAKE_CASE			:Optional[str] = None	)       ->					Tuple[str]:




             '''simple docstring'''
             if not os.path.isdir(SCREAMING_SNAKE_CASE	):
                    logger.error(f"Vocabulary path ({save_directory}) should be a directory"	)
                    return
             _a    : str     					=os.path.join(
                 SCREAMING_SNAKE_CASE     ,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]	)

             if os.path.abspath(self.vocab_file	) != os.path.abspath(SCREAMING_SNAKE_CASE	) and os.path.isfile(self.vocab_file	):
                    copyfile(self.vocab_file     ,						SCREAMING_SNAKE_CASE	)
             elif not os.path.isfile(self.vocab_file	):
                    with open(SCREAMING_SNAKE_CASE     ,						"""wb"""	) as fi:
                           _a    : Union[str, Any]     					=self.sp_model.serialized_model_proto()
                           fi.write(SCREAMING_SNAKE_CASE	)

             return (out_vocab_file,)






'''simple docstring'''







import contextlib
import os
import sqlitea

import pytest

from datasets import Dataset, Features, Value
from datasets.io.sql import SqlDatasetReader, SqlDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases, require_sqlalchemy


def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	Any    ,_UpperCAmelCase	:	str  )  ->			Dict:
       assert isinstance(_UpperCAmelCase    ,_UpperCAmelCase  )
       assert dataset.num_rows == 4
       assert dataset.num_columns == 3
       assert dataset.column_names == ["col_1", "col_2", "col_3"]
       for feature, expected_dtype in expected_features.items():
              assert dataset.features[feature].dtype == expected_dtype


@require_sqlalchemy
@pytest.mark.parametrize("""keep_in_memory"""    ,[False, True]  )
def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	Optional[Any]    ,_UpperCAmelCase	:	Optional[int]    ,_UpperCAmelCase	:	Optional[int]    ,_UpperCAmelCase	:	str  )  ->			Optional[Any]:
       _a    : Any     					=tmp_path / """cache"""
       _a    : int     					={"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
       with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
              _a    : Tuple     					=SqlDatasetReader(
                  """dataset"""    ,"""sqlite:///""" + sqlite_path    ,cache_dir=_UpperCAmelCase    ,keep_in_memory=_UpperCAmelCase  ).read()
       _check_sql_dataset(_UpperCAmelCase    ,_UpperCAmelCase  )


@require_sqlalchemy
@pytest.mark.parametrize(
    """features"""    ,[
        None,
        {"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""},
        {"""col_1""": """string""", """col_2""": """string""", """col_3""": """string"""},
        {"""col_1""": """int32""", """col_2""": """int32""", """col_3""": """int32"""},
        {"""col_1""": """float32""", """col_2""": """float32""", """col_3""": """float32"""},
    ]    ,)
def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	List[Any]    ,_UpperCAmelCase	:	Dict    ,_UpperCAmelCase	:	Optional[Any]    ,_UpperCAmelCase	:	int  )  ->			List[Any]:
       _a    : Union[str, Any]     					=tmp_path / """cache"""
       _a    : str     					={"""col_1""": """string""", """col_2""": """int64""", """col_3""": """float64"""}
       _a    : Optional[int]     					=features.copy() if features else default_expected_features
       _a    : Union[str, Any]     					=(
           Features({feature: Value(_UpperCAmelCase  ) for feature, dtype in features.items()}  ) if features is not None else None
       )
       _a    : Optional[Any]     					=SqlDatasetReader("""dataset"""    ,"""sqlite:///""" + sqlite_path    ,features=_UpperCAmelCase    ,cache_dir=_UpperCAmelCase  ).read()
       _check_sql_dataset(_UpperCAmelCase    ,_UpperCAmelCase  )


def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	Optional[Any]  )  ->			List[str]:
       with contextlib.closing(sqlitea.connect(_UpperCAmelCase  )  ) as con:
              _a    : Any     					=con.cursor()
              cur.execute("""SELECT * FROM dataset"""  )
              for row in cur:
                     yield row


@require_sqlalchemy
def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	Dict    ,_UpperCAmelCase	:	Tuple    ,_UpperCAmelCase	:	List[str]  )  ->			Union[str, Any]:
       _a    : Union[str, Any]     					=tmp_path / """cache"""
       _a    : Union[str, Any]     					=os.path.join(_UpperCAmelCase    ,"""tmp.sql"""  )
       _a    : Tuple     					=SqlDatasetReader("""dataset"""    ,"""sqlite:///""" + sqlite_path    ,cache_dir=_UpperCAmelCase  ).read()
       SqlDatasetWriter(_UpperCAmelCase    ,"""dataset"""    ,"""sqlite:///""" + output_sqlite_path    ,num_proc=1  ).write()

       _a    : Tuple     					=iter_sql_file(_UpperCAmelCase  )
       _a    : List[Any]     					=iter_sql_file(_UpperCAmelCase  )

       for rowa, rowa in zip(_UpperCAmelCase    ,_UpperCAmelCase  ):
              assert rowa == rowa


@require_sqlalchemy
def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	Optional[int]    ,_UpperCAmelCase	:	Any    ,_UpperCAmelCase	:	List[Any]  )  ->			Optional[int]:
       _a    : int     					=tmp_path / """cache"""
       _a    : Any     					=os.path.join(_UpperCAmelCase    ,"""tmp.sql"""  )
       _a    : Union[str, Any]     					=SqlDatasetReader("""dataset"""    ,"""sqlite:///""" + sqlite_path    ,cache_dir=_UpperCAmelCase  ).read()
       SqlDatasetWriter(_UpperCAmelCase    ,"""dataset"""    ,"""sqlite:///""" + output_sqlite_path    ,num_proc=2  ).write()

       _a    : List[Any]     					=iter_sql_file(_UpperCAmelCase  )
       _a    : str     					=iter_sql_file(_UpperCAmelCase  )

       for rowa, rowa in zip(_UpperCAmelCase    ,_UpperCAmelCase  ):
              assert rowa == rowa


@require_sqlalchemy
def 	SCREAMING_SNAKE_CASE_					(  _UpperCAmelCase	:	Union[str, Any]    ,_UpperCAmelCase	:	str    ,_UpperCAmelCase	:	List[Any]  )  ->			List[str]:
       _a    : List[str]     					=tmp_path / """cache"""
       _a    : Dict     					=os.path.join(_UpperCAmelCase    ,"""tmp.sql"""  )
       _a    : Optional[Any]     					=SqlDatasetReader("""dataset"""    ,"""sqlite:///""" + sqlite_path    ,cache_dir=_UpperCAmelCase  ).read()
       with pytest.raises(_UpperCAmelCase  ):
              SqlDatasetWriter(_UpperCAmelCase    ,"""dataset"""    ,"""sqlite:///""" + output_sqlite_path    ,num_proc=0  ).write()


