




import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
					import sqlitea

					import sqlalchemy







class     __lowerCAmelCase (       __magic_name__ ):

			"""simple docstring"""







			def __init__(      self			,						lowerCamelCase__			,						lowerCamelCase__			,						lowerCamelCase__ = None			,						lowerCamelCase__ = None			,						lowerCamelCase__ = False			,						**lowerCamelCase__			,						)    ->      str:



							'''simple docstring'''
							super().__init__(features=lowerCamelCase__			,						cache_dir=lowerCamelCase__			,						keep_in_memory=lowerCamelCase__			,						**lowerCamelCase__							)
							__lowerCamelCase		       =    Sql(
							    cache_dir=lowerCamelCase__			,						features=lowerCamelCase__			,						sql=lowerCamelCase__			,						con=lowerCamelCase__			,						**lowerCamelCase__			,						)






			def       lowercase_    (      self							)    ->      int:



							'''simple docstring'''
							__lowerCamelCase		       =    None
							__lowerCamelCase		       =    None
							__lowerCamelCase		       =    None
							__lowerCamelCase		       =    None

							self.builder.download_and_prepare(
							    download_config=lowerCamelCase__			,						download_mode=lowerCamelCase__			,						verification_mode=lowerCamelCase__			,						base_path=lowerCamelCase__			,						)

							# Build dataset for splits
							__lowerCamelCase		       =    self.builder.as_dataset(
							    split='train'			,						verification_mode=lowerCamelCase__			,						in_memory=self.keep_in_memory							)
							return dataset







class     __lowerCAmelCase :

			"""simple docstring"""







			def __init__(      self			,						lowerCamelCase__			,						lowerCamelCase__			,						lowerCamelCase__			,						lowerCamelCase__ = None			,						lowerCamelCase__ = None			,						**lowerCamelCase__			,						)    ->      List[str]:



							'''simple docstring'''
							if num_proc is not None and num_proc <= 0:
											raise ValueError(f"""num_proc {num_proc} must be an integer > 0."""							)

							__lowerCamelCase		       =    dataset
							__lowerCamelCase		       =    name
							__lowerCamelCase		       =    con
							__lowerCamelCase		       =    batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
							__lowerCamelCase		       =    num_proc
							__lowerCamelCase		       =    to_sql_kwargs


			def       lowercase_    (      self							)    ->      int:



							'''simple docstring'''
							__lowerCamelCase		       =    self.to_sql_kwargs.pop('sql'			,						lowerCamelCase__							)
							__lowerCamelCase		       =    self.to_sql_kwargs.pop('con'			,						lowerCamelCase__							)
							__lowerCamelCase		       =    self.to_sql_kwargs.pop('index'			,						lowerCamelCase__							)

							__lowerCamelCase		       =    self._write(index=lowerCamelCase__			,						**self.to_sql_kwargs							)
							return written


			def       lowercase_    (      self			,						lowerCamelCase__							)    ->      Union[str, Any]:



							'''simple docstring'''
							__lowerCamelCase       ,						__lowerCamelCase       ,						__lowerCamelCase		       =    args
							__lowerCamelCase		       =    {**to_sql_kwargs, 'if_exists': 'append'} if offset > 0 else to_sql_kwargs
							__lowerCamelCase		       =    query_table(
							    table=self.dataset.data			,						key=slice(lowerCamelCase__			,						offset + self.batch_size							)			,						indices=self.dataset._indices			,						)
							__lowerCamelCase		       =    batch.to_pandas()
							__lowerCamelCase		       =    df.to_sql(self.name			,						self.con			,						index=lowerCamelCase__			,						**lowerCamelCase__							)
							return num_rows or len(lowerCamelCase__							)






			def       lowercase_    (      self			,						lowerCamelCase__			,						**lowerCamelCase__							)    ->      int:



							'''simple docstring'''
							__lowerCamelCase		       =    0

							if self.num_proc is None or self.num_proc == 1:
											for offset in logging.tqdm(
											    range(0			,						len(self.dataset							)			,						self.batch_size							)			,						unit='ba'			,						disable=not logging.is_progress_bar_enabled()			,						desc='Creating SQL from Arrow format'			,						):
															written += self._batch_sql((offset, index, to_sql_kwargs)							)
							else:
											__lowerCamelCase       ,						__lowerCamelCase		       =    len(self.dataset							), self.batch_size
											with multiprocessing.Pool(self.num_proc							) as pool:
															for num_rows in logging.tqdm(
															    pool.imap(
															        self._batch_sql			,						[(offset, index, to_sql_kwargs) for offset in range(0			,						lowerCamelCase__			,						lowerCamelCase__							)]			,						)			,						total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size			,						unit='ba'			,						disable=not logging.is_progress_bar_enabled()			,						desc='Creating SQL from Arrow format'			,						):
																			written += num_rows

							return written








from typing import List, Optional

import numpy as np

from ...processing_utils import ProcessorMixin
from ...utils import to_numpy







class     __lowerCAmelCase (       __magic_name__ ):

			"""simple docstring"""







			snake_case_   		=       '''EncodecFeatureExtractor'''
			snake_case_   		=       ('''T5Tokenizer''', '''T5TokenizerFast''')


			def __init__(      self			,						lowerCamelCase__			,						lowerCamelCase__							)    ->      int:



							'''simple docstring'''
							super().__init__(lowerCamelCase__			,						lowerCamelCase__							)
							__lowerCamelCase		       =    self.feature_extractor
							__lowerCamelCase		       =    False


			def       lowercase_    (      self			,						lowerCamelCase__=None			,						lowerCamelCase__=None			,						lowerCamelCase__=True							)    ->      List[Any]:



							'''simple docstring'''
							return self.tokenizer.get_decoder_prompt_ids(task=lowerCamelCase__			,						language=lowerCamelCase__			,						no_timestamps=lowerCamelCase__							)


			def __call__(      self			,						*lowerCamelCase__			,						**lowerCamelCase__							)    ->      Dict:



							'''simple docstring'''
							# For backward compatibility
							if self._in_target_context_manager:
											return self.current_processor(*lowerCamelCase__			,						**lowerCamelCase__							)

							__lowerCamelCase		       =    kwargs.pop('audio'			,						lowerCamelCase__							)
							__lowerCamelCase		       =    kwargs.pop('sampling_rate'			,						lowerCamelCase__							)
							__lowerCamelCase		       =    kwargs.pop('text'			,						lowerCamelCase__							)
							if len(lowerCamelCase__							) > 0:
											__lowerCamelCase		       =    args[0]
											__lowerCamelCase		       =    args[1:]

							if audio is None and text is None:
											raise ValueError('You need to specify either an `audio` or `text` input to process.'							)

							if text is not None:
											__lowerCamelCase		       =    self.tokenizer(lowerCamelCase__			,						**lowerCamelCase__							)

							if audio is not None:
											__lowerCamelCase		       =    self.feature_extractor(lowerCamelCase__			,						*lowerCamelCase__			,						sampling_rate=lowerCamelCase__			,						**lowerCamelCase__							)

							if audio is None:
											return inputs

							elif text is None:
											return audio_inputs

							else:
											__lowerCamelCase		       =    audio_inputs['input_values']
											if "padding_mask" in audio_inputs:
															__lowerCamelCase		       =    audio_inputs['padding_mask']
											return inputs


			def       lowercase_    (      self			,						*lowerCamelCase__			,						**lowerCamelCase__							)    ->      Union[str, Any]:



							'''simple docstring'''
							__lowerCamelCase		       =    kwargs.pop('audio'			,						lowerCamelCase__							)
							__lowerCamelCase		       =    kwargs.pop('padding_mask'			,						lowerCamelCase__							)

							if len(lowerCamelCase__							) > 0:
											__lowerCamelCase		       =    args[0]
											__lowerCamelCase		       =    args[1:]

							if audio_values is not None:
											return self._decode_audio(lowerCamelCase__			,						padding_mask=lowerCamelCase__							)
							else:
											return self.tokenizer.batch_decode(*lowerCamelCase__			,						**lowerCamelCase__							)


			def       lowercase_    (      self			,						*lowerCamelCase__			,						**lowerCamelCase__							)    ->      List[Any]:



							'''simple docstring'''
							return self.tokenizer.decode(*lowerCamelCase__			,						**lowerCamelCase__							)






			def       lowercase_    (      self			,						lowerCamelCase__			,						lowerCamelCase__ = None							)    ->      List[np.ndarray]:



							'''simple docstring'''
							__lowerCamelCase		       =    to_numpy(lowerCamelCase__							)
							__lowerCamelCase       ,						__lowerCamelCase       ,						__lowerCamelCase		       =    audio_values.shape

							if padding_mask is None:
											return list(lowerCamelCase__							)

							__lowerCamelCase		       =    to_numpy(lowerCamelCase__							)

							# match the sequence length of the padding mask to the generated audio arrays by padding with the **non-padding**
							# token (so that the generated audio values are **not** treated as padded tokens)
							__lowerCamelCase		       =    seq_len - padding_mask.shape[-1]
							__lowerCamelCase		       =    1 - self.feature_extractor.padding_value
							__lowerCamelCase		       =    np.pad(lowerCamelCase__			,						((0, 0), (0, difference))			,						'constant'			,						constant_values=lowerCamelCase__							)

							__lowerCamelCase		       =    audio_values.tolist()
							for i in range(lowerCamelCase__							):
											__lowerCamelCase		       =    np.asarray(audio_values[i]							)[
											    padding_mask[i][None, :] != self.feature_extractor.padding_value
											]
											__lowerCamelCase		       =    sliced_audio.reshape(lowerCamelCase__			,						-1							)

							return audio_values


