





"""simple docstring"""





from typing import List, Optional, Union

import numpy as np
import torch
import torchaudio.compliance.kaldi as ta_kaldi

from ...feature_extraction_sequence_utils import SequenceFeatureExtractor
from ...feature_extraction_utils import BatchFeature
from ...utils import PaddingStrategy, TensorType, logging


SCREAMING_SNAKE_CASE__					=   logging.get_logger(__name__)

class 		lowercase					(					_UpperCAmelCase					):
      _SCREAMING_SNAKE_CASE					=      ['input_features', 'attention_mask']





      def __init__(       self      ,						lowercase=80      ,						lowercase=16_000      ,						lowercase=80      ,						lowercase=0.0      ,						lowercase=True      ,						lowercase=True      ,						lowercase=True      ,						**lowercase      ,						)		->     Optional[Any]:
            super().__init__(feature_size=lowercase      ,						sampling_rate=lowercase      ,						padding_value=lowercase      ,						**lowercase )
            lowerCAmelCase  						=       num_mel_bins
            lowerCAmelCase  						=       do_ceptral_normalize
            lowerCAmelCase  						=       normalize_means
            lowerCAmelCase  						=       normalize_vars
            lowerCAmelCase  						=       True





      def     _snake_case			(       self      ,						lowercase      ,						)		->     np.ndarray:
            lowerCAmelCase  						=       waveform * (2**15)  # Kaldi compliance: 16-bit signed integers
            lowerCAmelCase  						=       torch.from_numpy(lowercase ).unsqueeze(0 )
            lowerCAmelCase  						=       ta_kaldi.fbank(lowercase      ,						num_mel_bins=self.num_mel_bins      ,						sample_frequency=self.sampling_rate )
            return features.numpy()





      @staticmethod
      def     _snake_case			(       lowercase      ,						lowercase      ,						lowercase = True      ,						lowercase = True      ,						lowercase = 0.0      ,						)		->     np.ndarray:
            # make sure we normalize float32 arrays
            if normalize_means:
                  lowerCAmelCase  						=       x[:input_length].mean(axis=0 )
                  lowerCAmelCase  						=       np.subtract(lowercase      ,						lowercase )
            if normalize_vars:
                  lowerCAmelCase  						=       x[:input_length].std(axis=0 )
                  lowerCAmelCase  						=       np.divide(lowercase      ,						lowercase )

            if input_length < x.shape[0]:
                  lowerCAmelCase  						=       padding_value

            # make sure array is in float32
            lowerCAmelCase  						=       x.astype(np.floataa )

            return x





      def     _snake_case			(       self      ,						lowercase      ,						lowercase = None )		->     List[np.ndarray]:
            lowerCAmelCase  						=       attention_mask.sum(-1 ) if attention_mask is not None else [x.shape[0] for x in input_features]
            return [
                self.utterance_cmvn(lowercase      ,						lowercase      ,						self.normalize_means      ,						self.normalize_vars      ,						self.padding_value )
                for x, n in zip(lowercase      ,						lowercase )
            ]





      def __call__(       self      ,						lowercase      ,						lowercase = False      ,						lowercase = None      ,						lowercase = False      ,						lowercase = None      ,						lowercase = None      ,						lowercase = None      ,						lowercase = None      ,						**lowercase      ,						)		->     BatchFeature:

            if sampling_rate is not None:
                  if sampling_rate != self.sampling_rate:
                        raise ValueError(
                            f'The model corresponding to this feature extractor: {self} was trained using a sampling rate of'
                            f' {self.sampling_rate}. Please make sure that the provided `raw_speech` input was sampled with'
                            f' {self.sampling_rate} and not {sampling_rate}.' )
            else:
                  logger.warning(
                      """It is strongly recommended to pass the `sampling_rate` argument to this function. """
                      """Failing to do so can result in silent errors that might be hard to debug.""" )

            lowerCAmelCase  						=       isinstance(lowercase      ,						np.ndarray ) and len(raw_speech.shape ) > 1
            if is_batched_numpy and len(raw_speech.shape ) > 2:
                  raise ValueError(f'Only mono-channel audio is supported for input to {self}' )
            lowerCAmelCase  						=       is_batched_numpy or (
                isinstance(lowercase      ,						(list, tuple) ) and (isinstance(raw_speech[0]      ,						(np.ndarray, tuple, list) ))
            )

            if is_batched:
                  lowerCAmelCase  						=       [np.asarray(lowercase      ,						dtype=np.floataa ) for speech in raw_speech]
            elif not is_batched and not isinstance(lowercase      ,						np.ndarray ):
                  lowerCAmelCase  						=       np.asarray(lowercase      ,						dtype=np.floataa )
            elif isinstance(lowercase      ,						np.ndarray ) and raw_speech.dtype is np.dtype(np.floataa ):
                  lowerCAmelCase  						=       raw_speech.astype(np.floataa )

            # always return batch
            if not is_batched:
                  lowerCAmelCase  						=       [raw_speech]

            # extract fbank features
            lowerCAmelCase  						=       [self._extract_fbank_features(lowercase ) for waveform in raw_speech]

            # convert into correct format for padding
            lowerCAmelCase  						=       BatchFeature({"""input_features""": features} )

            lowerCAmelCase  						=       self.pad(
                lowercase      ,						padding=lowercase      ,						max_length=lowercase      ,						truncation=lowercase      ,						pad_to_multiple_of=lowercase      ,						return_attention_mask=lowercase      ,						**lowercase      ,						)

            # make sure list is in array format
            lowerCAmelCase  						=       padded_inputs.get("""input_features""" )
            if isinstance(input_features[0]      ,						lowercase ):
                  lowerCAmelCase  						=       [np.asarray(lowercase      ,						dtype=np.floataa ) for feature in input_features]

            lowerCAmelCase  						=       padded_inputs.get("""attention_mask""" )
            if attention_mask is not None:
                  lowerCAmelCase  						=       [np.asarray(lowercase      ,						dtype=np.intaa ) for array in attention_mask]

            # Utterance-level cepstral mean and variance normalization
            if self.do_ceptral_normalize:
                  lowerCAmelCase  						=       (
                      np.array(lowercase      ,						dtype=np.intaa )
                      if self._get_padding_strategies(lowercase      ,						max_length=lowercase ) is not PaddingStrategy.DO_NOT_PAD
                      else None
                  )
                  lowerCAmelCase  						=       self.normalize(
                      padded_inputs["""input_features"""]      ,						attention_mask=lowercase )

            if return_tensors is not None:
                  lowerCAmelCase  						=       padded_inputs.convert_to_tensors(lowercase )

            return padded_inputs







import io
import json

import fsspec
import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.json import JsonDatasetReader, JsonDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases

def 			lowercase_				(			_lowerCamelCase							:       Optional[Any]		,	_lowerCamelCase							:       int):
    assert isinstance(_lowerCamelCase		,	_lowerCamelCase)
    assert dataset.num_rows == 4
    assert dataset.num_columns == 3
    assert dataset.column_names == ["col_1", "col_2", "col_3"]
    for feature, expected_dtype in expected_features.items():
        assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize("keep_in_memory"		,	[False, True])
def 			lowercase_				(			_lowerCamelCase							:       List[Any]		,	_lowerCamelCase							:       Any		,	_lowerCamelCase							:       str):
    lowercase__ :   Optional[int]    					=				tmp_path / "cache"
    lowercase__ :   str    					=				{"col_1": "string", "col_2": "int64", "col_3": "float64"}
    with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
        lowercase__ :   Union[str, Any]    					=				JsonDatasetReader(_lowerCamelCase		,	cache_dir=_lowerCamelCase		,	keep_in_memory=_lowerCamelCase).read()
    _check_json_dataset(_lowerCamelCase		,	_lowerCamelCase)

@pytest.mark.parametrize(
    "features"		,	[
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]		,	)
def 			lowercase_				(			_lowerCamelCase							:       List[str]		,	_lowerCamelCase							:       Dict		,	_lowerCamelCase							:       Dict):
    lowercase__ :   List[Any]    					=				tmp_path / "cache"
    lowercase__ :   Union[str, Any]    					=				{"col_1": "string", "col_2": "int64", "col_3": "float64"}
    lowercase__ :   List[Any]    					=				features.copy() if features else default_expected_features
    lowercase__ :   List[Any]    					=				(
        Features({feature: Value(_lowerCamelCase) for feature, dtype in features.items()}) if features is not None else None
    )
    lowercase__ :   Any    					=				JsonDatasetReader(_lowerCamelCase		,	features=_lowerCamelCase		,	cache_dir=_lowerCamelCase).read()
    _check_json_dataset(_lowerCamelCase		,	_lowerCamelCase)

@pytest.mark.parametrize(
    "features"		,	[
        None,
        {"col_3": "float64", "col_1": "string", "col_2": "int64"},
    ]		,	)
def 			lowercase_				(			_lowerCamelCase							:       Any		,	_lowerCamelCase							:       Any		,	_lowerCamelCase							:       List[str]):
    lowercase__ :   Optional[Any]    					=				tmp_path / "cache"
    lowercase__ :   Tuple    					=				{"col_3": "float64", "col_1": "string", "col_2": "int64"}
    lowercase__ :   List[Any]    					=				features.copy() if features else default_expected_features
    lowercase__ :   int    					=				(
        Features({feature: Value(_lowerCamelCase) for feature, dtype in features.items()}) if features is not None else None
    )
    lowercase__ :   Any    					=				JsonDatasetReader(_lowerCamelCase		,	features=_lowerCamelCase		,	cache_dir=_lowerCamelCase).read()
    assert isinstance(_lowerCamelCase		,	_lowerCamelCase)
    assert dataset.num_rows == 2
    assert dataset.num_columns == 3
    assert dataset.column_names == ["col_3", "col_1", "col_2"]
    for feature, expected_dtype in expected_features.items():
        assert dataset.features[feature].dtype == expected_dtype

def 			lowercase_				(			_lowerCamelCase							:       List[Any]		,	_lowerCamelCase							:       Optional[int]):
    # jsonl_312_path features are {"col_3": "float64", "col_1": "string", "col_2": "int64"}
    lowercase__ :   Any    					=				{"col_2": "int64", "col_3": "float64", "col_1": "string"}
    lowercase__ :   str    					=				features.copy()
    lowercase__ :   str    					=				(
        Features({feature: Value(_lowerCamelCase) for feature, dtype in features.items()}) if features is not None else None
    )
    lowercase__ :   Optional[int]    					=				tmp_path / "cache"
    lowercase__ :   Any    					=				JsonDatasetReader(_lowerCamelCase		,	features=_lowerCamelCase		,	cache_dir=_lowerCamelCase).read()
    assert isinstance(_lowerCamelCase		,	_lowerCamelCase)
    assert dataset.num_rows == 2
    assert dataset.num_columns == 3
    assert dataset.column_names == ["col_2", "col_3", "col_1"]
    for feature, expected_dtype in expected_features.items():
        assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize("split"		,	[None, NamedSplit("train"), "train", "test"])
def 			lowercase_				(			_lowerCamelCase							:       Dict		,	_lowerCamelCase							:       Optional[int]		,	_lowerCamelCase							:       List[str]):
    lowercase__ :   Union[str, Any]    					=				tmp_path / "cache"
    lowercase__ :   List[Any]    					=				{"col_1": "string", "col_2": "int64", "col_3": "float64"}
    lowercase__ :   Union[str, Any]    					=				JsonDatasetReader(_lowerCamelCase		,	cache_dir=_lowerCamelCase		,	split=_lowerCamelCase).read()
    _check_json_dataset(_lowerCamelCase		,	_lowerCamelCase)
    assert dataset.split == split if split else "train"

@pytest.mark.parametrize("path_type"		,	[str, list])
def 			lowercase_				(			_lowerCamelCase							:       Union[str, Any]		,	_lowerCamelCase							:       str		,	_lowerCamelCase							:       int):
    if issubclass(_lowerCamelCase		,	_lowerCamelCase):
        lowercase__ :   Tuple    					=				jsonl_path
    elif issubclass(_lowerCamelCase		,	_lowerCamelCase):
        lowercase__ :   str    					=				[jsonl_path]
    lowercase__ :   str    					=				tmp_path / "cache"
    lowercase__ :   Optional[Any]    					=				{"col_1": "string", "col_2": "int64", "col_3": "float64"}
    lowercase__ :   Tuple    					=				JsonDatasetReader(_lowerCamelCase		,	cache_dir=_lowerCamelCase).read()
    _check_json_dataset(_lowerCamelCase		,	_lowerCamelCase)

def 			lowercase_				(			_lowerCamelCase							:       List[Any]		,	_lowerCamelCase							:       Optional[Any]		,	_lowerCamelCase							:       Optional[int]=("train",)):
    assert isinstance(_lowerCamelCase		,	_lowerCamelCase)
    for split in splits:
        lowercase__ :   Optional[Any]    					=				dataset_dict[split]
        assert dataset.num_rows == 4
        assert dataset.num_columns == 3
        assert dataset.column_names == ["col_1", "col_2", "col_3"]
        for feature, expected_dtype in expected_features.items():
            assert dataset.features[feature].dtype == expected_dtype

@pytest.mark.parametrize("keep_in_memory"		,	[False, True])
def 			lowercase_				(			_lowerCamelCase							:       Union[str, Any]		,	_lowerCamelCase							:       str		,	_lowerCamelCase							:       str):
    lowercase__ :   List[str]    					=				tmp_path / "cache"
    lowercase__ :   str    					=				{"col_1": "string", "col_2": "int64", "col_3": "float64"}
    with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
        lowercase__ :   Optional[Any]    					=				JsonDatasetReader({"train": jsonl_path}		,	cache_dir=_lowerCamelCase		,	keep_in_memory=_lowerCamelCase).read()
    _check_json_datasetdict(_lowerCamelCase		,	_lowerCamelCase)

@pytest.mark.parametrize(
    "features"		,	[
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]		,	)
def 			lowercase_				(			_lowerCamelCase							:       Any		,	_lowerCamelCase							:       List[str]		,	_lowerCamelCase							:       List[str]):
    lowercase__ :   str    					=				tmp_path / "cache"
    lowercase__ :   Tuple    					=				{"col_1": "string", "col_2": "int64", "col_3": "float64"}
    lowercase__ :   Tuple    					=				features.copy() if features else default_expected_features
    lowercase__ :   Union[str, Any]    					=				(
        Features({feature: Value(_lowerCamelCase) for feature, dtype in features.items()}) if features is not None else None
    )
    lowercase__ :   Tuple    					=				JsonDatasetReader({"train": jsonl_path}		,	features=_lowerCamelCase		,	cache_dir=_lowerCamelCase).read()
    _check_json_datasetdict(_lowerCamelCase		,	_lowerCamelCase)

@pytest.mark.parametrize("split"		,	[None, NamedSplit("train"), "train", "test"])
def 			lowercase_				(			_lowerCamelCase							:       str		,	_lowerCamelCase							:       Dict		,	_lowerCamelCase							:       Tuple):
    if split:
        lowercase__ :   Tuple    					=				{split: jsonl_path}
    else:
        lowercase__ :   Tuple    					=				"train"
        lowercase__ :   int    					=				{"train": jsonl_path, "test": jsonl_path}
    lowercase__ :   Dict    					=				tmp_path / "cache"
    lowercase__ :   Union[str, Any]    					=				{"col_1": "string", "col_2": "int64", "col_3": "float64"}
    lowercase__ :   Union[str, Any]    					=				JsonDatasetReader(_lowerCamelCase		,	cache_dir=_lowerCamelCase).read()
    _check_json_datasetdict(_lowerCamelCase		,	_lowerCamelCase		,	splits=list(path.keys()))
    assert all(dataset[split].split == split for split in path.keys())

def 			lowercase_				(			_lowerCamelCase							:       Union[str, Any]):
    return json.load(_lowerCamelCase)

def 			lowercase_				(			_lowerCamelCase							:       Optional[int]):
    return [json.loads(_lowerCamelCase) for line in buffer]



class   snake_case_       :






   @pytest.mark.parametrize("lines, load_json_function"	,						[(True, load_json_lines), (False, load_json)]  )
   def       __UpperCamelCase     (      self				:     List[Any]	,						lowercase_				:     Any	,						lowercase_				:     Optional[Any]	,						lowercase_				:     Dict  )					->						Optional[Any]:
       with io.BytesIO() as buffer:
           JsonDatasetWriter(lowercase_	,						lowercase_	,						lines=lowercase_  ).write()
           buffer.seek(0  )
           lowercase__ :   Optional[int]    					=				load_json_function(lowercase_  )
       assert isinstance(lowercase_	,						lowercase_  )
       assert isinstance(exported_content[0]	,						lowercase_  )
       assert len(lowercase_  ) == 10






   @pytest.mark.parametrize(
       "orient, container, keys, len_at"	,						[
           ("records", list, {"tokens", "labels", "answers", "id"}, None),
           ("split", dict, {"columns", "data"}, "data"),
           ("index", dict, set("0123456789"  ), None),
           ("columns", dict, {"tokens", "labels", "answers", "id"}, "tokens"),
           ("values", list, None, None),
           ("table", dict, {"schema", "data"}, "data"),
       ]	,						)
   def       __UpperCamelCase     (      self				:     str	,						lowercase_				:     int	,						lowercase_				:     str	,						lowercase_				:     Union[str, Any]	,						lowercase_				:     Dict	,						lowercase_				:     Tuple  )					->						List[str]:
       with io.BytesIO() as buffer:
           JsonDatasetWriter(lowercase_	,						lowercase_	,						lines=lowercase_	,						orient=lowercase_  ).write()
           buffer.seek(0  )
           lowercase__ :   str    					=				load_json(lowercase_  )
       assert isinstance(lowercase_	,						lowercase_  )
       if keys:
           if container is dict:
               assert exported_content.keys() == keys
           else:
               assert exported_content[0].keys() == keys
       else:
           assert not hasattr(lowercase_	,						"keys"  ) and not hasattr(exported_content[0]	,						"keys"  )
       if len_at:
           assert len(exported_content[len_at]  ) == 10
       else:
           assert len(lowercase_  ) == 10






   @pytest.mark.parametrize("lines, load_json_function"	,						[(True, load_json_lines), (False, load_json)]  )
   def       __UpperCamelCase     (      self				:     List[Any]	,						lowercase_				:     int	,						lowercase_				:     Optional[Any]	,						lowercase_				:     Dict  )					->						Optional[int]:
       with io.BytesIO() as buffer:
           JsonDatasetWriter(lowercase_	,						lowercase_	,						lines=lowercase_	,						num_proc=2  ).write()
           buffer.seek(0  )
           lowercase__ :   str    					=				load_json_function(lowercase_  )
       assert isinstance(lowercase_	,						lowercase_  )
       assert isinstance(exported_content[0]	,						lowercase_  )
       assert len(lowercase_  ) == 10






   @pytest.mark.parametrize(
       "orient, container, keys, len_at"	,						[
           ("records", list, {"tokens", "labels", "answers", "id"}, None),
           ("split", dict, {"columns", "data"}, "data"),
           ("index", dict, set("0123456789"  ), None),
           ("columns", dict, {"tokens", "labels", "answers", "id"}, "tokens"),
           ("values", list, None, None),
           ("table", dict, {"schema", "data"}, "data"),
       ]	,						)
   def       __UpperCamelCase     (      self				:     Union[str, Any]	,						lowercase_				:     Dict	,						lowercase_				:     Dict	,						lowercase_				:     Optional[int]	,						lowercase_				:     Optional[Any]	,						lowercase_				:     Dict  )					->						Any:
       with io.BytesIO() as buffer:
           JsonDatasetWriter(lowercase_	,						lowercase_	,						lines=lowercase_	,						orient=lowercase_	,						num_proc=2  ).write()
           buffer.seek(0  )
           lowercase__ :   Optional[Any]    					=				load_json(lowercase_  )
       assert isinstance(lowercase_	,						lowercase_  )
       if keys:
           if container is dict:
               assert exported_content.keys() == keys
           else:
               assert exported_content[0].keys() == keys
       else:
           assert not hasattr(lowercase_	,						"keys"  ) and not hasattr(exported_content[0]	,						"keys"  )
       if len_at:
           assert len(exported_content[len_at]  ) == 10
       else:
           assert len(lowercase_  ) == 10






   def       __UpperCamelCase     (      self				:     Dict	,						lowercase_				:     List[str]  )					->						str:
       with pytest.raises(lowercase_  ):
           with io.BytesIO() as buffer:
               JsonDatasetWriter(lowercase_	,						lowercase_	,						num_proc=0  )







   @pytest.mark.parametrize("compression, extension"	,						[("gzip", "gz"), ("bz2", "bz2"), ("xz", "xz")]  )
   def       __UpperCamelCase     (      self				:     List[Any]	,						lowercase_				:     Tuple	,						lowercase_				:     List[str]	,						lowercase_				:     Optional[Any]	,						lowercase_				:     str	,						lowercase_				:     List[Any]  )					->						Any:
       lowercase__ :   Dict    					=				tmp_path_factory.mktemp("data"  ) / F'''test.json.{extension}'''
       lowercase__ :   Optional[int]    					=				str(shared_datadir / F'''test_file.json.{extension}'''  )
       JsonDatasetWriter(lowercase_	,						lowercase_	,						compression=lowercase_  ).write()

       with fsspec.open(lowercase_	,						"rb"	,						compression="infer"  ) as f:
           lowercase__ :   List[Any]    					=				f.read()
       with fsspec.open(lowercase_	,						"rb"	,						compression="infer"  ) as f:
           lowercase__ :   str    					=				f.read()
       assert exported_content == original_content



