


"""simple docstring"""







import pyarrow.parquet as pq
import pytest

from datasets import Audio, Dataset, DatasetDict, Features, NamedSplit, Sequence, Value, config
from datasets.features.image import Image
from datasets.io.parquet import ParquetDatasetReader, ParquetDatasetWriter, get_writer_batch_size

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases
def 					a__			(				SCREAMING_SNAKE_CASE    :    str				,      SCREAMING_SNAKE_CASE    :    Dict				):


  '''simple docstring'''







  assert isinstance(SCREAMING_SNAKE_CASE				,      SCREAMING_SNAKE_CASE				)
  assert dataset.num_rows == 4
  assert dataset.num_columns == 3
  assert dataset.column_names == ["col_1", "col_2", "col_3"]
  for feature, expected_dtype in expected_features.items():
    assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize("keep_in_memory"				,      [False, True]				)
def 					a__			(				SCREAMING_SNAKE_CASE    :    List[Any]				,      SCREAMING_SNAKE_CASE    :    List[Any]				,      SCREAMING_SNAKE_CASE    :    Dict				):


  '''simple docstring'''







  lowerCAmelCase     :     List[str]								=							tmp_path / "cache"
  lowerCAmelCase     :     Optional[Any]								=							{"col_1": "string", "col_2": "int64", "col_3": "float64"}
  with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
    lowerCAmelCase     :     List[Any]								=							ParquetDatasetReader(SCREAMING_SNAKE_CASE				,      cache_dir=SCREAMING_SNAKE_CASE				,      keep_in_memory=SCREAMING_SNAKE_CASE				).read()
  _check_parquet_dataset(SCREAMING_SNAKE_CASE				,      SCREAMING_SNAKE_CASE				)
@pytest.mark.parametrize(
    "features"				,      [
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]				,      )
def 					a__			(				SCREAMING_SNAKE_CASE    :    str				,      SCREAMING_SNAKE_CASE    :    List[Any]				,      SCREAMING_SNAKE_CASE    :    Optional[int]				):


  '''simple docstring'''







  lowerCAmelCase     :     Optional[int]								=							tmp_path / "cache"
  lowerCAmelCase     :     str								=							{"col_1": "string", "col_2": "int64", "col_3": "float64"}
  lowerCAmelCase     :     Optional[int]								=							features.copy() if features else default_expected_features
  lowerCAmelCase     :     Tuple								=							(
      Features({feature: Value(SCREAMING_SNAKE_CASE				) for feature, dtype in features.items()}				) if features is not None else None
  )
  lowerCAmelCase     :     Optional[Any]								=							ParquetDatasetReader(SCREAMING_SNAKE_CASE				,      features=SCREAMING_SNAKE_CASE				,      cache_dir=SCREAMING_SNAKE_CASE				).read()
  _check_parquet_dataset(SCREAMING_SNAKE_CASE				,      SCREAMING_SNAKE_CASE				)
@pytest.mark.parametrize("split"				,      [None, NamedSplit("train"				), "train", "test"]				)
def 					a__			(				SCREAMING_SNAKE_CASE    :    List[Any]				,      SCREAMING_SNAKE_CASE    :    str				,      SCREAMING_SNAKE_CASE    :    Optional[Any]				):


  '''simple docstring'''







  lowerCAmelCase     :     int								=							tmp_path / "cache"
  lowerCAmelCase     :     str								=							{"col_1": "string", "col_2": "int64", "col_3": "float64"}
  lowerCAmelCase     :     List[Any]								=							ParquetDatasetReader(SCREAMING_SNAKE_CASE				,      cache_dir=SCREAMING_SNAKE_CASE				,      split=SCREAMING_SNAKE_CASE				).read()
  _check_parquet_dataset(SCREAMING_SNAKE_CASE				,      SCREAMING_SNAKE_CASE				)
  assert dataset.split == split if split else "train"
@pytest.mark.parametrize("path_type"				,      [str, list]				)
def 					a__			(				SCREAMING_SNAKE_CASE    :    List[Any]				,      SCREAMING_SNAKE_CASE    :    Dict				,      SCREAMING_SNAKE_CASE    :    int				):


  '''simple docstring'''







  if issubclass(SCREAMING_SNAKE_CASE				,      SCREAMING_SNAKE_CASE				):
    lowerCAmelCase     :     str								=							parquet_path
  elif issubclass(SCREAMING_SNAKE_CASE				,      SCREAMING_SNAKE_CASE				):
    lowerCAmelCase     :     Tuple								=							[parquet_path]
  lowerCAmelCase     :     List[Any]								=							tmp_path / "cache"
  lowerCAmelCase     :     Dict								=							{"col_1": "string", "col_2": "int64", "col_3": "float64"}
  lowerCAmelCase     :     Optional[int]								=							ParquetDatasetReader(SCREAMING_SNAKE_CASE				,      cache_dir=SCREAMING_SNAKE_CASE				).read()
  _check_parquet_dataset(SCREAMING_SNAKE_CASE				,      SCREAMING_SNAKE_CASE				)
def 					a__			(				SCREAMING_SNAKE_CASE    :    Tuple				,      SCREAMING_SNAKE_CASE    :    str				,      SCREAMING_SNAKE_CASE    :    int=("train",)				):


  '''simple docstring'''







  assert isinstance(SCREAMING_SNAKE_CASE				,      SCREAMING_SNAKE_CASE				)
  for split in splits:
    lowerCAmelCase     :     List[str]								=							dataset_dict[split]
    assert dataset.num_rows == 4
    assert dataset.num_columns == 3
    assert dataset.column_names == ["col_1", "col_2", "col_3"]
    for feature, expected_dtype in expected_features.items():
      assert dataset.features[feature].dtype == expected_dtype
@pytest.mark.parametrize("keep_in_memory"				,      [False, True]				)
def 					a__			(				SCREAMING_SNAKE_CASE    :    Optional[int]				,      SCREAMING_SNAKE_CASE    :    List[str]				,      SCREAMING_SNAKE_CASE    :    int				):


  '''simple docstring'''







  lowerCAmelCase     :     List[str]								=							tmp_path / "cache"
  lowerCAmelCase     :     Any								=							{"col_1": "string", "col_2": "int64", "col_3": "float64"}
  with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
    lowerCAmelCase     :     Union[str, Any]								=							ParquetDatasetReader(
        {"train": parquet_path}				,      cache_dir=SCREAMING_SNAKE_CASE				,      keep_in_memory=SCREAMING_SNAKE_CASE				).read()
  _check_parquet_datasetdict(SCREAMING_SNAKE_CASE				,      SCREAMING_SNAKE_CASE				)
@pytest.mark.parametrize(
    "features"				,      [
        None,
        {"col_1": "string", "col_2": "int64", "col_3": "float64"},
        {"col_1": "string", "col_2": "string", "col_3": "string"},
        {"col_1": "int32", "col_2": "int32", "col_3": "int32"},
        {"col_1": "float32", "col_2": "float32", "col_3": "float32"},
    ]				,      )
def 					a__			(				SCREAMING_SNAKE_CASE    :    Tuple				,      SCREAMING_SNAKE_CASE    :    Optional[Any]				,      SCREAMING_SNAKE_CASE    :    Tuple				):


  '''simple docstring'''







  lowerCAmelCase     :     List[Any]								=							tmp_path / "cache"
  lowerCAmelCase     :     int								=							{"col_1": "string", "col_2": "int64", "col_3": "float64"}
  lowerCAmelCase     :     Optional[int]								=							features.copy() if features else default_expected_features
  lowerCAmelCase     :     int								=							(
      Features({feature: Value(SCREAMING_SNAKE_CASE				) for feature, dtype in features.items()}				) if features is not None else None
  )
  lowerCAmelCase     :     Tuple								=							ParquetDatasetReader({"train": parquet_path}				,      features=SCREAMING_SNAKE_CASE				,      cache_dir=SCREAMING_SNAKE_CASE				).read()
  _check_parquet_datasetdict(SCREAMING_SNAKE_CASE				,      SCREAMING_SNAKE_CASE				)
@pytest.mark.parametrize("split"				,      [None, NamedSplit("train"				), "train", "test"]				)
def 					a__			(				SCREAMING_SNAKE_CASE    :    List[Any]				,      SCREAMING_SNAKE_CASE    :    List[str]				,      SCREAMING_SNAKE_CASE    :    Optional[Any]				):


  '''simple docstring'''







  if split:
    lowerCAmelCase     :     Any								=							{split: parquet_path}
  else:
    lowerCAmelCase     :     Optional[int]								=							"train"
    lowerCAmelCase     :     Union[str, Any]								=							{"train": parquet_path, "test": parquet_path}
  lowerCAmelCase     :     Tuple								=							tmp_path / "cache"
  lowerCAmelCase     :     str								=							{"col_1": "string", "col_2": "int64", "col_3": "float64"}
  lowerCAmelCase     :     Union[str, Any]								=							ParquetDatasetReader(SCREAMING_SNAKE_CASE				,      cache_dir=SCREAMING_SNAKE_CASE				).read()
  _check_parquet_datasetdict(SCREAMING_SNAKE_CASE				,      SCREAMING_SNAKE_CASE				,      splits=list(path.keys()				)				)
  assert all(dataset[split].split == split for split in path.keys()				)
def 					a__			(				SCREAMING_SNAKE_CASE    :    str				,      SCREAMING_SNAKE_CASE    :    List[str]				):


  '''simple docstring'''







  lowerCAmelCase     :     Optional[Any]								=							ParquetDatasetWriter(SCREAMING_SNAKE_CASE				,      tmp_path / "foo.parquet"				)
  assert writer.write() > 0
  lowerCAmelCase     :     List[Any]								=							pq.ParquetFile(tmp_path / "foo.parquet"				)
  lowerCAmelCase     :     Optional[int]								=							pf.read()
  assert dataset.data.table == output_table
def 					a__			(				SCREAMING_SNAKE_CASE    :    List[Any]				,      SCREAMING_SNAKE_CASE    :    Dict				):


  '''simple docstring'''







  lowerCAmelCase     :     Optional[int]								=							str(shared_datadir / "test_image_rgb.jpg"				)
  lowerCAmelCase     :     Optional[Any]								=							{"image": [image_path]}
  lowerCAmelCase     :     List[str]								=							Features({"image": Image()}				)
  lowerCAmelCase     :     List[Any]								=							Dataset.from_dict(SCREAMING_SNAKE_CASE				,      features=SCREAMING_SNAKE_CASE				)
  lowerCAmelCase     :     Tuple								=							ParquetDatasetWriter(SCREAMING_SNAKE_CASE				,      tmp_path / "foo.parquet"				)
  assert writer.write() > 0

  lowerCAmelCase     :     Optional[int]								=							Dataset.from_parquet(str(tmp_path / "foo.parquet"				)				)
  assert dataset.features == reloaded_dataset.features

  lowerCAmelCase     :     Tuple								=							ParquetDatasetReader(str(tmp_path / "foo.parquet"				)				,      streaming=SCREAMING_SNAKE_CASE				).read()
  assert dataset.features == reloaded_iterable_dataset.features





@pytest.mark.parametrize(
    "feature, expected"				,      [
        (Features({"foo": Value("int32"				)}				), None),
        (Features({"image": Image(), "foo": Value("int32"				)}				), config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS),
        (Features({"nested": Sequence(Audio()				)}				), config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS),
    ]				,      )
def 					a__			(				SCREAMING_SNAKE_CASE    :    Dict				,      SCREAMING_SNAKE_CASE    :    Union[str, Any]				):


  '''simple docstring'''







  assert get_writer_batch_size(SCREAMING_SNAKE_CASE				) == expected









"""simple docstring"""







import unittest

from huggingface_hub import hf_hub_download

from transformers import MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING, VideoMAEFeatureExtractor
from transformers.pipelines import VideoClassificationPipeline, pipeline
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_decord,
    require_tf,
    require_torch,
    require_torch_or_tf,
    require_vision,
)

from .test_pipelines_common import ANY


@is_pipeline_test
@require_torch_or_tf
@require_vision
@require_decord
class       SCREAMING_SNAKE_CASE__				(		unittest.TestCase				):
       """simple docstring"""


       a  :				int   =MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING

       def        lowercase__							(						self		,     snake_case__		,     snake_case__		,     snake_case__						):






         """simple docstring"""





         lowerCAmelCase     :     Tuple								=							hf_hub_download(
             repo_id="nateraw/video-demo"		,     filename="archery.mp4"		,     repo_type="dataset"						)
         lowerCAmelCase     :     Dict								=							VideoClassificationPipeline(model=snake_case__		,     image_processor=snake_case__		,     top_k=2						)
         lowerCAmelCase     :     Any								=							[
             example_video_filepath,
             "https://huggingface.co/datasets/nateraw/video-demo/resolve/main/archery.mp4",
         ]
         return video_classifier, examples

       def        lowercase__							(						self		,     snake_case__		,     snake_case__						):






         """simple docstring"""





         for example in examples:
           lowerCAmelCase     :     str								=							video_classifier(snake_case__						)

           self.assertEqual(
               snake_case__		,     [
                   {"score": ANY(snake_case__						), "label": ANY(snake_case__						)},
                   {"score": ANY(snake_case__						), "label": ANY(snake_case__						)},
               ]		,     )

       @require_torch
       def        lowercase__							(						self						):






         """simple docstring"""





         lowerCAmelCase     :     Union[str, Any]								=							"hf-internal-testing/tiny-random-VideoMAEForVideoClassification"
         lowerCAmelCase     :     str								=							VideoMAEFeatureExtractor(
             size={"shortest_edge": 10}		,     crop_size={"height": 10, "width": 10}						)
         lowerCAmelCase     :     int								=							pipeline(
             "video-classification"		,     model=snake_case__		,     feature_extractor=snake_case__		,     frame_sampling_rate=4						)

         lowerCAmelCase     :     Optional[int]								=							hf_hub_download(repo_id="nateraw/video-demo"		,     filename="archery.mp4"		,     repo_type="dataset"						)
         lowerCAmelCase     :     Union[str, Any]								=							video_classifier(snake_case__		,     top_k=2						)
         self.assertEqual(
             nested_simplify(snake_case__		,     decimals=4						)		,     [{"score": 0.5199, "label": "LABEL_0"}, {"score": 0.4801, "label": "LABEL_1"}]		,     )

         lowerCAmelCase     :     Tuple								=							video_classifier(
             [
                 video_file_path,
                 video_file_path,
             ]		,     top_k=2		,     )
         self.assertEqual(
             nested_simplify(snake_case__		,     decimals=4						)		,     [
                 [{"score": 0.5199, "label": "LABEL_0"}, {"score": 0.4801, "label": "LABEL_1"}],
                 [{"score": 0.5199, "label": "LABEL_0"}, {"score": 0.4801, "label": "LABEL_1"}],
             ]		,     )







       @require_tf
       def        lowercase__							(						self						):






         """simple docstring"""





         pass





