'''simple docstring'''




import inspect
import re
from hashlib import shaaaa
from typing import Dict, List

from .arrow import arrow
from .audiofolder import audiofolder
from .csv import csv
from .imagefolder import imagefolder
from .json import json
from .pandas import pandas
from .parquet import parquet
from .sql import sql  # noqa F401
from .text import text

def 	UpperCamelCase_							(					_UpperCAmelCase  :      List[str]    )    ->    str:


       """simple docstring"""







       _UpperCAmelCase     :  Tuple							      =    []
       for line in lines:
              _UpperCAmelCase     :  Optional[Any]							      =    re.sub(R"#.*"			,	""			,	_UpperCAmelCase    )  # remove comments
              if line:
                     filtered_lines.append(_UpperCAmelCase    )
       _UpperCAmelCase     :  Optional[int]							      =    "\n".join(_UpperCAmelCase    )

       # Make a hash from all this code
       _UpperCAmelCase     :  Optional[int]							      =    full_str.encode("utf-8"    )
       return shaaaa(_UpperCAmelCase    ).hexdigest()


# get importable module names and hash for caching
__SCREAMING_SNAKE_CASE       :     Optional[Any]     =  {
    """csv""": (csv.__name__, _hash_python_lines(inspect.getsource(csv).splitlines())),
    """json""": (json.__name__, _hash_python_lines(inspect.getsource(json).splitlines())),
    """pandas""": (pandas.__name__, _hash_python_lines(inspect.getsource(pandas).splitlines())),
    """parquet""": (parquet.__name__, _hash_python_lines(inspect.getsource(parquet).splitlines())),
    """arrow""": (arrow.__name__, _hash_python_lines(inspect.getsource(arrow).splitlines())),
    """text""": (text.__name__, _hash_python_lines(inspect.getsource(text).splitlines())),
    """imagefolder""": (imagefolder.__name__, _hash_python_lines(inspect.getsource(imagefolder).splitlines())),
    """audiofolder""": (audiofolder.__name__, _hash_python_lines(inspect.getsource(audiofolder).splitlines())),
}

# Used to infer the module to use based on the data files extensions
__SCREAMING_SNAKE_CASE       :     Tuple     =  {
    """.csv""": ("""csv""", {}),
    """.tsv""": ("""csv""", {"""sep""": """\t"""}),
    """.json""": ("""json""", {}),
    """.jsonl""": ("""json""", {}),
    """.parquet""": ("""parquet""", {}),
    """.arrow""": ("""arrow""", {}),
    """.txt""": ("""text""", {}),
}
_EXTENSION_TO_MODULE.update({ext: ("""imagefolder""", {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ("""imagefolder""", {}) for ext in imagefolder.ImageFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext: ("""audiofolder""", {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
_EXTENSION_TO_MODULE.update({ext.upper(): ("""audiofolder""", {}) for ext in audiofolder.AudioFolder.EXTENSIONS})
__SCREAMING_SNAKE_CASE       :     str     =  {"""imagefolder""", """audiofolder"""}

# Used to filter data files based on extensions given a module name
__SCREAMING_SNAKE_CASE       :     Dict[str, List[str]]     =  {}
for _ext, (_module, _) in _EXTENSION_TO_MODULE.items():
  _MODULE_TO_EXTENSIONS.setdefault(_module, []).append(_ext)

_MODULE_TO_EXTENSIONS["imagefolder"].append(""".zip""")


_MODULE_TO_EXTENSIONS["audiofolder"].append(""".zip""")





"""simple docstring"""




import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import datasets
import datasets.config

from .utils import require_beam
class 					lowerCAmelCase	(					datasets.BeamBasedBuilder    ):




		'''simple docstring'''







		def 							__A						(	self				)     ->							Dict:
								return datasets.DatasetInfo(
								    features=datasets.Features({'content': datasets.Value('string'				)}				)						,       supervised_keys=lowerCAmelCase__						,       )







		def 							__A						(	self						,       lowerCAmelCase__						,       lowerCAmelCase__				)     ->							str:
								return [datasets.SplitGenerator(name=datasets.Split.TRAIN						,       gen_kwargs={'examples': get_test_dummy_examples()}				)]







		def 							__A						(	self						,       lowerCAmelCase__						,       lowerCAmelCase__				)     ->							Optional[int]:
								import apache_beam as beam

								return pipeline | "Load Examples" >> beam.Create(lowerCAmelCase__				)





class 					lowerCAmelCase	(					datasets.BeamBasedBuilder    ):




		'''simple docstring'''







		def 							__A						(	self				)     ->							int:
								return datasets.DatasetInfo(
								    features=datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string'				)}				)}				)						,       supervised_keys=lowerCAmelCase__						,       )







		def 							__A						(	self						,       lowerCAmelCase__						,       lowerCAmelCase__				)     ->							Dict:
								return [
								    datasets.SplitGenerator(name=datasets.Split.TRAIN						,       gen_kwargs={'examples': get_test_nested_examples()}				)
								]







		def 							__A						(	self						,       lowerCAmelCase__						,       lowerCAmelCase__				)     ->							Optional[Any]:
								import apache_beam as beam

								return pipeline | "Load Examples" >> beam.Create(lowerCAmelCase__				)





def     lowercase			()   ->						str:
						return [(i, {"content": content}) for i, content in enumerate(['foo', 'bar', 'foobar']  )]


def     lowercase			()   ->						Optional[Any]:
						return [(i, {"a": {"b": [content]}}) for i, content in enumerate(['foo', 'bar', 'foobar']  )]


class 					lowerCAmelCase	(					lowerCamelCase_    ):




		'''simple docstring'''







		@require_beam
		def 							__A						(	self				)     ->							Optional[int]:
								SCREAMING_SNAKE_CASE													=					len(get_test_dummy_examples()				)
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
														SCREAMING_SNAKE_CASE													=					DummyBeamDataset(cache_dir=lowerCAmelCase__						,       beam_runner='DirectRunner'				)
														builder.download_and_prepare()
														self.assertTrue(
														    os.path.exists(
														        os.path.join(lowerCAmelCase__						,       builder.name						,       'default'						,       '0.0.0'						,       F'{builder.name}-train.arrow'				)				)				)
														self.assertDictEqual(builder.info.features						,       datasets.Features({'content': datasets.Value('string'				)}				)				)
														SCREAMING_SNAKE_CASE													=					builder.as_dataset()
														self.assertEqual(dset['train'].num_rows						,       lowerCAmelCase__				)
														self.assertEqual(dset['train'].info.splits['train'].num_examples						,       lowerCAmelCase__				)
														self.assertDictEqual(dset['train'][0]						,       get_test_dummy_examples()[0][1]				)
														self.assertDictEqual(
														    dset['train'][expected_num_examples - 1]						,       get_test_dummy_examples()[expected_num_examples - 1][1]				)
														self.assertTrue(
														    os.path.exists(os.path.join(lowerCAmelCase__						,       builder.name						,       'default'						,       '0.0.0'						,       'dataset_info.json'				)				)				)
														del dset







		@require_beam
		def 							__A						(	self				)     ->							int:
								import apache_beam as beam

								SCREAMING_SNAKE_CASE													=					beam.io.parquetio.WriteToParquet

								SCREAMING_SNAKE_CASE													=					len(get_test_dummy_examples()				)
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
														SCREAMING_SNAKE_CASE													=					DummyBeamDataset(cache_dir=lowerCAmelCase__						,       beam_runner='DirectRunner'				)
														with patch('apache_beam.io.parquetio.WriteToParquet'				) as write_parquet_mock:
																				SCREAMING_SNAKE_CASE													=					partial(lowerCAmelCase__						,       num_shards=2				)
																				builder.download_and_prepare()
														self.assertTrue(
														    os.path.exists(
														        os.path.join(
														            lowerCAmelCase__						,       builder.name						,       'default'						,       '0.0.0'						,       F'{builder.name}-train-00000-of-00002.arrow'				)				)				)
														self.assertTrue(
														    os.path.exists(
														        os.path.join(
														            lowerCAmelCase__						,       builder.name						,       'default'						,       '0.0.0'						,       F'{builder.name}-train-00000-of-00002.arrow'				)				)				)
														self.assertDictEqual(builder.info.features						,       datasets.Features({'content': datasets.Value('string'				)}				)				)
														SCREAMING_SNAKE_CASE													=					builder.as_dataset()
														self.assertEqual(dset['train'].num_rows						,       lowerCAmelCase__				)
														self.assertEqual(dset['train'].info.splits['train'].num_examples						,       lowerCAmelCase__				)
														# Order is not preserved when sharding, so we just check that all the elements are there
														self.assertListEqual(sorted(dset['train']['content']				)						,       sorted(['foo', 'bar', 'foobar']				)				)
														self.assertTrue(
														    os.path.exists(os.path.join(lowerCAmelCase__						,       builder.name						,       'default'						,       '0.0.0'						,       'dataset_info.json'				)				)				)
														del dset







		@require_beam
		def 							__A						(	self				)     ->							Optional[Any]:
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
														SCREAMING_SNAKE_CASE													=					DummyBeamDataset(cache_dir=lowerCAmelCase__				)
														self.assertRaises(datasets.builder.MissingBeamOptions						,       builder.download_and_prepare				)







		@require_beam
		def 							__A						(	self				)     ->							int:
								SCREAMING_SNAKE_CASE													=					len(get_test_nested_examples()				)
								with tempfile.TemporaryDirectory() as tmp_cache_dir:
														SCREAMING_SNAKE_CASE													=					NestedBeamDataset(cache_dir=lowerCAmelCase__						,       beam_runner='DirectRunner'				)
														builder.download_and_prepare()
														self.assertTrue(
														    os.path.exists(
														        os.path.join(lowerCAmelCase__						,       builder.name						,       'default'						,       '0.0.0'						,       F'{builder.name}-train.arrow'				)				)				)
														self.assertDictEqual(
														    builder.info.features						,       datasets.Features({'a': datasets.Sequence({'b': datasets.Value('string'				)}				)}				)				)
														SCREAMING_SNAKE_CASE													=					builder.as_dataset()
														self.assertEqual(dset['train'].num_rows						,       lowerCAmelCase__				)
														self.assertEqual(dset['train'].info.splits['train'].num_examples						,       lowerCAmelCase__				)
														self.assertDictEqual(dset['train'][0]						,       get_test_nested_examples()[0][1]				)
														self.assertDictEqual(
														    dset['train'][expected_num_examples - 1]						,       get_test_nested_examples()[expected_num_examples - 1][1]				)
														self.assertTrue(
														    os.path.exists(os.path.join(lowerCAmelCase__						,       builder.name						,       'default'						,       '0.0.0'						,       'dataset_info.json'				)				)				)
														del dset


