"""simple docstring"""







import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


__SCREAMING_SNAKE_CASE    	=datasets.utils.logging.get_logger(__name__)







@dataclass
class        UpperCamelCase		(			datasets.BuilderConfig  ):
   lowercase							=  1_0_0_0_0
   lowercase							=  None
   lowercase							=  None







class        UpperCamelCase		(			datasets.ArrowBasedBuilder  ):
   lowercase							=  ParquetConfig
   def 			_UpperCAmelCase   (			self			)	->      int:


        '''simple docstring'''


        return datasets.DatasetInfo(features=self.config.features			)
   def 			_UpperCAmelCase   (			self     ,__UpperCamelCase			)	->      Any:


        '''simple docstring'''


        if not self.config.data_files:
             raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}'''			)
        lowercase_	:				Union[str, Any]													=	dl_manager.download_and_extract(self.config.data_files			)
        if isinstance(_lowerCamelCase     ,(str, list, tuple)			):
             lowercase_	:				Dict													=	data_files
             if isinstance(_lowerCamelCase     ,_lowerCamelCase			):
                  lowercase_	:				List[Any]													=	[files]
             # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
             lowercase_	:				List[Any]													=	[dl_manager.iter_files(_lowerCamelCase			) for file in files]
             return [datasets.SplitGenerator(name=datasets.Split.TRAIN     ,gen_kwargs={'files': files}			)]
        lowercase_	:				List[str]													=	[]
        for split_name, files in data_files.items():
             if isinstance(_lowerCamelCase     ,_lowerCamelCase			):
                  lowercase_	:				Dict													=	[files]
             # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
             lowercase_	:				Any													=	[dl_manager.iter_files(_lowerCamelCase			) for file in files]
             # Infer features is they are stoed in the arrow schema
             if self.info.features is None:
                  for file in itertools.chain.from_iterable(_lowerCamelCase			):
                       with open(_lowerCamelCase     ,'rb'			) as f:
                            lowercase_	:				int													=	datasets.Features.from_arrow_schema(pq.read_schema(_lowerCamelCase			)			)
                       break
             splits.append(datasets.SplitGenerator(name=_lowerCamelCase     ,gen_kwargs={'files': files}			)			)
        return splits
   def 			_UpperCAmelCase   (			self     ,__UpperCamelCase			)	->      Dict:


        '''simple docstring'''


        if self.info.features is not None:
             # more expensive cast to support nested features with keys in a different order
             # allows str <-> int/float or str to Audio for example
             lowercase_	:				Dict													=	table_cast(_lowerCamelCase     ,self.info.features.arrow_schema			)
        return pa_table







   def 			_UpperCAmelCase   (			self     ,__UpperCamelCase			)	->      Optional[Any]:


        '''simple docstring'''


        lowercase_	:				str													=	self.info.features.arrow_schema if self.info.features is not None else None
        if self.info.features is not None and self.config.columns is not None:
             if sorted(field.name for field in schema			) != sorted(self.config.columns			):
                  raise ValueError(
                      f'''Tried to load parquet data with columns \'{self.config.columns}\' with mismatching features \'{self.info.features}\''''			)
        for file_idx, file in enumerate(itertools.chain.from_iterable(_lowerCamelCase			)			):
             with open(_lowerCamelCase     ,'rb'			) as f:
                  lowercase_	:				str													=	pq.ParquetFile(_lowerCamelCase			)
                  try:
                       for batch_idx, record_batch in enumerate(
                           parquet_file.iter_batches(batch_size=self.config.batch_size     ,columns=self.config.columns			)			):
                            lowercase_	:				int													=	pa.Table.from_batches([record_batch]			)
                            # Uncomment for debugging (will print the Arrow table size and elements)
                            # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                            # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                            yield f'''{file_idx}_{batch_idx}''', self._cast_table(_lowerCamelCase			)
                  except ValueError as e:
                       logger.error(f'''Failed to read file \'{file}\' with error {type(_lowerCamelCase			)}: {e}'''			)
                       raise





import sys

from .dependency_versions_table import deps
from .utils.versions import require_version, require_version_core


# define which module versions we always want to check at run time
# (usually the ones defined in `install_requires` in setup.py)
#
# order specific notes:
# - tqdm must be checked before tokenizers

snake_case				:    Union[str, Any]       				=     '''python tqdm regex requests packaging filelock numpy tokenizers'''.split()
if sys.version_info < (3, 7):
     pkgs_to_check_at_runtime.append('''dataclasses''')
if sys.version_info < (3, 8):
     pkgs_to_check_at_runtime.append('''importlib_metadata''')

for pkg in pkgs_to_check_at_runtime:
     if pkg in deps:
          if pkg == "tokenizers":
               # must be loaded here, or else tqdm check may fail
               from .utils import is_tokenizers_available

               if not is_tokenizers_available():
                    continue  # not required, check version only if installed

          require_version_core(deps[pkg])
     else:
          raise ValueError(F"""can't find {pkg} in {deps.keys()}, check dependency_versions_table.py""")
def 				__lowerCamelCase						( UpperCAmelCase_						:		Any ,  UpperCAmelCase_						:		Optional[int]=None						):




      """simple docstring"""





      require_version(deps[pkg] ,  UpperCAmelCase_						)





