


import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
    import sqlitea

    import sqlalchemy



class A__     ( A__  ):



  """simple docstring"""



  def __init__( self			:  Dict				,				lowerCamelCase__			:  Union[str, "sqlalchemy.sql.Selectable"]				,				lowerCamelCase__			:  Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"]				,				lowerCamelCase__			:  Optional[Features] = None				,				lowerCamelCase__			:  str = None				,				lowerCamelCase__			:  bool = False				,				**lowerCamelCase__			:  Optional[int]				,				):
        super().__init__(features=lowerCamelCase__				,				cache_dir=lowerCamelCase__				,				keep_in_memory=lowerCamelCase__				,				**lowerCamelCase__       )
        a__	:  str     =   Sql(
            cache_dir=lowerCamelCase__				,				features=lowerCamelCase__				,				sql=lowerCamelCase__				,				con=lowerCamelCase__				,				**lowerCamelCase__				,				)

  def    _UpperCamelCase( self			:  Tuple       ):
        a__	:  Optional[Any]     =   None
        a__	:  Dict     =   None
        a__	:  Union[str, Any]     =   None
        a__	:  Union[str, Any]     =   None

        self.builder.download_and_prepare(
            download_config=lowerCamelCase__				,				download_mode=lowerCamelCase__				,				verification_mode=lowerCamelCase__				,				base_path=lowerCamelCase__				,				)

        # Build dataset for splits
        a__	:  List[str]     =   self.builder.as_dataset(
            split="train"				,				verification_mode=lowerCamelCase__				,				in_memory=self.keep_in_memory       )
        return dataset




class A__     :



  """simple docstring"""



  def __init__( self			:  List[Any]				,				lowerCamelCase__			:  Dataset				,				lowerCamelCase__			:  str				,				lowerCamelCase__			:  Union[str, "sqlalchemy.engine.Connection", "sqlalchemy.engine.Engine", "sqlite3.Connection"]				,				lowerCamelCase__			:  Optional[int] = None				,				lowerCamelCase__			:  Optional[int] = None				,				**lowerCamelCase__			:  Optional[Any]				,				):
        if num_proc is not None and num_proc <= 0:
              raise ValueError(f'''num_proc {num_proc} must be an integer > 0.'''       )

        a__	:  Any     =   dataset
        a__	:  str     =   name
        a__	:  Tuple     =   con
        a__	:  List[Any]     =   batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
        a__	:  Any     =   num_proc
        a__	:  Tuple     =   to_sql_kwargs

  def    _UpperCamelCase( self			:  List[Any]       ):
        a__	:  Any     =   self.to_sql_kwargs.pop("sql"				,				lowerCamelCase__       )
        a__	:  int     =   self.to_sql_kwargs.pop("con"				,				lowerCamelCase__       )
        a__	:  int     =   self.to_sql_kwargs.pop("index"				,				lowerCamelCase__       )

        a__	:  int     =   self._write(index=lowerCamelCase__				,				**self.to_sql_kwargs       )
        return written

  def    _UpperCamelCase( self			:  Any				,				lowerCamelCase__			:  List[str]       ):
        a__,	a__,	a__	:  Union[str, Any]     =   args
        a__	:  Any     =   {**to_sql_kwargs, "if_exists": "append"} if offset > 0 else to_sql_kwargs
        a__	:  Tuple     =   query_table(
            table=self.dataset.data				,				key=slice(lowerCamelCase__				,				offset + self.batch_size       )				,				indices=self.dataset._indices				,				)
        a__	:  str     =   batch.to_pandas()
        a__	:  List[Any]     =   df.to_sql(self.name				,				self.con				,				index=lowerCamelCase__				,				**lowerCamelCase__       )
        return num_rows or len(lowerCamelCase__       )

  def    _UpperCamelCase( self			:  Optional[Any]				,				lowerCamelCase__			:  Optional[int]				,				**lowerCamelCase__			:  Optional[Any]       ):
        a__	:  str     =   0

        if self.num_proc is None or self.num_proc == 1:
              for offset in logging.tqdm(
                  range(0				,				len(self.dataset       )				,				self.batch_size       )				,				unit="ba"				,				disable=not logging.is_progress_bar_enabled()				,				desc="Creating SQL from Arrow format"				,				):
                    written += self._batch_sql((offset, index, to_sql_kwargs)       )
        else:
              a__,	a__	:  List[str]     =   len(self.dataset       ), self.batch_size
              with multiprocessing.Pool(self.num_proc       ) as pool:
                    for num_rows in logging.tqdm(
                        pool.imap(
                            self._batch_sql				,				[(offset, index, to_sql_kwargs) for offset in range(0				,				lowerCamelCase__				,				lowerCamelCase__       )]				,				)				,				total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size				,				unit="ba"				,				disable=not logging.is_progress_bar_enabled()				,				desc="Creating SQL from Arrow format"				,				):
                          written += num_rows

        return written


'''simple docstring'''

import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoImageProcessor, ViTImageProcessor
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_image_processing import CustomImageProcessor  # noqa E402


UpperCamelCase__  :						str					=      get_tests_dir('fixtures')


class 					_lowercase						(   unittest.TestCase ):
      '''simple docstring'''


      def 		lowerCAmelCase__       (					self				)      ->      int:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[int] 					=	mock.Mock()
         UpperCAmelCase__				:					Dict 					=	500
         UpperCAmelCase__				:					Optional[int] 					=	{}
         UpperCAmelCase__				:					List[str] 					=	HTTPError
         UpperCAmelCase__				:					Tuple 					=	{}

         # Download this model to make sure it's in the cache.
         UpperCAmelCase__				:					Any 					=	ViTImageProcessor.from_pretrained('''hf-internal-testing/tiny-random-vit'''				)
         # Under the mock environment we get a 500 error when trying to reach the model.
         with mock.patch('''requests.Session.request'''       ,return_value=lowerCamelCase_				) as mock_head:
            UpperCAmelCase__				:					Tuple 					=	ViTImageProcessor.from_pretrained('''hf-internal-testing/tiny-random-vit'''				)
            # This check we did call the fake head request
            mock_head.assert_called()




      def 		lowerCAmelCase__       (					self				)      ->      Tuple:



         '''simple docstring'''



         UpperCAmelCase__				:					Dict 					=	ViTImageProcessor.from_pretrained(
             '''https://huggingface.co/hf-internal-testing/tiny-random-vit/resolve/main/preprocessor_config.json'''				)




      def 		lowerCAmelCase__       (					self				)      ->      Union[str, Any]:



         '''simple docstring'''



         with self.assertRaises(lowerCamelCase_				):
            # config is in subfolder, the following should not work without specifying the subfolder
            UpperCAmelCase__				:					int 					=	AutoImageProcessor.from_pretrained('''hf-internal-testing/stable-diffusion-all-variants'''				)

         UpperCAmelCase__				:					Optional[Any] 					=	AutoImageProcessor.from_pretrained(
             '''hf-internal-testing/stable-diffusion-all-variants'''       ,subfolder='''feature_extractor'''				)

         self.assertIsNotNone(lowerCamelCase_				)




@is_staging_test
class 					_lowercase						(   unittest.TestCase ):
      '''simple docstring'''


      @classmethod
      def 		lowerCAmelCase__       (					cls				)      ->      Optional[Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[int] 					=	TOKEN
         HfFolder.save_token(lowerCamelCase_				)




      @classmethod
      def 		lowerCAmelCase__       (					cls				)      ->      List[str]:



         '''simple docstring'''



         try:
            delete_repo(token=cls._token       ,repo_id='''test-image-processor'''				)
         except HTTPError:
            pass

         try:
            delete_repo(token=cls._token       ,repo_id='''valid_org/test-image-processor-org'''				)
         except HTTPError:
            pass

         try:
            delete_repo(token=cls._token       ,repo_id='''test-dynamic-image-processor'''				)
         except HTTPError:
            pass




      def 		lowerCAmelCase__       (					self				)      ->      Union[str, Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					Optional[Any] 					=	ViTImageProcessor.from_pretrained(lowerCamelCase_				)
         image_processor.push_to_hub('''test-image-processor'''       ,use_auth_token=self._token				)

         UpperCAmelCase__				:					Any 					=	ViTImageProcessor.from_pretrained(f'''{USER}/test-image-processor'''				)
         for k, v in image_processor.__dict__.items():
            self.assertEqual(lowerCamelCase_       ,getattr(lowerCamelCase_       ,lowerCamelCase_				)				)

         # Reset repo
         delete_repo(token=self._token       ,repo_id='''test-image-processor'''				)

         # Push to hub via save_pretrained
         with tempfile.TemporaryDirectory() as tmp_dir:
            image_processor.save_pretrained(
                lowerCamelCase_       ,repo_id='''test-image-processor'''       ,push_to_hub=lowerCamelCase_       ,use_auth_token=self._token				)

         UpperCAmelCase__				:					List[Any] 					=	ViTImageProcessor.from_pretrained(f'''{USER}/test-image-processor'''				)
         for k, v in image_processor.__dict__.items():
            self.assertEqual(lowerCamelCase_       ,getattr(lowerCamelCase_       ,lowerCamelCase_				)				)




      def 		lowerCAmelCase__       (					self				)      ->      List[Any]:



         '''simple docstring'''



         UpperCAmelCase__				:					Union[str, Any] 					=	ViTImageProcessor.from_pretrained(lowerCamelCase_				)
         image_processor.push_to_hub('''valid_org/test-image-processor'''       ,use_auth_token=self._token				)

         UpperCAmelCase__				:					Any 					=	ViTImageProcessor.from_pretrained('''valid_org/test-image-processor'''				)
         for k, v in image_processor.__dict__.items():
            self.assertEqual(lowerCamelCase_       ,getattr(lowerCamelCase_       ,lowerCamelCase_				)				)

         # Reset repo
         delete_repo(token=self._token       ,repo_id='''valid_org/test-image-processor'''				)

         # Push to hub via save_pretrained
         with tempfile.TemporaryDirectory() as tmp_dir:
            image_processor.save_pretrained(
                lowerCamelCase_       ,repo_id='''valid_org/test-image-processor-org'''       ,push_to_hub=lowerCamelCase_       ,use_auth_token=self._token				)

         UpperCAmelCase__				:					Any 					=	ViTImageProcessor.from_pretrained('''valid_org/test-image-processor-org'''				)
         for k, v in image_processor.__dict__.items():
            self.assertEqual(lowerCamelCase_       ,getattr(lowerCamelCase_       ,lowerCamelCase_				)				)




      def 		lowerCAmelCase__       (					self				)      ->      str:



         '''simple docstring'''



         CustomImageProcessor.register_for_auto_class()
         UpperCAmelCase__				:					List[str] 					=	CustomImageProcessor.from_pretrained(lowerCamelCase_				)

         image_processor.push_to_hub('''test-dynamic-image-processor'''       ,use_auth_token=self._token				)

         # This has added the proper auto_map field to the config
         self.assertDictEqual(
             image_processor.auto_map       ,{'''AutoImageProcessor''': '''custom_image_processing.CustomImageProcessor'''}       ,)

         UpperCAmelCase__				:					Any 					=	AutoImageProcessor.from_pretrained(
             f'''{USER}/test-dynamic-image-processor'''       ,trust_remote_code=lowerCamelCase_				)
         # Can't make an isinstance check because the new_image_processor is from the CustomImageProcessor class of a dynamic module
         self.assertEqual(new_image_processor.__class__.__name__       ,'''CustomImageProcessor'''				)




