





from __future__ import annotations

import copy
import tempfile
import unittest

from transformers import CONFIG_MAPPING, AutoConfig, BertConfig, GPTaConfig, TaConfig, TapasConfig, is_tf_available
from transformers.testing_utils import (
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tensorflow_probability,
    require_tf,
    slow,
)

from ..bert.test_modeling_bert import BertModelTester


if is_tf_available():
 from transformers import (
     TFAutoModel,
     TFAutoModelForCausalLM,
     TFAutoModelForMaskedLM,
     TFAutoModelForPreTraining,
     TFAutoModelForQuestionAnswering,
     TFAutoModelForSeqaSeqLM,
     TFAutoModelForSequenceClassification,
     TFAutoModelForTableQuestionAnswering,
     TFAutoModelForTokenClassification,
     TFAutoModelWithLMHead,
     TFBertForMaskedLM,
     TFBertForPreTraining,
     TFBertForQuestionAnswering,
     TFBertForSequenceClassification,
     TFBertModel,
     TFFunnelBaseModel,
     TFFunnelModel,
     TFGPTaLMHeadModel,
     TFRobertaForMaskedLM,
     TFTaForConditionalGeneration,
     TFTapasForQuestionAnswering,
 )
 from transformers.models.auto.modeling_tf_auto import (
     TF_MODEL_FOR_CAUSAL_LM_MAPPING,
     TF_MODEL_FOR_MASKED_LM_MAPPING,
     TF_MODEL_FOR_PRETRAINING_MAPPING,
     TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
     TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
     TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
     TF_MODEL_MAPPING,
 )
 from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
 from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
 from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
 from transformers.models.tapas.modeling_tf_tapas import TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST


class     __magic_name__     (							a__							):





     """simple docstring"""



     __UpperCamelCase    			=   "new-model"


if is_tf_available():


 class     __magic_name__     (							a__							):





      """simple docstring"""



      __UpperCamelCase    			=   NewModelConfig


@require_tf
class     __magic_name__     (							unittest.TestCase							):





     """simple docstring"""



     @slow
     def   SCREAMING_SNAKE_CASE						(							self				:Tuple     ):





        '''simple docstring'''




        A_    : Any								=	'bert-base-cased'
        A_    : Tuple								=	AutoConfig.from_pretrained(SCREAMING_SNAKE_CASE_     )
        self.assertIsNotNone(SCREAMING_SNAKE_CASE_     )
        self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )

        A_    : Dict								=	TFAutoModel.from_pretrained(SCREAMING_SNAKE_CASE_     )
        self.assertIsNotNone(SCREAMING_SNAKE_CASE_     )
        self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )



     @slow
     def   SCREAMING_SNAKE_CASE						(							self				:Optional[int]     ):





        '''simple docstring'''




        A_    : List[str]								=	'bert-base-cased'
        A_    : Dict								=	AutoConfig.from_pretrained(SCREAMING_SNAKE_CASE_     )
        self.assertIsNotNone(SCREAMING_SNAKE_CASE_     )
        self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )

        A_    : Any								=	TFAutoModelForPreTraining.from_pretrained(SCREAMING_SNAKE_CASE_     )
        self.assertIsNotNone(SCREAMING_SNAKE_CASE_     )
        self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )



     @slow
     def   SCREAMING_SNAKE_CASE						(							self				:Optional[Any]     ):





        '''simple docstring'''




        for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           A_    : int								=	AutoConfig.from_pretrained(SCREAMING_SNAKE_CASE_     )
           self.assertIsNotNone(SCREAMING_SNAKE_CASE_     )
           self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )

           A_    : Tuple								=	TFAutoModelForCausalLM.from_pretrained(SCREAMING_SNAKE_CASE_     )
           A_    : Tuple								=	TFAutoModelForCausalLM.from_pretrained(SCREAMING_SNAKE_CASE_	,					output_loading_info=SCREAMING_SNAKE_CASE_     )
           self.assertIsNotNone(SCREAMING_SNAKE_CASE_     )
           self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )



     @slow
     def   SCREAMING_SNAKE_CASE						(							self				:int     ):





        '''simple docstring'''




        for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           A_    : Optional[int]								=	AutoConfig.from_pretrained(SCREAMING_SNAKE_CASE_     )
           self.assertIsNotNone(SCREAMING_SNAKE_CASE_     )
           self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )

           A_    : Optional[Any]								=	TFAutoModelWithLMHead.from_pretrained(SCREAMING_SNAKE_CASE_     )
           self.assertIsNotNone(SCREAMING_SNAKE_CASE_     )
           self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )



     @slow
     def   SCREAMING_SNAKE_CASE						(							self				:int     ):





        '''simple docstring'''




        for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           A_    : Dict								=	AutoConfig.from_pretrained(SCREAMING_SNAKE_CASE_     )
           self.assertIsNotNone(SCREAMING_SNAKE_CASE_     )
           self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )

           A_    : str								=	TFAutoModelForMaskedLM.from_pretrained(SCREAMING_SNAKE_CASE_     )
           A_    : Union[str, Any]								=	TFAutoModelForMaskedLM.from_pretrained(SCREAMING_SNAKE_CASE_	,					output_loading_info=SCREAMING_SNAKE_CASE_     )
           self.assertIsNotNone(SCREAMING_SNAKE_CASE_     )
           self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )



     @slow
     def   SCREAMING_SNAKE_CASE						(							self				:List[str]     ):





        '''simple docstring'''




        for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
           A_    : Dict								=	AutoConfig.from_pretrained(SCREAMING_SNAKE_CASE_     )
           self.assertIsNotNone(SCREAMING_SNAKE_CASE_     )
           self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )

           A_    : Any								=	TFAutoModelForSeqaSeqLM.from_pretrained(SCREAMING_SNAKE_CASE_     )
           A_    : Any								=	TFAutoModelForSeqaSeqLM.from_pretrained(SCREAMING_SNAKE_CASE_	,					output_loading_info=SCREAMING_SNAKE_CASE_     )
           self.assertIsNotNone(SCREAMING_SNAKE_CASE_     )
           self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )



     @slow
     def   SCREAMING_SNAKE_CASE						(							self				:Union[str, Any]     ):





        '''simple docstring'''




        for model_name in ["bert-base-uncased"]:
           A_    : Optional[int]								=	AutoConfig.from_pretrained(SCREAMING_SNAKE_CASE_     )
           self.assertIsNotNone(SCREAMING_SNAKE_CASE_     )
           self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )

           A_    : int								=	TFAutoModelForSequenceClassification.from_pretrained(SCREAMING_SNAKE_CASE_     )
           self.assertIsNotNone(SCREAMING_SNAKE_CASE_     )
           self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )



     @slow
     def   SCREAMING_SNAKE_CASE						(							self				:Optional[int]     ):





        '''simple docstring'''




        for model_name in ["bert-base-uncased"]:
           A_    : Union[str, Any]								=	AutoConfig.from_pretrained(SCREAMING_SNAKE_CASE_     )
           self.assertIsNotNone(SCREAMING_SNAKE_CASE_     )
           self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )

           A_    : Optional[Any]								=	TFAutoModelForQuestionAnswering.from_pretrained(SCREAMING_SNAKE_CASE_     )
           self.assertIsNotNone(SCREAMING_SNAKE_CASE_     )
           self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )



     @slow
     @require_tensorflow_probability
     def   SCREAMING_SNAKE_CASE						(							self				:List[str]     ):





        '''simple docstring'''




        for model_name in TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
           A_    : Any								=	AutoConfig.from_pretrained(SCREAMING_SNAKE_CASE_     )
           self.assertIsNotNone(SCREAMING_SNAKE_CASE_     )
           self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )

           A_    : str								=	TFAutoModelForTableQuestionAnswering.from_pretrained(SCREAMING_SNAKE_CASE_     )
           A_    : List[str]								=	TFAutoModelForTableQuestionAnswering.from_pretrained(
               SCREAMING_SNAKE_CASE_	,					output_loading_info=SCREAMING_SNAKE_CASE_     )
           self.assertIsNotNone(SCREAMING_SNAKE_CASE_     )
           self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )



     def   SCREAMING_SNAKE_CASE						(							self				:Dict     ):





        '''simple docstring'''




        A_    : Any								=	TFAutoModelWithLMHead.from_pretrained(SCREAMING_SNAKE_CASE_     )
        self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )
        self.assertEqual(model.num_parameters()	,					14_410     )
        self.assertEqual(model.num_parameters(only_trainable=SCREAMING_SNAKE_CASE_     )	,					14_410     )



     def   SCREAMING_SNAKE_CASE						(							self				:Optional[Any]     ):





        '''simple docstring'''




        A_    : int								=	TFAutoModelWithLMHead.from_pretrained(SCREAMING_SNAKE_CASE_     )
        self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )
        self.assertEqual(model.num_parameters()	,					14_410     )
        self.assertEqual(model.num_parameters(only_trainable=SCREAMING_SNAKE_CASE_     )	,					14_410     )



     def   SCREAMING_SNAKE_CASE						(							self				:List[str]     ):





        '''simple docstring'''




        A_    : Tuple								=	TFAutoModel.from_pretrained("sgugger/funnel-random-tiny"     )
        self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )

        A_    : str								=	copy.deepcopy(model.config     )
        A_    : Union[str, Any]								=	['FunnelBaseModel']
        A_    : Union[str, Any]								=	TFAutoModel.from_config(SCREAMING_SNAKE_CASE_     )
        self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )

        with tempfile.TemporaryDirectory() as tmp_dir:
           model.save_pretrained(SCREAMING_SNAKE_CASE_     )
           A_    : List[Any]								=	TFAutoModel.from_pretrained(SCREAMING_SNAKE_CASE_     )
           self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )



     def   SCREAMING_SNAKE_CASE						(							self				:Any     ):





        '''simple docstring'''




        try:
           AutoConfig.register("new-model"	,					SCREAMING_SNAKE_CASE_     )

           A_    : Dict								=	[
               TFAutoModel,
               TFAutoModelForCausalLM,
               TFAutoModelForMaskedLM,
               TFAutoModelForPreTraining,
               TFAutoModelForQuestionAnswering,
               TFAutoModelForSequenceClassification,
               TFAutoModelForTokenClassification,
           ]

           for auto_class in auto_classes:
              with self.subTest(auto_class.__name__     ):
                 # Wrong config class will raise an error
                 with self.assertRaises(SCREAMING_SNAKE_CASE_     ):
                    auto_class.register(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )
                 auto_class.register(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )
                 # Trying to register something existing in the Transformers library will raise an error
                 with self.assertRaises(SCREAMING_SNAKE_CASE_     ):
                    auto_class.register(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )

                 # Now that the config is registered, it can be used as any other config with the auto-API
                 A_    : Optional[Any]								=	BertModelTester(self     ).get_config()
                 A_    : Union[str, Any]								=	NewModelConfig(**tiny_config.to_dict()     )
                 A_    : List[Any]								=	auto_class.from_config(SCREAMING_SNAKE_CASE_     )
                 self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )

                 with tempfile.TemporaryDirectory() as tmp_dir:
                    model.save_pretrained(SCREAMING_SNAKE_CASE_     )
                    A_    : Optional[Any]								=	auto_class.from_pretrained(SCREAMING_SNAKE_CASE_     )
                    self.assertIsInstance(SCREAMING_SNAKE_CASE_	,					SCREAMING_SNAKE_CASE_     )

        finally:
           if "new-model" in CONFIG_MAPPING._extra_content:
              del CONFIG_MAPPING._extra_content["new-model"]
           for mapping in (
               TF_MODEL_MAPPING,
               TF_MODEL_FOR_PRETRAINING_MAPPING,
               TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
               TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
               TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
               TF_MODEL_FOR_CAUSAL_LM_MAPPING,
               TF_MODEL_FOR_MASKED_LM_MAPPING,
           ):
              if NewModelConfig in mapping._extra_content:
                 del mapping._extra_content[NewModelConfig]



     def   SCREAMING_SNAKE_CASE						(							self				:str     ):





        '''simple docstring'''




        with self.assertRaisesRegex(
            SCREAMING_SNAKE_CASE_	,					"bert-base is not a local folder and is not a valid model identifier"     ):
           A_    : List[Any]								=	TFAutoModel.from_pretrained("bert-base"     )



     def   SCREAMING_SNAKE_CASE						(							self				:List[str]     ):





        '''simple docstring'''




        with self.assertRaisesRegex(
            SCREAMING_SNAKE_CASE_	,					R"aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"     ):
           A_    : List[Any]								=	TFAutoModel.from_pretrained(SCREAMING_SNAKE_CASE_	,					revision="aaaaaa"     )



     def   SCREAMING_SNAKE_CASE						(							self				:str     ):





        '''simple docstring'''




        with self.assertRaisesRegex(
            SCREAMING_SNAKE_CASE_	,					"hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin"	,					):
           A_    : int								=	TFAutoModel.from_pretrained("hf-internal-testing/config-no-model"     )



     def   SCREAMING_SNAKE_CASE						(							self				:Optional[int]     ):





        '''simple docstring'''




        with self.assertRaisesRegex(SCREAMING_SNAKE_CASE_	,					"Use `from_pt=True` to load this model"     ):
           A_    : Optional[int]								=	TFAutoModel.from_pretrained("hf-internal-testing/tiny-bert-pt-only"     )





     def   SCREAMING_SNAKE_CASE						(							self				:Optional[int]     ):





        '''simple docstring'''




        A_    : Union[str, Any]								=	TFAutoModel.from_pretrained("hf-internal-testing/tiny-random-bert"     )
        with RequestCounter() as counter:
           A_    : str								=	TFAutoModel.from_pretrained("hf-internal-testing/tiny-random-bert"     )
           self.assertEqual(counter.get_request_count	,					0     )
           self.assertEqual(counter.head_request_count	,					1     )
           self.assertEqual(counter.other_request_count	,					0     )

        # With a sharded checkpoint
        A_    : Tuple								=	TFAutoModel.from_pretrained("ArthurZ/tiny-random-bert-sharded"     )
        with RequestCounter() as counter:
           A_    : Optional[int]								=	TFAutoModel.from_pretrained("ArthurZ/tiny-random-bert-sharded"     )
           self.assertEqual(counter.get_request_count	,					0     )
           self.assertEqual(counter.head_request_count	,					1     )
           self.assertEqual(counter.other_request_count	,					0     )





import os
import sys
import warnings
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, List, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen
from ..table import array_cast
from ..utils.file_utils import is_local_path
from ..utils.py_utils import first_non_null_value, no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
    import PIL.Image

    from .features import FeatureType


__UpperCAmelCase          =       None
__UpperCAmelCase          =       '''<''' if sys.byteorder == '''little''' else '''>'''
# Origin: https://github.com/python-pillow/Pillow/blob/698951e19e19972aeed56df686868f1329981c12/src/PIL/Image.py#L3126 minus "|i1" which values are not preserved correctly when saving and loading an image
__UpperCAmelCase          =       [
    np.dtype('''|b1'''),
    np.dtype('''|u1'''),
    np.dtype('''<u2'''),
    np.dtype('''>u2'''),
    np.dtype('''<i2'''),
    np.dtype('''>i2'''),
    np.dtype('''<u4'''),
    np.dtype('''>u4'''),
    np.dtype('''<i4'''),
    np.dtype('''>i4'''),
    np.dtype('''<f4'''),
    np.dtype('''>f4'''),
    np.dtype('''<f8'''),
    np.dtype('''>f8'''),
]



@dataclass
class 				lowerCAmelCase_		:
       UpperCAmelCase__				:					bool        =  True
       UpperCAmelCase__				:					Optional[str]        =  None
       # Automatically constructed
       UpperCAmelCase__				:					ClassVar[str]        =  "PIL.Image.Image"
       UpperCAmelCase__				:					ClassVar[Any]        =  pa.struct({"bytes": pa.binary(), "path": pa.string()}							)
       UpperCAmelCase__				:					str        =  field(default="Image"       ,    init=a__       ,    repr=a__							)
       def __call__(			self       )     ->		Any:
          return self.pa_type
       def 					snake_case_       (			self,			SCREAMING_SNAKE_CASE_       )     ->		dict:
          if config.PIL_AVAILABLE:
             import PIL.Image
          else:
             raise ImportError('To support encoding images, please install \'Pillow\'.'       )

          if isinstance(SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_       ):
             UpperCamelCase    :  Optional[int]							=	np.array(SCREAMING_SNAKE_CASE_       )

          if isinstance(SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_       ):
             return {"path": value, "bytes": None}
          elif isinstance(SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_       ):
             return {"path": None, "bytes": value}
          elif isinstance(SCREAMING_SNAKE_CASE_,			np.ndarray       ):
             # convert the image array to PNG/TIFF bytes
             return encode_np_array(SCREAMING_SNAKE_CASE_       )
          elif isinstance(SCREAMING_SNAKE_CASE_,			PIL.Image.Image       ):
             # convert the PIL image to bytes (default format is PNG/TIFF)
             return encode_pil_image(SCREAMING_SNAKE_CASE_       )
          elif value.get('path'       ) is not None and os.path.isfile(value['path']       ):
             # we set "bytes": None to not duplicate the data if they're already available locally
             return {"bytes": None, "path": value.get('path'       )}
          elif value.get('bytes'       ) is not None or value.get('path'       ) is not None:
             # store the image bytes, and path is used to infer the image format using the file extension
             return {"bytes": value.get('bytes'       ), "path": value.get('path'       )}
          else:
             raise ValueError(
                 F"""An image sample should have one of 'path' or 'bytes' but they are missing or None in {value}."""       )
       def 					snake_case_       (			self,			SCREAMING_SNAKE_CASE_,			SCREAMING_SNAKE_CASE_=None       )     ->		"PIL.Image.Image":
          if not self.decode:
             raise RuntimeError('Decoding is disabled for this feature. Please use Image(decode=True) instead.'       )

          if config.PIL_AVAILABLE:
             import PIL.Image
          else:
             raise ImportError('To support decoding images, please install \'Pillow\'.'       )

          if token_per_repo_id is None:
             UpperCamelCase    :  Any							=	{}

          UpperCamelCase	,     UpperCamelCase    :  Union[str, Any]							=	value['path'], value['bytes']
          if bytes_ is None:
             if path is None:
                raise ValueError(F"""An image should have one of 'path' or 'bytes' but both are None in {value}."""       )
             else:
                if is_local_path(SCREAMING_SNAKE_CASE_       ):
                   UpperCamelCase    :  int							=	PIL.Image.open(SCREAMING_SNAKE_CASE_       )
                else:
                   UpperCamelCase    :  int							=	path.split('::'       )[-1]
                   try:
                      UpperCamelCase    :  Optional[Any]							=	string_to_dict(SCREAMING_SNAKE_CASE_,			config.HUB_DATASETS_URL       )['repo_id']
                      UpperCamelCase    :  str							=	token_per_repo_id.get(SCREAMING_SNAKE_CASE_       )
                   except ValueError:
                      UpperCamelCase    :  Tuple							=	None
                   with xopen(SCREAMING_SNAKE_CASE_,			'rb',			use_auth_token=SCREAMING_SNAKE_CASE_       ) as f:
                      UpperCamelCase    :  Optional[int]							=	BytesIO(f.read()       )
                   UpperCamelCase    :  int							=	PIL.Image.open(bytes_       )
          else:
             UpperCamelCase    :  Optional[int]							=	PIL.Image.open(BytesIO(bytes_       )       )
          image.load()  # to avoid "Too many open files" errors
          return image
       def 					snake_case_       (			self       )     ->		Union["FeatureType", Dict[str, "FeatureType"]]:
          from .features import Value

          return (
              self
              if self.decode
              else {
                  "bytes": Value('binary'       ),
                  "path": Value('string'       ),
              }
          )
       def 					snake_case_       (			self,			SCREAMING_SNAKE_CASE_       )     ->		pa.StructArray:
          if pa.types.is_string(storage.type       ):
             UpperCamelCase    :  List[str]							=	pa.array([None] * len(SCREAMING_SNAKE_CASE_       ),			type=pa.binary()       )
             UpperCamelCase    :  Dict							=	pa.StructArray.from_arrays([bytes_array, storage],			['bytes', 'path'],			mask=storage.is_null()       )
          elif pa.types.is_binary(storage.type       ):
             UpperCamelCase    :  Optional[int]							=	pa.array([None] * len(SCREAMING_SNAKE_CASE_       ),			type=pa.string()       )
             UpperCamelCase    :  Union[str, Any]							=	pa.StructArray.from_arrays([storage, path_array],			['bytes', 'path'],			mask=storage.is_null()       )
          elif pa.types.is_struct(storage.type       ):
             if storage.type.get_field_index('bytes'       ) >= 0:
                UpperCamelCase    :  List[str]							=	storage.field('bytes'       )
             else:
                UpperCamelCase    :  List[str]							=	pa.array([None] * len(SCREAMING_SNAKE_CASE_       ),			type=pa.binary()       )
             if storage.type.get_field_index('path'       ) >= 0:
                UpperCamelCase    :  List[str]							=	storage.field('path'       )
             else:
                UpperCamelCase    :  Optional[Any]							=	pa.array([None] * len(SCREAMING_SNAKE_CASE_       ),			type=pa.string()       )
             UpperCamelCase    :  Optional[int]							=	pa.StructArray.from_arrays([bytes_array, path_array],			['bytes', 'path'],			mask=storage.is_null()       )
          elif pa.types.is_list(storage.type       ):
             UpperCamelCase    :  Optional[Any]							=	pa.array(
                 [encode_np_array(np.array(SCREAMING_SNAKE_CASE_       )       )['bytes'] if arr is not None else None for arr in storage.to_pylist()],			type=pa.binary(),			)
             UpperCamelCase    :  List[str]							=	pa.array([None] * len(SCREAMING_SNAKE_CASE_       ),			type=pa.string()       )
             UpperCamelCase    :  int							=	pa.StructArray.from_arrays(
                 [bytes_array, path_array],			['bytes', 'path'],			mask=bytes_array.is_null()       )
          return array_cast(SCREAMING_SNAKE_CASE_,			self.pa_type       )






       def 					snake_case_       (			self,			SCREAMING_SNAKE_CASE_       )     ->		pa.StructArray:

          @no_op_if_value_is_null
          def path_to_bytes(SCREAMING_SNAKE_CASE_       ):
             with xopen(SCREAMING_SNAKE_CASE_,			'rb'       ) as f:
                UpperCamelCase    :  Optional[int]							=	f.read()
             return bytes_

          UpperCamelCase    :  Union[str, Any]							=	pa.array(
              [
                  (path_to_bytes(x['path']       ) if x['bytes'] is None else x['bytes']) if x is not None else None
                  for x in storage.to_pylist()
              ],			type=pa.binary(),			)
          UpperCamelCase    :  Any							=	pa.array(
              [os.path.basename(SCREAMING_SNAKE_CASE_       ) if path is not None else None for path in storage.field('path'       ).to_pylist()],			type=pa.string(),			)
          UpperCamelCase    :  int							=	pa.StructArray.from_arrays([bytes_array, path_array],			['bytes', 'path'],			mask=bytes_array.is_null()       )
          return array_cast(SCREAMING_SNAKE_CASE_,			self.pa_type       )






def 		UpperCamelCase	(   )	->     List[str]:
   if config.PIL_AVAILABLE:
      import PIL.Image
   else:
      raise ImportError('To support encoding images, please install \'Pillow\'.'	)

   global _IMAGE_COMPRESSION_FORMATS
   if _IMAGE_COMPRESSION_FORMATS is None:
      PIL.Image.init()
      UpperCamelCase    :  Dict							=	list(set(PIL.Image.OPEN.keys()	) & set(PIL.Image.SAVE.keys()	)	)
   return _IMAGE_COMPRESSION_FORMATS
def 		UpperCamelCase	(   snake_case__       :    "PIL.Image.Image"	)	->     bytes:
   UpperCamelCase    :  Any							=	BytesIO()
   if image.format in list_image_compression_formats():
      UpperCamelCase    :  Tuple							=	image.format
   else:
      UpperCamelCase    :  List[str]							=	'PNG' if image.mode in ['1', 'L', 'LA', 'RGB', 'RGBA'] else 'TIFF'
   image.save(snake_case__       ,     format=snake_case__	)
   return buffer.getvalue()
def 		UpperCamelCase	(   snake_case__       :    "PIL.Image.Image"	)	->     dict:
   if hasattr(snake_case__       ,     'filename'	) and image.filename != "":
      return {"path": image.filename, "bytes": None}
   else:
      return {"path": None, "bytes": image_to_bytes(snake_case__	)}
def 		UpperCamelCase	(   snake_case__       :    np.ndarray	)	->     dict:
   if config.PIL_AVAILABLE:
      import PIL.Image
   else:
      raise ImportError('To support encoding images, please install \'Pillow\'.'	)

   UpperCamelCase    :  Union[str, Any]							=	array.dtype
   UpperCamelCase    :  List[Any]							=	dtype.byteorder if dtype.byteorder != '=' else _NATIVE_BYTEORDER
   UpperCamelCase    :  Optional[Any]							=	dtype.kind
   UpperCamelCase    :  Any							=	dtype.itemsize

   UpperCamelCase    :  int							=	None

   # Multi-channel array case (only np.dtype("|u1") is allowed)
   if array.shape[2:]:
      UpperCamelCase    :  Optional[Any]							=	np.dtype('|u1'	)
      if dtype_kind not in ["u", "i"]:
         raise TypeError(
             F"""Unsupported array dtype {dtype} for image encoding. Only {dest_dtype} is supported for multi-channel arrays."""	)
      if dtype is not dest_dtype:
         warnings.warn(F"""Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'"""	)
    # Exact match
   elif dtype in _VALID_IMAGE_ARRAY_DTPYES:
      UpperCamelCase    :  List[Any]							=	dtype
   else:  # Downcast the type within the kind (np.can_cast(from_type, to_type, casting="same_kind") doesn't behave as expected, so do it manually)
      while dtype_itemsize >= 1:
         UpperCamelCase    :  Dict							=	dtype_byteorder + dtype_kind + str(snake_case__	)
         UpperCamelCase    :  str							=	np.dtype(snake_case__	)
         if dest_dtype in _VALID_IMAGE_ARRAY_DTPYES:
            warnings.warn(F"""Downcasting array dtype {dtype} to {dest_dtype} to be compatible with 'Pillow'"""	)
            break
         else:
            dtype_itemsize //= 2

   if dest_dtype is None:
      raise TypeError(
          F"""Cannot convert dtype {dtype} to a valid image dtype. Valid image dtypes: {_VALID_IMAGE_ARRAY_DTPYES}"""	)
   UpperCamelCase    :  Union[str, Any]							=	PIL.Image.fromarray(array.astype(snake_case__	)	)
   return {"path": None, "bytes": image_to_bytes(snake_case__	)}


def 		UpperCamelCase	(   snake_case__       :    Union[List[str], List[dict], List[np.ndarray], List["PIL.Image.Image"]]	)	->     List[dict]:
   if config.PIL_AVAILABLE:
      import PIL.Image
   else:
      raise ImportError('To support encoding images, please install \'Pillow\'.'	)

   if objs:
      UpperCamelCase	,     UpperCamelCase    :  Union[str, Any]							=	first_non_null_value(snake_case__	)
      if isinstance(snake_case__       ,     snake_case__	):
         return [{"path": obj, "bytes": None} if obj is not None else None for obj in objs]
      if isinstance(snake_case__       ,     np.ndarray	):
         UpperCamelCase    :  List[Any]							=	no_op_if_value_is_null(snake_case__	)
         return [obj_to_image_dict_func(snake_case__	) for obj in objs]
      elif isinstance(snake_case__       ,     PIL.Image.Image	):
         UpperCamelCase    :  Optional[int]							=	no_op_if_value_is_null(snake_case__	)
         return [obj_to_image_dict_func(snake_case__	) for obj in objs]
      else:
         return objs
   else:
      return objs



