
'''simple docstring'''

import itertools
import random
import unittest

import numpy as np

from transformers import WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST, WavaVecaConfig, WavaVecaFeatureExtractor
from transformers.testing_utils import require_torch, slow

from ...test_sequence_feature_extraction_common import SequenceFeatureExtractionTestMixin


UpperCamelCase_        =							random.Random()


def 						_UpperCAmelCase					(	_lowerCamelCase    :							Any	,   _lowerCamelCase    :							Union[str, Any]=1.0	,   _lowerCamelCase    :							int=None	,   _lowerCamelCase    :							Tuple=None		)     ->    Tuple:
  if rng is None:
    _lowerCAmelCase							:	str 					=       global_rng

  _lowerCAmelCase							:	int 					=       []
  for batch_idx in range(shape[0]		):
    values.append([]		)
    for _ in range(shape[1]		):
      values[-1].append(rng.random() * scale		)

  return values


class 	a_	(unittest.TestCase     ):




       def __init__(  self , snake_case_ , snake_case_=7 , snake_case_=4_0_0 , snake_case_=2_0_0_0 , snake_case_=1 , snake_case_=0.0 , snake_case_=1_6_0_0_0 , snake_case_=True , snake_case_=True , ):
         _lowerCAmelCase							:	Dict 					=       parent
         _lowerCAmelCase							:	str 					=       batch_size
         _lowerCAmelCase							:	Any 					=       min_seq_length
         _lowerCAmelCase							:	Optional[Any] 					=       max_seq_length
         _lowerCAmelCase							:	Optional[Any] 					=       (self.max_seq_length - self.min_seq_length) // (self.batch_size - 1)
         _lowerCAmelCase							:	int 					=       feature_size
         _lowerCAmelCase							:	Union[str, Any] 					=       padding_value
         _lowerCAmelCase							:	Optional[int] 					=       sampling_rate
         _lowerCAmelCase							:	Tuple 					=       return_attention_mask
         _lowerCAmelCase							:	str 					=       do_normalize




       def 				__UpperCamelCase							(  self	):
         return {
             "feature_size": self.feature_size,
             "padding_value": self.padding_value,
             "sampling_rate": self.sampling_rate,
             "return_attention_mask": self.return_attention_mask,
             "do_normalize": self.do_normalize,
         }







       def 				__UpperCamelCase							(  self , snake_case_=False , snake_case_=False	):







         def _flatten(snake_case_	):
           return list(itertools.chain(*SCREAMING_SNAKE_CASE_	)	)

         if equal_length:
           _lowerCAmelCase							:	List[Any] 					=       floats_list((self.batch_size, self.max_seq_length)	)
         else:
           # make sure that inputs increase in size
           _lowerCAmelCase							:	Optional[int] 					=       [
               _flatten(floats_list((x, self.feature_size)	)	)
               for x in range(self.min_seq_length , self.max_seq_length , self.seq_length_diff	)
           ]

         if numpify:
           _lowerCAmelCase							:	int 					=       [np.asarray(SCREAMING_SNAKE_CASE_	) for x in speech_inputs]

         return speech_inputs



class 	a_	(__lowercase				,      unittest.TestCase     ):
       __lowerCAmelCase     :					Optional[Any]     =   WavaVecaFeatureExtractor




       def 				__UpperCamelCase							(  self	):
         _lowerCAmelCase							:	int 					=       WavaVecaFeatureExtractionTester(self	)




       def 				__UpperCamelCase							(  self , snake_case_	):
         self.assertTrue(np.all(np.mean(SCREAMING_SNAKE_CASE_ , axis=0	) < 1E-3	)	)
         self.assertTrue(np.all(np.abs(np.var(SCREAMING_SNAKE_CASE_ , axis=0	) - 1	) < 1E-3	)	)




       def 				__UpperCamelCase							(  self	):
         _lowerCAmelCase							:	Dict 					=       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()	)
         # create three inputs of length 800, 1000, and 1200
         _lowerCAmelCase							:	Any 					=       [floats_list((1, x)	)[0] for x in range(8_0_0 , 1_4_0_0 , 2_0_0	)]
         _lowerCAmelCase							:	Optional[int] 					=       [np.asarray(SCREAMING_SNAKE_CASE_	) for speech_input in speech_inputs]

         # Test not batched input
         _lowerCAmelCase							:	Any 					=       feat_extract(speech_inputs[0] , return_tensors="""np"""	).input_values
         _lowerCAmelCase							:	Any 					=       feat_extract(np_speech_inputs[0] , return_tensors="""np"""	).input_values
         self.assertTrue(np.allclose(SCREAMING_SNAKE_CASE_ , SCREAMING_SNAKE_CASE_ , atol=1E-3	)	)

         # Test batched
         _lowerCAmelCase							:	Optional[Any] 					=       feat_extract(SCREAMING_SNAKE_CASE_ , return_tensors="""np"""	).input_values
         _lowerCAmelCase							:	Tuple 					=       feat_extract(SCREAMING_SNAKE_CASE_ , return_tensors="""np"""	).input_values
         for enc_seq_a, enc_seq_a in zip(SCREAMING_SNAKE_CASE_ , SCREAMING_SNAKE_CASE_	):
           self.assertTrue(np.allclose(SCREAMING_SNAKE_CASE_ , SCREAMING_SNAKE_CASE_ , atol=1E-3	)	)

         # Test 2-D numpy arrays are batched.
         _lowerCAmelCase							:	Optional[int] 					=       [floats_list((1, x)	)[0] for x in (8_0_0, 8_0_0, 8_0_0)]
         _lowerCAmelCase							:	List[Any] 					=       np.asarray(SCREAMING_SNAKE_CASE_	)
         _lowerCAmelCase							:	List[str] 					=       feat_extract(SCREAMING_SNAKE_CASE_ , return_tensors="""np"""	).input_values
         _lowerCAmelCase							:	List[str] 					=       feat_extract(SCREAMING_SNAKE_CASE_ , return_tensors="""np"""	).input_values
         for enc_seq_a, enc_seq_a in zip(SCREAMING_SNAKE_CASE_ , SCREAMING_SNAKE_CASE_	):
           self.assertTrue(np.allclose(SCREAMING_SNAKE_CASE_ , SCREAMING_SNAKE_CASE_ , atol=1E-3	)	)




       def 				__UpperCamelCase							(  self	):
         _lowerCAmelCase							:	Optional[int] 					=       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()	)
         _lowerCAmelCase							:	Optional[Any] 					=       [floats_list((1, x)	)[0] for x in range(8_0_0 , 1_4_0_0 , 2_0_0	)]

         _lowerCAmelCase							:	Any 					=       ["""longest""", """max_length""", """do_not_pad"""]
         _lowerCAmelCase							:	List[Any] 					=       [None, 1_6_0_0, None]
         for max_length, padding in zip(SCREAMING_SNAKE_CASE_ , SCREAMING_SNAKE_CASE_	):
           _lowerCAmelCase							:	Optional[Any] 					=       feat_extract(SCREAMING_SNAKE_CASE_ , padding=SCREAMING_SNAKE_CASE_ , max_length=SCREAMING_SNAKE_CASE_ , return_tensors="""np"""	)
           _lowerCAmelCase							:	int 					=       processed.input_values

           self._check_zero_mean_unit_variance(input_values[0][:8_0_0]	)
           self.assertTrue(input_values[0][8_0_0:].sum() < 1E-6	)
           self._check_zero_mean_unit_variance(input_values[1][:1_0_0_0]	)
           self.assertTrue(input_values[0][1_0_0_0:].sum() < 1E-6	)
           self._check_zero_mean_unit_variance(input_values[2][:1_2_0_0]	)




       def 				__UpperCamelCase							(  self	):
         _lowerCAmelCase							:	List[str] 					=       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()	)
         _lowerCAmelCase							:	List[Any] 					=       range(8_0_0 , 1_4_0_0 , 2_0_0	)
         _lowerCAmelCase							:	Tuple 					=       [floats_list((1, x)	)[0] for x in lengths]

         _lowerCAmelCase							:	List[str] 					=       ["""longest""", """max_length""", """do_not_pad"""]
         _lowerCAmelCase							:	int 					=       [None, 1_6_0_0, None]

         for max_length, padding in zip(SCREAMING_SNAKE_CASE_ , SCREAMING_SNAKE_CASE_	):
           _lowerCAmelCase							:	str 					=       feat_extract(SCREAMING_SNAKE_CASE_ , max_length=SCREAMING_SNAKE_CASE_ , padding=SCREAMING_SNAKE_CASE_	)
           _lowerCAmelCase							:	List[str] 					=       processed.input_values

           self._check_zero_mean_unit_variance(input_values[0][:8_0_0]	)
           self._check_zero_mean_unit_variance(input_values[1][:1_0_0_0]	)
           self._check_zero_mean_unit_variance(input_values[2][:1_2_0_0]	)




       def 				__UpperCamelCase							(  self	):
         _lowerCAmelCase							:	Optional[Any] 					=       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()	)
         _lowerCAmelCase							:	str 					=       [floats_list((1, x)	)[0] for x in range(8_0_0 , 1_4_0_0 , 2_0_0	)]
         _lowerCAmelCase							:	str 					=       feat_extract(
             SCREAMING_SNAKE_CASE_ , truncation=SCREAMING_SNAKE_CASE_ , max_length=1_0_0_0 , padding="""max_length""" , return_tensors="""np"""	)
         _lowerCAmelCase							:	str 					=       processed.input_values

         self._check_zero_mean_unit_variance(input_values[0, :8_0_0]	)
         self._check_zero_mean_unit_variance(input_values[1]	)
         self._check_zero_mean_unit_variance(input_values[2]	)




       def 				__UpperCamelCase							(  self	):
         _lowerCAmelCase							:	Optional[int] 					=       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()	)
         _lowerCAmelCase							:	Dict 					=       [floats_list((1, x)	)[0] for x in range(8_0_0 , 1_4_0_0 , 2_0_0	)]
         _lowerCAmelCase							:	Dict 					=       feat_extract(
             SCREAMING_SNAKE_CASE_ , truncation=SCREAMING_SNAKE_CASE_ , max_length=1_0_0_0 , padding="""longest""" , return_tensors="""np"""	)
         _lowerCAmelCase							:	str 					=       processed.input_values

         self._check_zero_mean_unit_variance(input_values[0, :8_0_0]	)
         self._check_zero_mean_unit_variance(input_values[1, :1_0_0_0]	)
         self._check_zero_mean_unit_variance(input_values[2]	)

         # make sure that if max_length < longest -> then pad to max_length
         self.assertTrue(input_values.shape == (3, 1_0_0_0)	)

         _lowerCAmelCase							:	int 					=       [floats_list((1, x)	)[0] for x in range(8_0_0 , 1_4_0_0 , 2_0_0	)]
         _lowerCAmelCase							:	List[str] 					=       feat_extract(
             SCREAMING_SNAKE_CASE_ , truncation=SCREAMING_SNAKE_CASE_ , max_length=2_0_0_0 , padding="""longest""" , return_tensors="""np"""	)
         _lowerCAmelCase							:	Tuple 					=       processed.input_values

         self._check_zero_mean_unit_variance(input_values[0, :8_0_0]	)
         self._check_zero_mean_unit_variance(input_values[1, :1_0_0_0]	)
         self._check_zero_mean_unit_variance(input_values[2]	)

         # make sure that if max_length > longest -> then pad to longest
         self.assertTrue(input_values.shape == (3, 1_2_0_0)	)




       @require_torch
       def 				__UpperCamelCase							(  self	):
         import torch

         _lowerCAmelCase							:	List[str] 					=       self.feature_extraction_class(**self.feat_extract_tester.prepare_feat_extract_dict()	)
         _lowerCAmelCase							:	Any 					=       np.random.rand(1_0_0	).astype(np.floataa	)
         _lowerCAmelCase							:	Tuple 					=       np_speech_inputs.tolist()

         for inputs in [py_speech_inputs, np_speech_inputs]:
           _lowerCAmelCase							:	Any 					=       feature_extractor.pad([{"""input_values""": inputs}] , return_tensors="""np"""	)
           self.assertTrue(np_processed.input_values.dtype == np.floataa	)
           _lowerCAmelCase							:	Optional[Any] 					=       feature_extractor.pad([{"""input_values""": inputs}] , return_tensors="""pt"""	)
           self.assertTrue(pt_processed.input_values.dtype == torch.floataa	)







       @slow
       @require_torch
       def 				__UpperCamelCase							(  self	):







         for model_id in WAV_2_VEC_2_PRETRAINED_MODEL_ARCHIVE_LIST:
           _lowerCAmelCase							:	Dict 					=       WavaVecaConfig.from_pretrained(SCREAMING_SNAKE_CASE_	)
           _lowerCAmelCase							:	Dict 					=       WavaVecaFeatureExtractor.from_pretrained(SCREAMING_SNAKE_CASE_	)

           # only "layer" feature extraction norm should make use of
           # attention_mask
           self.assertEqual(feat_extract.return_attention_mask , config.feat_extract_norm == """layer"""	)




"""simple docstring"""




import collections.abc
from typing import Optional, Tuple, Union

import torch
import torch.utils.checkpoint
from torch import nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...modeling_outputs import BaseModelOutputWithNoAttention, ImageClassifierOutputWithNoAttention
from ...modeling_utils import PreTrainedModel
from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
from .configuration_poolformer import PoolFormerConfig


lowerCAmelCase__				=					logging.get_logger(__name__)

# General docstring
lowerCAmelCase__				=					'PoolFormerConfig'

# Base docstring
lowerCAmelCase__				=					'sail/poolformer_s12'
lowerCAmelCase__				=					[1, 512, 7, 7]

# Image classification docstring
lowerCAmelCase__				=					'sail/poolformer_s12'
lowerCAmelCase__				=					'tabby, tabby cat'

lowerCAmelCase__				=					[
    'sail/poolformer_s12',
    # See all PoolFormer models at https://huggingface.co/models?filter=poolformer
]



def 							_lowerCamelCase			(			__a,				__a = 0.0,				__a = False     ):
		if drop_prob == 0.0 or not training:
				return input
		SCREAMING_SNAKE_CASE_    			=			1 - drop_prob
		SCREAMING_SNAKE_CASE_    			=			(input.shape[0],) + (1,) * (input.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets
		SCREAMING_SNAKE_CASE_    			=			keep_prob + torch.rand(__a,				dtype=input.dtype,				device=input.device     )
		random_tensor.floor_()  # binarize
		SCREAMING_SNAKE_CASE_    			=			input.div(__a     ) * random_tensor
		return output





class 			snake_case  (				nn.Module ):

							def __init__(self							,							SCREAMING_SNAKE_CASE_ = None     ):

									"""simple docstring"""







									super().__init__()
									SCREAMING_SNAKE_CASE_    			=			drop_prob

							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									return drop_path(SCREAMING_SNAKE_CASE_							,							self.drop_prob							,							self.training     )







							def _lowercase       (self     ):

									"""simple docstring"""







									return "p={}".format(self.drop_prob     )







class 			snake_case  (				nn.Module ):

							def __init__(self							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_=None     ):

									"""simple docstring"""







									super().__init__()
									SCREAMING_SNAKE_CASE_    			=			patch_size if isinstance(SCREAMING_SNAKE_CASE_							,							collections.abc.Iterable     ) else (patch_size, patch_size)
									SCREAMING_SNAKE_CASE_    			=			stride if isinstance(SCREAMING_SNAKE_CASE_							,							collections.abc.Iterable     ) else (stride, stride)
									SCREAMING_SNAKE_CASE_    			=			padding if isinstance(SCREAMING_SNAKE_CASE_							,							collections.abc.Iterable     ) else (padding, padding)

									SCREAMING_SNAKE_CASE_    			=			nn.Convad(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							kernel_size=SCREAMING_SNAKE_CASE_							,							stride=SCREAMING_SNAKE_CASE_							,							padding=SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			norm_layer(SCREAMING_SNAKE_CASE_     ) if norm_layer else nn.Identity()







							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			self.projection(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			self.norm(SCREAMING_SNAKE_CASE_     )
									return embeddings







class 			snake_case  (				nn.GroupNorm ):

							def __init__(self							,							SCREAMING_SNAKE_CASE_							,							**SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									super().__init__(1							,							SCREAMING_SNAKE_CASE_							,							**SCREAMING_SNAKE_CASE_     )







class 			snake_case  (				nn.Module ):

							def __init__(self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									super().__init__()
									SCREAMING_SNAKE_CASE_    			=			nn.AvgPoolad(SCREAMING_SNAKE_CASE_							,							stride=1							,							padding=pool_size // 2							,							count_include_pad=SCREAMING_SNAKE_CASE_     )







							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									return self.pool(SCREAMING_SNAKE_CASE_     ) - hidden_states







class 			snake_case  (				nn.Module ):

							def __init__(self							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									super().__init__()
									SCREAMING_SNAKE_CASE_    			=			nn.Convad(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							1     )
									SCREAMING_SNAKE_CASE_    			=			nn.Convad(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							1     )
									SCREAMING_SNAKE_CASE_    			=			PoolFormerDropPath(SCREAMING_SNAKE_CASE_     )
									if isinstance(config.hidden_act							,							SCREAMING_SNAKE_CASE_     ):
											SCREAMING_SNAKE_CASE_    			=			ACTaFN[config.hidden_act]
									else:
											SCREAMING_SNAKE_CASE_    			=			config.hidden_act







							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			self.conva(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			self.act_fn(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			self.drop(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			self.conva(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			self.drop(SCREAMING_SNAKE_CASE_     )

									return hidden_states







class 			snake_case  (				nn.Module ):

							def __init__(self							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									super().__init__()
									SCREAMING_SNAKE_CASE_    			=			PoolFormerPooling(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			PoolFormerOutput(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			PoolFormerGroupNorm(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			PoolFormerGroupNorm(SCREAMING_SNAKE_CASE_     )

									# Useful for training neural nets
									SCREAMING_SNAKE_CASE_    			=			PoolFormerDropPath(SCREAMING_SNAKE_CASE_     ) if drop_path > 0.0 else nn.Identity()
									SCREAMING_SNAKE_CASE_    			=			config.use_layer_scale
									if config.use_layer_scale:
											SCREAMING_SNAKE_CASE_    			=			nn.Parameter(
											    config.layer_scale_init_value * torch.ones((SCREAMING_SNAKE_CASE_)     )							,							requires_grad=SCREAMING_SNAKE_CASE_     )
											SCREAMING_SNAKE_CASE_    			=			nn.Parameter(
											    config.layer_scale_init_value * torch.ones((SCREAMING_SNAKE_CASE_)     )							,							requires_grad=SCREAMING_SNAKE_CASE_     )







							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									if self.use_layer_scale:
											SCREAMING_SNAKE_CASE_    			=			self.pooling(self.before_norm(SCREAMING_SNAKE_CASE_     )     )
											SCREAMING_SNAKE_CASE_    			=			self.layer_scale_a.unsqueeze(-1     ).unsqueeze(-1     ) * pooling_output
											# First residual connection
											SCREAMING_SNAKE_CASE_    			=			hidden_states + self.drop_path(SCREAMING_SNAKE_CASE_     )
											SCREAMING_SNAKE_CASE_    			=			()

											SCREAMING_SNAKE_CASE_    			=			self.output(self.after_norm(SCREAMING_SNAKE_CASE_     )     )
											SCREAMING_SNAKE_CASE_    			=			self.layer_scale_a.unsqueeze(-1     ).unsqueeze(-1     ) * layer_output
											# Second residual connection
											SCREAMING_SNAKE_CASE_    			=			hidden_states + self.drop_path(SCREAMING_SNAKE_CASE_     )

											SCREAMING_SNAKE_CASE_    			=			(output,) + outputs
											return outputs

									else:
											SCREAMING_SNAKE_CASE_    			=			self.drop_path(self.pooling(self.before_norm(SCREAMING_SNAKE_CASE_     )     )     )
											# First residual connection
											SCREAMING_SNAKE_CASE_    			=			pooling_output + hidden_states
											SCREAMING_SNAKE_CASE_    			=			()

											# Second residual connection inside the PoolFormerOutput block
											SCREAMING_SNAKE_CASE_    			=			self.drop_path(self.output(self.after_norm(SCREAMING_SNAKE_CASE_     )     )     )
											SCREAMING_SNAKE_CASE_    			=			hidden_states + layer_output

											SCREAMING_SNAKE_CASE_    			=			(output,) + outputs
											return outputs







class 			snake_case  (				nn.Module ):

							def __init__(self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									super().__init__()
									SCREAMING_SNAKE_CASE_    			=			config
									# stochastic depth decay rule
									SCREAMING_SNAKE_CASE_    			=			[x.item() for x in torch.linspace(0							,							config.drop_path_rate							,							sum(config.depths     )     )]

									# patch embeddings
									SCREAMING_SNAKE_CASE_    			=			[]
									for i in range(config.num_encoder_blocks     ):
											embeddings.append(
											    PoolFormerEmbeddings(
											        patch_size=config.patch_sizes[i]							,							stride=config.strides[i]							,							padding=config.padding[i]							,							num_channels=config.num_channels if i == 0 else config.hidden_sizes[i - 1]							,							hidden_size=config.hidden_sizes[i]							,							)     )
									SCREAMING_SNAKE_CASE_    			=			nn.ModuleList(SCREAMING_SNAKE_CASE_     )

									# Transformer blocks
									SCREAMING_SNAKE_CASE_    			=			[]
									SCREAMING_SNAKE_CASE_    			=			0
									for i in range(config.num_encoder_blocks     ):
											# each block consists of layers
											SCREAMING_SNAKE_CASE_    			=			[]
											if i != 0:
													cur += config.depths[i - 1]
											for j in range(config.depths[i]     ):
													layers.append(
													    PoolFormerLayer(
													        SCREAMING_SNAKE_CASE_							,							num_channels=config.hidden_sizes[i]							,							pool_size=config.pool_size							,							hidden_size=config.hidden_sizes[i]							,							intermediate_size=int(config.hidden_sizes[i] * config.mlp_ratio     )							,							drop_path=dpr[cur + j]							,							)     )
											blocks.append(nn.ModuleList(SCREAMING_SNAKE_CASE_     )     )

									SCREAMING_SNAKE_CASE_    			=			nn.ModuleList(SCREAMING_SNAKE_CASE_     )







							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_=False							,							SCREAMING_SNAKE_CASE_=True     ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			() if output_hidden_states else None

									SCREAMING_SNAKE_CASE_    			=			pixel_values
									for idx, layers in enumerate(zip(self.patch_embeddings							,							self.block     )     ):
											SCREAMING_SNAKE_CASE_			,SCREAMING_SNAKE_CASE_    			=			layers
											# Get patch embeddings from hidden_states
											SCREAMING_SNAKE_CASE_    			=			embedding_layer(SCREAMING_SNAKE_CASE_     )
											# Send the embeddings through the blocks
											for _, blk in enumerate(SCREAMING_SNAKE_CASE_     ):
													SCREAMING_SNAKE_CASE_    			=			blk(SCREAMING_SNAKE_CASE_     )
													SCREAMING_SNAKE_CASE_    			=			layer_outputs[0]

											if output_hidden_states:
													SCREAMING_SNAKE_CASE_    			=			all_hidden_states + (hidden_states,)

									if not return_dict:
											return tuple(v for v in [hidden_states, all_hidden_states] if v is not None     )

									return BaseModelOutputWithNoAttention(last_hidden_state=SCREAMING_SNAKE_CASE_							,							hidden_states=SCREAMING_SNAKE_CASE_     )







class 			snake_case  (				__lowercase ):
							UpperCAmelCase__      =   PoolFormerConfig
							UpperCAmelCase__      =   '''poolformer'''
							UpperCAmelCase__      =   '''pixel_values'''
							UpperCAmelCase__      =   True

							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									if isinstance(SCREAMING_SNAKE_CASE_							,							(nn.Linear, nn.Convad)     ):
											module.weight.data.normal_(mean=0.0							,							std=self.config.initializer_range     )
											if module.bias is not None:
													module.bias.data.zero_()
									elif isinstance(SCREAMING_SNAKE_CASE_							,							nn.LayerNorm     ):
											module.bias.data.zero_()
											module.weight.data.fill_(1.0     )







							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_=False     ):

									"""simple docstring"""







									if isinstance(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     ):
											SCREAMING_SNAKE_CASE_    			=			value







lowerCAmelCase__				=					r'\n    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use\n    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and\n    behavior.\n\n    Parameters:\n        config ([`PoolFormerConfig`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.\n'

lowerCAmelCase__				=					r'\n    Args:\n        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):\n            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See\n            [`PoolFormerImageProcessor.__call__`] for details.\n'





@add_start_docstrings(
    '''The bare PoolFormer Model transformer outputting raw hidden-states without any specific head on top.'''							,      __lowercase							,      )
class 			snake_case  (				__lowercase ):

							def __init__(self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									super().__init__(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			config

									SCREAMING_SNAKE_CASE_    			=			PoolFormerEncoder(SCREAMING_SNAKE_CASE_     )

									# Initialize weights and apply final processing
									self.post_init()

							def _lowercase       (self     ):

									"""simple docstring"""







									return self.embeddings.patch_embeddings







							@add_start_docstrings_to_model_forward(SCREAMING_SNAKE_CASE_     )
							@add_code_sample_docstrings(
							    checkpoint=_CHECKPOINT_FOR_DOC							,							output_type=SCREAMING_SNAKE_CASE_							,							config_class=_CONFIG_FOR_DOC							,							modality='''vision'''							,							expected_output=_EXPECTED_OUTPUT_SHAPE							,							)
							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_ = None							,							SCREAMING_SNAKE_CASE_ = None							,							SCREAMING_SNAKE_CASE_ = None							,							):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			(
									    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
									)
									SCREAMING_SNAKE_CASE_    			=			return_dict if return_dict is not None else self.config.use_return_dict

									if pixel_values is None:
											raise ValueError('''You have to specify pixel_values'''     )

									SCREAMING_SNAKE_CASE_    			=			self.encoder(
									    SCREAMING_SNAKE_CASE_							,							output_hidden_states=SCREAMING_SNAKE_CASE_							,							return_dict=SCREAMING_SNAKE_CASE_							,							)
									SCREAMING_SNAKE_CASE_    			=			encoder_outputs[0]

									if not return_dict:
											return (sequence_output, None) + encoder_outputs[1:]

									return BaseModelOutputWithNoAttention(
									    last_hidden_state=SCREAMING_SNAKE_CASE_							,							hidden_states=encoder_outputs.hidden_states							,							)







class 			snake_case  (				nn.Module ):

							def __init__(self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									super().__init__()
									SCREAMING_SNAKE_CASE_    			=			nn.Linear(config.hidden_size							,							config.hidden_size     )







							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			self.dense(SCREAMING_SNAKE_CASE_     )
									return output







@add_start_docstrings(
    '''
    PoolFormer Model transformer with an image classification head on top
    '''							,      __lowercase							,      )
class 			snake_case  (				__lowercase ):

							def __init__(self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									super().__init__(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			config.num_labels
									SCREAMING_SNAKE_CASE_    			=			PoolFormerModel(SCREAMING_SNAKE_CASE_     )

									# Final norm
									SCREAMING_SNAKE_CASE_    			=			PoolFormerGroupNorm(config.hidden_sizes[-1]     )
									# Classifier head
									SCREAMING_SNAKE_CASE_    			=			(
									    nn.Linear(config.hidden_sizes[-1]							,							config.num_labels     ) if config.num_labels > 0 else nn.Identity()
									)

									# Initialize weights and apply final processing
									self.post_init()







							@add_start_docstrings_to_model_forward(SCREAMING_SNAKE_CASE_     )
							@add_code_sample_docstrings(
							    checkpoint=_IMAGE_CLASS_CHECKPOINT							,							output_type=SCREAMING_SNAKE_CASE_							,							config_class=_CONFIG_FOR_DOC							,							expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT							,							)
							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_ = None							,							SCREAMING_SNAKE_CASE_ = None							,							SCREAMING_SNAKE_CASE_ = None							,							SCREAMING_SNAKE_CASE_ = None							,							):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			return_dict if return_dict is not None else self.config.use_return_dict

									SCREAMING_SNAKE_CASE_    			=			self.poolformer(
									    SCREAMING_SNAKE_CASE_							,							output_hidden_states=SCREAMING_SNAKE_CASE_							,							return_dict=SCREAMING_SNAKE_CASE_							,							)

									SCREAMING_SNAKE_CASE_    			=			outputs[0]

									SCREAMING_SNAKE_CASE_    			=			self.classifier(self.norm(SCREAMING_SNAKE_CASE_     ).mean([-2, -1]     )     )

									SCREAMING_SNAKE_CASE_    			=			None
									if labels is not None:
											if self.config.problem_type is None:
													if self.num_labels == 1:
															SCREAMING_SNAKE_CASE_    			=			'''regression'''
													elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
															SCREAMING_SNAKE_CASE_    			=			'''single_label_classification'''
													else:
															SCREAMING_SNAKE_CASE_    			=			'''multi_label_classification'''

											if self.config.problem_type == "regression":
													SCREAMING_SNAKE_CASE_    			=			MSELoss()
													if self.num_labels == 1:
															SCREAMING_SNAKE_CASE_    			=			loss_fct(logits.squeeze()							,							labels.squeeze()     )
													else:
															SCREAMING_SNAKE_CASE_    			=			loss_fct(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     )
											elif self.config.problem_type == "single_label_classification":
													SCREAMING_SNAKE_CASE_    			=			CrossEntropyLoss()
													SCREAMING_SNAKE_CASE_    			=			loss_fct(logits.view(-1							,							self.num_labels     )							,							labels.view(-1     )     )
											elif self.config.problem_type == "multi_label_classification":
													SCREAMING_SNAKE_CASE_    			=			BCEWithLogitsLoss()
													SCREAMING_SNAKE_CASE_    			=			loss_fct(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     )

									if not return_dict:
											SCREAMING_SNAKE_CASE_    			=			(logits,) + outputs[2:]
											return ((loss,) + output) if loss is not None else output

									return ImageClassifierOutputWithNoAttention(loss=SCREAMING_SNAKE_CASE_							,							logits=SCREAMING_SNAKE_CASE_							,							hidden_states=outputs.hidden_states     )