
"""simple docstring"""




import argparse
import json
from pathlib import Path

import requests
import timm
import torch
from huggingface_hub import hf_hub_download
from PIL import Image
from timm.data import resolve_data_config
from timm.data.transforms_factory import create_transform

from transformers import (
    BitConfig,
    ViTHybridConfig,
    ViTHybridForImageClassification,
    ViTHybridImageProcessor,
    ViTHybridModel,
)
from transformers.image_utils import PILImageResampling
from transformers.utils import logging


logging.set_verbosity_info()
lowerCAmelCase__				=					logging.get_logger(__name__)



def 							_lowerCamelCase			(			__a,				__a=False     ):
		SCREAMING_SNAKE_CASE_    			=			[]

		# fmt: off
		# stem:
		rename_keys.append(('''cls_token''', '''vit.embeddings.cls_token''')     )
		rename_keys.append(('''pos_embed''', '''vit.embeddings.position_embeddings''')     )

		rename_keys.append(('''patch_embed.proj.weight''', '''vit.embeddings.patch_embeddings.projection.weight''')     )
		rename_keys.append(('''patch_embed.proj.bias''', '''vit.embeddings.patch_embeddings.projection.bias''')     )

		# backbone
		rename_keys.append(('''patch_embed.backbone.stem.conv.weight''', '''vit.embeddings.patch_embeddings.backbone.bit.embedder.convolution.weight''')     )
		rename_keys.append(('''patch_embed.backbone.stem.norm.weight''', '''vit.embeddings.patch_embeddings.backbone.bit.embedder.norm.weight''')     )
		rename_keys.append(('''patch_embed.backbone.stem.norm.bias''', '''vit.embeddings.patch_embeddings.backbone.bit.embedder.norm.bias''')     )

		for stage_idx in range(len(config.backbone_config.depths     )     ):
				for layer_idx in range(config.backbone_config.depths[stage_idx]     ):
						rename_keys.append((F'patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.conv1.weight', F'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.conv1.weight')     )
						rename_keys.append((F'patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm1.weight', F'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm1.weight')     )
						rename_keys.append((F'patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm1.bias', F'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm1.bias')     )
						rename_keys.append((F'patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.conv2.weight', F'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.conv2.weight')     )
						rename_keys.append((F'patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm2.weight', F'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm2.weight')     )
						rename_keys.append((F'patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm2.bias', F'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm2.bias')     )
						rename_keys.append((F'patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.conv3.weight', F'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.conv3.weight')     )
						rename_keys.append((F'patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm3.weight', F'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm3.weight')     )
						rename_keys.append((F'patch_embed.backbone.stages.{stage_idx}.blocks.{layer_idx}.norm3.bias', F'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.{layer_idx}.norm3.bias')     )

				rename_keys.append((F'patch_embed.backbone.stages.{stage_idx}.blocks.0.downsample.conv.weight', F'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.0.downsample.conv.weight')     )
				rename_keys.append((F'patch_embed.backbone.stages.{stage_idx}.blocks.0.downsample.norm.weight', F'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.0.downsample.norm.weight')     )
				rename_keys.append((F'patch_embed.backbone.stages.{stage_idx}.blocks.0.downsample.norm.bias', F'vit.embeddings.patch_embeddings.backbone.bit.encoder.stages.{stage_idx}.layers.0.downsample.norm.bias')     )

		# transformer encoder
		for i in range(config.num_hidden_layers     ):
				# encoder layers: output projection, 2 feedforward neural networks and 2 layernorms
				rename_keys.append((F'blocks.{i}.norm1.weight', F'vit.encoder.layer.{i}.layernorm_before.weight')     )
				rename_keys.append((F'blocks.{i}.norm1.bias', F'vit.encoder.layer.{i}.layernorm_before.bias')     )
				rename_keys.append((F'blocks.{i}.attn.proj.weight', F'vit.encoder.layer.{i}.attention.output.dense.weight')     )
				rename_keys.append((F'blocks.{i}.attn.proj.bias', F'vit.encoder.layer.{i}.attention.output.dense.bias')     )
				rename_keys.append((F'blocks.{i}.norm2.weight', F'vit.encoder.layer.{i}.layernorm_after.weight')     )
				rename_keys.append((F'blocks.{i}.norm2.bias', F'vit.encoder.layer.{i}.layernorm_after.bias')     )
				rename_keys.append((F'blocks.{i}.mlp.fc1.weight', F'vit.encoder.layer.{i}.intermediate.dense.weight')     )
				rename_keys.append((F'blocks.{i}.mlp.fc1.bias', F'vit.encoder.layer.{i}.intermediate.dense.bias')     )
				rename_keys.append((F'blocks.{i}.mlp.fc2.weight', F'vit.encoder.layer.{i}.output.dense.weight')     )
				rename_keys.append((F'blocks.{i}.mlp.fc2.bias', F'vit.encoder.layer.{i}.output.dense.bias')     )

		if base_model:
				# layernorm + pooler
				rename_keys.extend(
				    [
				        ('''norm.weight''', '''layernorm.weight'''),
				        ('''norm.bias''', '''layernorm.bias'''),
				        ('''pre_logits.fc.weight''', '''pooler.dense.weight'''),
				        ('''pre_logits.fc.bias''', '''pooler.dense.bias'''),
				    ]     )

				# if just the base model, we should remove "vit" from all keys that start with "vit"
				SCREAMING_SNAKE_CASE_    			=			[(pair[0], pair[1][4:]) if pair[1].startswith('''vit'''     ) else pair for pair in rename_keys]
		else:
				# layernorm + classification head
				rename_keys.extend(
				    [
				        ('''norm.weight''', '''vit.layernorm.weight'''),
				        ('''norm.bias''', '''vit.layernorm.bias'''),
				        ('''head.weight''', '''classifier.weight'''),
				        ('''head.bias''', '''classifier.bias'''),
				    ]     )
		# fmt: on

		return rename_keys



def 							_lowerCamelCase			(			__a,				__a,				__a=False     ):
		for i in range(config.num_hidden_layers     ):
				if base_model:
						SCREAMING_SNAKE_CASE_    			=			''''''
				else:
						SCREAMING_SNAKE_CASE_    			=			'''vit.'''
				# read in weights + bias of input projection layer (in timm, this is a single matrix + bias)
				SCREAMING_SNAKE_CASE_    			=			state_dict.pop(F'blocks.{i}.attn.qkv.weight'     )
				SCREAMING_SNAKE_CASE_    			=			state_dict.pop(F'blocks.{i}.attn.qkv.bias'     )
				# next, add query, keys and values (in that order) to the state dict
				SCREAMING_SNAKE_CASE_    			=			in_proj_weight[
				    : config.hidden_size, :
				]
				SCREAMING_SNAKE_CASE_    			=			in_proj_bias[: config.hidden_size]
				SCREAMING_SNAKE_CASE_    			=			in_proj_weight[
				    config.hidden_size : config.hidden_size * 2, :
				]
				SCREAMING_SNAKE_CASE_    			=			in_proj_bias[
				    config.hidden_size : config.hidden_size * 2
				]
				SCREAMING_SNAKE_CASE_    			=			in_proj_weight[
				    -config.hidden_size :, :
				]
				SCREAMING_SNAKE_CASE_    			=			in_proj_bias[-config.hidden_size :]



def 							_lowerCamelCase			(			__a     ):
		SCREAMING_SNAKE_CASE_    			=			['''head.weight''', '''head.bias''']
		for k in ignore_keys:
				state_dict.pop(__a,				__a     )



def 							_lowerCamelCase			(			__a,				__a,				__a     ):
		SCREAMING_SNAKE_CASE_    			=			dct.pop(__a     )
		SCREAMING_SNAKE_CASE_    			=			val



def 							_lowerCamelCase			(			):
		SCREAMING_SNAKE_CASE_    			=			'''http://images.cocodataset.org/val2017/000000039769.jpg'''
		SCREAMING_SNAKE_CASE_    			=			Image.open(requests.get(__a,				stream=__a     ).raw     )
		return im



@torch.no_grad()
def 							_lowerCamelCase			(			__a,				__a,				__a=False     ):
		SCREAMING_SNAKE_CASE_    			=			BitConfig(
		    global_padding='''same''',				layer_type='''bottleneck''',				depths=(3, 4, 9),				out_features=['''stage3'''],				embedding_dynamic_padding=__a,				)
		SCREAMING_SNAKE_CASE_    			=			ViTHybridConfig(backbone_config=__a,				image_size=384,				num_labels=1_000     )
		SCREAMING_SNAKE_CASE_    			=			False

		# load original model from timm
		SCREAMING_SNAKE_CASE_    			=			timm.create_model(__a,				pretrained=__a     )
		timm_model.eval()

		# load state_dict of original model, remove and rename some keys
		SCREAMING_SNAKE_CASE_    			=			timm_model.state_dict()
		if base_model:
				remove_classification_head_(__a     )
		SCREAMING_SNAKE_CASE_    			=			create_rename_keys(__a,				__a     )
		for src, dest in rename_keys:
				rename_key(__a,				__a,				__a     )
		read_in_q_k_v(__a,				__a,				__a     )

		SCREAMING_SNAKE_CASE_    			=			'''huggingface/label-files'''
		SCREAMING_SNAKE_CASE_    			=			'''imagenet-1k-id2label.json'''
		SCREAMING_SNAKE_CASE_    			=			json.load(open(hf_hub_download(__a,				__a,				repo_type='''dataset'''     ),				'''r'''     )     )
		SCREAMING_SNAKE_CASE_    			=			{int(__a     ): v for k, v in idalabel.items()}
		SCREAMING_SNAKE_CASE_    			=			idalabel
		SCREAMING_SNAKE_CASE_    			=			{v: k for k, v in idalabel.items()}

		# load HuggingFace model
		if vit_name[-5:] == "in21k":
				SCREAMING_SNAKE_CASE_    			=			ViTHybridModel(__a     ).eval()
		else:
				SCREAMING_SNAKE_CASE_    			=			ViTHybridForImageClassification(__a     ).eval()
		model.load_state_dict(__a     )

		# create image processor
		SCREAMING_SNAKE_CASE_    			=			create_transform(**resolve_data_config({},				model=__a     )     )
		SCREAMING_SNAKE_CASE_    			=			transform.transforms

		SCREAMING_SNAKE_CASE_    			=			{
		    '''bilinear''': PILImageResampling.BILINEAR,
		    '''bicubic''': PILImageResampling.BICUBIC,
		    '''nearest''': PILImageResampling.NEAREST,
		}

		SCREAMING_SNAKE_CASE_    			=			ViTHybridImageProcessor(
		    do_resize=__a,				size={'''shortest_edge''': timm_transforms[0].size},				resample=pillow_resamplings[timm_transforms[0].interpolation.value],				do_center_crop=__a,				crop_size={'''height''': timm_transforms[1].size[0], '''width''': timm_transforms[1].size[1]},				do_normalize=__a,				image_mean=timm_transforms[-1].mean.tolist(),				image_std=timm_transforms[-1].std.tolist(),				)

		SCREAMING_SNAKE_CASE_    			=			prepare_img()
		SCREAMING_SNAKE_CASE_    			=			transform(__a     ).unsqueeze(0     )
		SCREAMING_SNAKE_CASE_    			=			processor(__a,				return_tensors='''pt'''     ).pixel_values

		# verify pixel values
		assert torch.allclose(__a,				__a     )

		# verify logits
		with torch.no_grad():
				SCREAMING_SNAKE_CASE_    			=			model(__a     )
				SCREAMING_SNAKE_CASE_    			=			outputs.logits

		print('''Predicted class:''',				logits.argmax(-1     ).item()     )
		if base_model:
				SCREAMING_SNAKE_CASE_    			=			timm_model.forward_features(__a     )
				assert timm_pooled_output.shape == outputs.pooler_output.shape
				assert torch.allclose(__a,				outputs.pooler_output,				atol=1E-3     )
		else:
				SCREAMING_SNAKE_CASE_    			=			timm_model(__a     )
				assert timm_logits.shape == outputs.logits.shape
				assert torch.allclose(__a,				outputs.logits,				atol=1E-3     )
		print('''Looks ok!'''     )

		if pytorch_dump_folder_path is not None:
				Path(__a     ).mkdir(exist_ok=__a     )
				print(F'Saving model {vit_name} to {pytorch_dump_folder_path}'     )
				model.save_pretrained(__a     )
				print(F'Saving processor to {pytorch_dump_folder_path}'     )
				processor.save_pretrained(__a     )

		if push_to_hub:
				print(F'Pushing model and processor to the hub {vit_name}'     )
				model.push_to_hub(F'ybelkada/{vit_name}'     )
				processor.push_to_hub(F'ybelkada/{vit_name}'     )


if __name__ == "__main__":
						lowerCAmelCase__				=					argparse.ArgumentParser()
						# Required parameters
						parser.add_argument(
						    '--vit_name',
						    default='vit_base_r50_s16_384',
						    type=str,
						    help='Name of the hybrid ViT timm model you\'d like to convert.',
						)
						parser.add_argument(
						    '--pytorch_dump_folder_path', default=None, type=str, help='Path to the output PyTorch model directory.'
						)
						parser.add_argument(
						    '--push_to_hub', action='store_true', help='Whether to upload the model to the HuggingFace hub.'
						)

						lowerCAmelCase__				=					parser.parse_args()
						convert_vit_checkpoint(args.vit_name, args.pytorch_dump_folder_path, args.push_to_hub)

"""simple docstring"""




import collections.abc
from typing import Optional, Tuple, Union

import torch
import torch.utils.checkpoint
from torch import nn
from torch.nn import BCEWithLogitsLoss, CrossEntropyLoss, MSELoss

from ...activations import ACTaFN
from ...modeling_outputs import BaseModelOutputWithNoAttention, ImageClassifierOutputWithNoAttention
from ...modeling_utils import PreTrainedModel
from ...utils import add_code_sample_docstrings, add_start_docstrings, add_start_docstrings_to_model_forward, logging
from .configuration_poolformer import PoolFormerConfig


lowerCAmelCase__				=					logging.get_logger(__name__)

# General docstring
lowerCAmelCase__				=					'PoolFormerConfig'

# Base docstring
lowerCAmelCase__				=					'sail/poolformer_s12'
lowerCAmelCase__				=					[1, 512, 7, 7]

# Image classification docstring
lowerCAmelCase__				=					'sail/poolformer_s12'
lowerCAmelCase__				=					'tabby, tabby cat'

lowerCAmelCase__				=					[
    'sail/poolformer_s12',
    # See all PoolFormer models at https://huggingface.co/models?filter=poolformer
]



def 							_lowerCamelCase			(			__a,				__a = 0.0,				__a = False     ):
		if drop_prob == 0.0 or not training:
				return input
		SCREAMING_SNAKE_CASE_    			=			1 - drop_prob
		SCREAMING_SNAKE_CASE_    			=			(input.shape[0],) + (1,) * (input.ndim - 1)  # work with diff dim tensors, not just 2D ConvNets
		SCREAMING_SNAKE_CASE_    			=			keep_prob + torch.rand(__a,				dtype=input.dtype,				device=input.device     )
		random_tensor.floor_()  # binarize
		SCREAMING_SNAKE_CASE_    			=			input.div(__a     ) * random_tensor
		return output





class 			snake_case  (				nn.Module ):

							def __init__(self							,							SCREAMING_SNAKE_CASE_ = None     ):

									"""simple docstring"""







									super().__init__()
									SCREAMING_SNAKE_CASE_    			=			drop_prob

							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									return drop_path(SCREAMING_SNAKE_CASE_							,							self.drop_prob							,							self.training     )







							def _lowercase       (self     ):

									"""simple docstring"""







									return "p={}".format(self.drop_prob     )







class 			snake_case  (				nn.Module ):

							def __init__(self							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_=None     ):

									"""simple docstring"""







									super().__init__()
									SCREAMING_SNAKE_CASE_    			=			patch_size if isinstance(SCREAMING_SNAKE_CASE_							,							collections.abc.Iterable     ) else (patch_size, patch_size)
									SCREAMING_SNAKE_CASE_    			=			stride if isinstance(SCREAMING_SNAKE_CASE_							,							collections.abc.Iterable     ) else (stride, stride)
									SCREAMING_SNAKE_CASE_    			=			padding if isinstance(SCREAMING_SNAKE_CASE_							,							collections.abc.Iterable     ) else (padding, padding)

									SCREAMING_SNAKE_CASE_    			=			nn.Convad(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							kernel_size=SCREAMING_SNAKE_CASE_							,							stride=SCREAMING_SNAKE_CASE_							,							padding=SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			norm_layer(SCREAMING_SNAKE_CASE_     ) if norm_layer else nn.Identity()







							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			self.projection(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			self.norm(SCREAMING_SNAKE_CASE_     )
									return embeddings







class 			snake_case  (				nn.GroupNorm ):

							def __init__(self							,							SCREAMING_SNAKE_CASE_							,							**SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									super().__init__(1							,							SCREAMING_SNAKE_CASE_							,							**SCREAMING_SNAKE_CASE_     )







class 			snake_case  (				nn.Module ):

							def __init__(self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									super().__init__()
									SCREAMING_SNAKE_CASE_    			=			nn.AvgPoolad(SCREAMING_SNAKE_CASE_							,							stride=1							,							padding=pool_size // 2							,							count_include_pad=SCREAMING_SNAKE_CASE_     )







							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									return self.pool(SCREAMING_SNAKE_CASE_     ) - hidden_states







class 			snake_case  (				nn.Module ):

							def __init__(self							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									super().__init__()
									SCREAMING_SNAKE_CASE_    			=			nn.Convad(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							1     )
									SCREAMING_SNAKE_CASE_    			=			nn.Convad(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							1     )
									SCREAMING_SNAKE_CASE_    			=			PoolFormerDropPath(SCREAMING_SNAKE_CASE_     )
									if isinstance(config.hidden_act							,							SCREAMING_SNAKE_CASE_     ):
											SCREAMING_SNAKE_CASE_    			=			ACTaFN[config.hidden_act]
									else:
											SCREAMING_SNAKE_CASE_    			=			config.hidden_act







							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			self.conva(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			self.act_fn(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			self.drop(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			self.conva(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			self.drop(SCREAMING_SNAKE_CASE_     )

									return hidden_states







class 			snake_case  (				nn.Module ):

							def __init__(self							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									super().__init__()
									SCREAMING_SNAKE_CASE_    			=			PoolFormerPooling(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			PoolFormerOutput(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			PoolFormerGroupNorm(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			PoolFormerGroupNorm(SCREAMING_SNAKE_CASE_     )

									# Useful for training neural nets
									SCREAMING_SNAKE_CASE_    			=			PoolFormerDropPath(SCREAMING_SNAKE_CASE_     ) if drop_path > 0.0 else nn.Identity()
									SCREAMING_SNAKE_CASE_    			=			config.use_layer_scale
									if config.use_layer_scale:
											SCREAMING_SNAKE_CASE_    			=			nn.Parameter(
											    config.layer_scale_init_value * torch.ones((SCREAMING_SNAKE_CASE_)     )							,							requires_grad=SCREAMING_SNAKE_CASE_     )
											SCREAMING_SNAKE_CASE_    			=			nn.Parameter(
											    config.layer_scale_init_value * torch.ones((SCREAMING_SNAKE_CASE_)     )							,							requires_grad=SCREAMING_SNAKE_CASE_     )







							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									if self.use_layer_scale:
											SCREAMING_SNAKE_CASE_    			=			self.pooling(self.before_norm(SCREAMING_SNAKE_CASE_     )     )
											SCREAMING_SNAKE_CASE_    			=			self.layer_scale_a.unsqueeze(-1     ).unsqueeze(-1     ) * pooling_output
											# First residual connection
											SCREAMING_SNAKE_CASE_    			=			hidden_states + self.drop_path(SCREAMING_SNAKE_CASE_     )
											SCREAMING_SNAKE_CASE_    			=			()

											SCREAMING_SNAKE_CASE_    			=			self.output(self.after_norm(SCREAMING_SNAKE_CASE_     )     )
											SCREAMING_SNAKE_CASE_    			=			self.layer_scale_a.unsqueeze(-1     ).unsqueeze(-1     ) * layer_output
											# Second residual connection
											SCREAMING_SNAKE_CASE_    			=			hidden_states + self.drop_path(SCREAMING_SNAKE_CASE_     )

											SCREAMING_SNAKE_CASE_    			=			(output,) + outputs
											return outputs

									else:
											SCREAMING_SNAKE_CASE_    			=			self.drop_path(self.pooling(self.before_norm(SCREAMING_SNAKE_CASE_     )     )     )
											# First residual connection
											SCREAMING_SNAKE_CASE_    			=			pooling_output + hidden_states
											SCREAMING_SNAKE_CASE_    			=			()

											# Second residual connection inside the PoolFormerOutput block
											SCREAMING_SNAKE_CASE_    			=			self.drop_path(self.output(self.after_norm(SCREAMING_SNAKE_CASE_     )     )     )
											SCREAMING_SNAKE_CASE_    			=			hidden_states + layer_output

											SCREAMING_SNAKE_CASE_    			=			(output,) + outputs
											return outputs







class 			snake_case  (				nn.Module ):

							def __init__(self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									super().__init__()
									SCREAMING_SNAKE_CASE_    			=			config
									# stochastic depth decay rule
									SCREAMING_SNAKE_CASE_    			=			[x.item() for x in torch.linspace(0							,							config.drop_path_rate							,							sum(config.depths     )     )]

									# patch embeddings
									SCREAMING_SNAKE_CASE_    			=			[]
									for i in range(config.num_encoder_blocks     ):
											embeddings.append(
											    PoolFormerEmbeddings(
											        patch_size=config.patch_sizes[i]							,							stride=config.strides[i]							,							padding=config.padding[i]							,							num_channels=config.num_channels if i == 0 else config.hidden_sizes[i - 1]							,							hidden_size=config.hidden_sizes[i]							,							)     )
									SCREAMING_SNAKE_CASE_    			=			nn.ModuleList(SCREAMING_SNAKE_CASE_     )

									# Transformer blocks
									SCREAMING_SNAKE_CASE_    			=			[]
									SCREAMING_SNAKE_CASE_    			=			0
									for i in range(config.num_encoder_blocks     ):
											# each block consists of layers
											SCREAMING_SNAKE_CASE_    			=			[]
											if i != 0:
													cur += config.depths[i - 1]
											for j in range(config.depths[i]     ):
													layers.append(
													    PoolFormerLayer(
													        SCREAMING_SNAKE_CASE_							,							num_channels=config.hidden_sizes[i]							,							pool_size=config.pool_size							,							hidden_size=config.hidden_sizes[i]							,							intermediate_size=int(config.hidden_sizes[i] * config.mlp_ratio     )							,							drop_path=dpr[cur + j]							,							)     )
											blocks.append(nn.ModuleList(SCREAMING_SNAKE_CASE_     )     )

									SCREAMING_SNAKE_CASE_    			=			nn.ModuleList(SCREAMING_SNAKE_CASE_     )







							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_=False							,							SCREAMING_SNAKE_CASE_=True     ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			() if output_hidden_states else None

									SCREAMING_SNAKE_CASE_    			=			pixel_values
									for idx, layers in enumerate(zip(self.patch_embeddings							,							self.block     )     ):
											SCREAMING_SNAKE_CASE_			,SCREAMING_SNAKE_CASE_    			=			layers
											# Get patch embeddings from hidden_states
											SCREAMING_SNAKE_CASE_    			=			embedding_layer(SCREAMING_SNAKE_CASE_     )
											# Send the embeddings through the blocks
											for _, blk in enumerate(SCREAMING_SNAKE_CASE_     ):
													SCREAMING_SNAKE_CASE_    			=			blk(SCREAMING_SNAKE_CASE_     )
													SCREAMING_SNAKE_CASE_    			=			layer_outputs[0]

											if output_hidden_states:
													SCREAMING_SNAKE_CASE_    			=			all_hidden_states + (hidden_states,)

									if not return_dict:
											return tuple(v for v in [hidden_states, all_hidden_states] if v is not None     )

									return BaseModelOutputWithNoAttention(last_hidden_state=SCREAMING_SNAKE_CASE_							,							hidden_states=SCREAMING_SNAKE_CASE_     )







class 			snake_case  (				__lowercase ):
							UpperCAmelCase__      =   PoolFormerConfig
							UpperCAmelCase__      =   '''poolformer'''
							UpperCAmelCase__      =   '''pixel_values'''
							UpperCAmelCase__      =   True

							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									if isinstance(SCREAMING_SNAKE_CASE_							,							(nn.Linear, nn.Convad)     ):
											module.weight.data.normal_(mean=0.0							,							std=self.config.initializer_range     )
											if module.bias is not None:
													module.bias.data.zero_()
									elif isinstance(SCREAMING_SNAKE_CASE_							,							nn.LayerNorm     ):
											module.bias.data.zero_()
											module.weight.data.fill_(1.0     )







							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_=False     ):

									"""simple docstring"""







									if isinstance(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     ):
											SCREAMING_SNAKE_CASE_    			=			value







lowerCAmelCase__				=					r'\n    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use\n    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and\n    behavior.\n\n    Parameters:\n        config ([`PoolFormerConfig`]): Model configuration class with all the parameters of the model.\n            Initializing with a config file does not load the weights associated with the model, only the\n            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.\n'

lowerCAmelCase__				=					r'\n    Args:\n        pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`):\n            Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See\n            [`PoolFormerImageProcessor.__call__`] for details.\n'





@add_start_docstrings(
    '''The bare PoolFormer Model transformer outputting raw hidden-states without any specific head on top.'''							,      __lowercase							,      )
class 			snake_case  (				__lowercase ):

							def __init__(self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									super().__init__(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			config

									SCREAMING_SNAKE_CASE_    			=			PoolFormerEncoder(SCREAMING_SNAKE_CASE_     )

									# Initialize weights and apply final processing
									self.post_init()

							def _lowercase       (self     ):

									"""simple docstring"""







									return self.embeddings.patch_embeddings







							@add_start_docstrings_to_model_forward(SCREAMING_SNAKE_CASE_     )
							@add_code_sample_docstrings(
							    checkpoint=_CHECKPOINT_FOR_DOC							,							output_type=SCREAMING_SNAKE_CASE_							,							config_class=_CONFIG_FOR_DOC							,							modality='''vision'''							,							expected_output=_EXPECTED_OUTPUT_SHAPE							,							)
							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_ = None							,							SCREAMING_SNAKE_CASE_ = None							,							SCREAMING_SNAKE_CASE_ = None							,							):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			(
									    output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states
									)
									SCREAMING_SNAKE_CASE_    			=			return_dict if return_dict is not None else self.config.use_return_dict

									if pixel_values is None:
											raise ValueError('''You have to specify pixel_values'''     )

									SCREAMING_SNAKE_CASE_    			=			self.encoder(
									    SCREAMING_SNAKE_CASE_							,							output_hidden_states=SCREAMING_SNAKE_CASE_							,							return_dict=SCREAMING_SNAKE_CASE_							,							)
									SCREAMING_SNAKE_CASE_    			=			encoder_outputs[0]

									if not return_dict:
											return (sequence_output, None) + encoder_outputs[1:]

									return BaseModelOutputWithNoAttention(
									    last_hidden_state=SCREAMING_SNAKE_CASE_							,							hidden_states=encoder_outputs.hidden_states							,							)







class 			snake_case  (				nn.Module ):

							def __init__(self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									super().__init__()
									SCREAMING_SNAKE_CASE_    			=			nn.Linear(config.hidden_size							,							config.hidden_size     )







							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			self.dense(SCREAMING_SNAKE_CASE_     )
									return output







@add_start_docstrings(
    '''
    PoolFormer Model transformer with an image classification head on top
    '''							,      __lowercase							,      )
class 			snake_case  (				__lowercase ):

							def __init__(self							,							SCREAMING_SNAKE_CASE_     ):

									"""simple docstring"""







									super().__init__(SCREAMING_SNAKE_CASE_     )
									SCREAMING_SNAKE_CASE_    			=			config.num_labels
									SCREAMING_SNAKE_CASE_    			=			PoolFormerModel(SCREAMING_SNAKE_CASE_     )

									# Final norm
									SCREAMING_SNAKE_CASE_    			=			PoolFormerGroupNorm(config.hidden_sizes[-1]     )
									# Classifier head
									SCREAMING_SNAKE_CASE_    			=			(
									    nn.Linear(config.hidden_sizes[-1]							,							config.num_labels     ) if config.num_labels > 0 else nn.Identity()
									)

									# Initialize weights and apply final processing
									self.post_init()







							@add_start_docstrings_to_model_forward(SCREAMING_SNAKE_CASE_     )
							@add_code_sample_docstrings(
							    checkpoint=_IMAGE_CLASS_CHECKPOINT							,							output_type=SCREAMING_SNAKE_CASE_							,							config_class=_CONFIG_FOR_DOC							,							expected_output=_IMAGE_CLASS_EXPECTED_OUTPUT							,							)
							def _lowercase       (self							,							SCREAMING_SNAKE_CASE_ = None							,							SCREAMING_SNAKE_CASE_ = None							,							SCREAMING_SNAKE_CASE_ = None							,							SCREAMING_SNAKE_CASE_ = None							,							):

									"""simple docstring"""







									SCREAMING_SNAKE_CASE_    			=			return_dict if return_dict is not None else self.config.use_return_dict

									SCREAMING_SNAKE_CASE_    			=			self.poolformer(
									    SCREAMING_SNAKE_CASE_							,							output_hidden_states=SCREAMING_SNAKE_CASE_							,							return_dict=SCREAMING_SNAKE_CASE_							,							)

									SCREAMING_SNAKE_CASE_    			=			outputs[0]

									SCREAMING_SNAKE_CASE_    			=			self.classifier(self.norm(SCREAMING_SNAKE_CASE_     ).mean([-2, -1]     )     )

									SCREAMING_SNAKE_CASE_    			=			None
									if labels is not None:
											if self.config.problem_type is None:
													if self.num_labels == 1:
															SCREAMING_SNAKE_CASE_    			=			'''regression'''
													elif self.num_labels > 1 and (labels.dtype == torch.long or labels.dtype == torch.int):
															SCREAMING_SNAKE_CASE_    			=			'''single_label_classification'''
													else:
															SCREAMING_SNAKE_CASE_    			=			'''multi_label_classification'''

											if self.config.problem_type == "regression":
													SCREAMING_SNAKE_CASE_    			=			MSELoss()
													if self.num_labels == 1:
															SCREAMING_SNAKE_CASE_    			=			loss_fct(logits.squeeze()							,							labels.squeeze()     )
													else:
															SCREAMING_SNAKE_CASE_    			=			loss_fct(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     )
											elif self.config.problem_type == "single_label_classification":
													SCREAMING_SNAKE_CASE_    			=			CrossEntropyLoss()
													SCREAMING_SNAKE_CASE_    			=			loss_fct(logits.view(-1							,							self.num_labels     )							,							labels.view(-1     )     )
											elif self.config.problem_type == "multi_label_classification":
													SCREAMING_SNAKE_CASE_    			=			BCEWithLogitsLoss()
													SCREAMING_SNAKE_CASE_    			=			loss_fct(SCREAMING_SNAKE_CASE_							,							SCREAMING_SNAKE_CASE_     )

									if not return_dict:
											SCREAMING_SNAKE_CASE_    			=			(logits,) + outputs[2:]
											return ((loss,) + output) if loss is not None else output

									return ImageClassifierOutputWithNoAttention(loss=SCREAMING_SNAKE_CASE_							,							logits=SCREAMING_SNAKE_CASE_							,							hidden_states=outputs.hidden_states     )