





import math
from dataclasses import dataclass
from typing import Optional, Tuple, Union

import numpy as np
import torch

from ..configuration_utils import ConfigMixin, register_to_config
from ..utils import BaseOutput, randn_tensor
from .scheduling_utils import SchedulerMixin


@dataclass
# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->UnCLIP
class        __magic_name__	(				lowercase_				):





     """simple docstring"""



     __UpperCamelCase            =  42
     __UpperCamelCase            =  None




def   __snake_case    (       _lowerCAmelCase			:			str					,					_lowerCAmelCase			:			Union[str, Any]=0.9_99					,					_lowerCAmelCase			:			List[Any]="cosine"					,					)			->    Optional[int]:

   if alpha_transform_type == "cosine":

      def alpha_bar_fn(_lowerCAmelCase			:			Optional[Any]						):
         return math.cos((t + 0.0_08) / 1.0_08 * math.pi / 2						) ** 2

   elif alpha_transform_type == "exp":

      def alpha_bar_fn(_lowerCAmelCase			:			Dict						):
         return math.exp(t * -12.0						)

   else:
      raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}"						)

   A_       :					Union[str, Any]        =      []
   for i in range(__lowercase						):
      A_       :					Dict        =      i / num_diffusion_timesteps
      A_       :					Dict        =      (i + 1) / num_diffusion_timesteps
      betas.append(min(1 - alpha_bar_fn(__lowercase						) / alpha_bar_fn(__lowercase						)					,					__lowercase						)						)
   return torch.tensor(__lowercase					,					dtype=torch.floataa						)





class        __magic_name__	(				lowercase_     ,					lowercase_				):





     """simple docstring"""



     @register_to_config
     def __init__(				self							:Union[str, Any]      ,   snake_case							:int = 1_000      ,   snake_case							:str = "fixed_small_log"      ,   snake_case							:bool = True      ,   snake_case							:Optional[float] = 1.0      ,   snake_case							:str = "epsilon"      ,   snake_case							:str = "squaredcos_cap_v2"      ,   ):





        '''simple docstring'''




        if beta_schedule != "squaredcos_cap_v2":
           raise ValueError("UnCLIPScheduler only supports `beta_schedule`: 'squaredcos_cap_v2'"	)

        A_       :					str        =      betas_for_alpha_bar(UpperCamelCase_	)

        A_       :					int        =      1.0 - self.betas
        A_       :					str        =      torch.cumprod(self.alphas      ,   dim=0	)
        A_       :					Optional[Any]        =      torch.tensor(1.0	)

        # standard deviation of the initial noise distribution
        A_       :					Dict        =      1.0

        # setable values
        A_       :					Any        =      None
        A_       :					Optional[Any]        =      torch.from_numpy(np.arange(0      ,   UpperCamelCase_	)[::-1].copy()	)

        A_       :					Any        =      variance_type



     def     SCREAMING_SNAKE_CASE		(				self							:List[str]      ,   snake_case							:torch.FloatTensor      ,   snake_case							:Optional[int] = None	):





        '''simple docstring'''




        return sample



     def     SCREAMING_SNAKE_CASE		(				self							:Optional[Any]      ,   snake_case							:int      ,   snake_case							:Union[str, torch.device] = None	):





        '''simple docstring'''




        A_       :					str        =      num_inference_steps
        A_       :					Tuple        =      (self.config.num_train_timesteps - 1) / (self.num_inference_steps - 1)
        A_       :					Optional[Any]        =      (np.arange(0      ,   UpperCamelCase_	) * step_ratio).round()[::-1].copy().astype(np.intaa	)
        A_       :					Optional[int]        =      torch.from_numpy(UpperCamelCase_	).to(UpperCamelCase_	)



     def     SCREAMING_SNAKE_CASE		(				self							:List[str]      ,   snake_case							:Any      ,   snake_case							:List[str]=None      ,   snake_case							:str=None      ,   snake_case							:Optional[int]=None	):





        '''simple docstring'''




        if prev_timestep is None:
           A_       :					str        =      t - 1

        A_       :					List[str]        =      self.alphas_cumprod[t]
        A_       :					Any        =      self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.one
        A_       :					Optional[Any]        =      1 - alpha_prod_t
        A_       :					Optional[Any]        =      1 - alpha_prod_t_prev

        if prev_timestep == t - 1:
           A_       :					List[Any]        =      self.betas[t]
        else:
           A_       :					Tuple        =      1 - alpha_prod_t / alpha_prod_t_prev

        # For t > 0, compute predicted variance βt (see formula (6) and (7) from https://arxiv.org/pdf/2006.11239.pdf)
        # and sample from it to get previous sample
        # x_{t-1} ~ N(pred_prev_sample, variance) == add variance to pred_sample
        A_       :					Union[str, Any]        =      beta_prod_t_prev / beta_prod_t * beta

        if variance_type is None:
           A_       :					List[Any]        =      self.config.variance_type

        # hacks - were probably added for training stability
        if variance_type == "fixed_small_log":
           A_       :					List[str]        =      torch.log(torch.clamp(UpperCamelCase_      ,   min=1e-20	)	)
           A_       :					Union[str, Any]        =      torch.exp(0.5 * variance	)
        elif variance_type == "learned_range":
           # NOTE difference with DDPM scheduler
           A_       :					Tuple        =      variance.log()
           A_       :					Union[str, Any]        =      beta.log()

           A_       :					Union[str, Any]        =      (predicted_variance + 1) / 2
           A_       :					Union[str, Any]        =      frac * max_log + (1 - frac) * min_log

        return variance



     def     SCREAMING_SNAKE_CASE		(				self							:Any      ,   snake_case							:torch.FloatTensor      ,   snake_case							:int      ,   snake_case							:torch.FloatTensor      ,   snake_case							:Optional[int] = None      ,   snake_case							:List[str]=None      ,   snake_case							:bool = True      ,   ):





        '''simple docstring'''




        A_       :					List[str]        =      timestep

        if model_output.shape[1] == sample.shape[1] * 2 and self.variance_type == "learned_range":
           A_   ,    A_       :					List[Any]        =      torch.split(UpperCamelCase_      ,   sample.shape[1]      ,   dim=1	)
        else:
           A_       :					str        =      None

        # 1. compute alphas, betas
        if prev_timestep is None:
           A_       :					Optional[Any]        =      t - 1

        A_       :					int        =      self.alphas_cumprod[t]
        A_       :					str        =      self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.one
        A_       :					int        =      1 - alpha_prod_t
        A_       :					Union[str, Any]        =      1 - alpha_prod_t_prev

        if prev_timestep == t - 1:
           A_       :					List[str]        =      self.betas[t]
           A_       :					Any        =      self.alphas[t]
        else:
           A_       :					Any        =      1 - alpha_prod_t / alpha_prod_t_prev
           A_       :					Optional[Any]        =      1 - beta

        # 2. compute predicted original sample from predicted noise also called
        # "predicted x_0" of formula (15) from https://arxiv.org/pdf/2006.11239.pdf
        if self.config.prediction_type == "epsilon":
           A_       :					str        =      (sample - beta_prod_t ** 0.5 * model_output) / alpha_prod_t ** 0.5
        elif self.config.prediction_type == "sample":
           A_       :					Any        =      model_output
        else:
           raise ValueError(
               f"prediction_type given as {self.config.prediction_type} must be one of `epsilon` or `sample`"
               " for the UnCLIPScheduler."	)

        # 3. Clip "predicted x_0"
        if self.config.clip_sample:
           A_       :					List[Any]        =      torch.clamp(
               UpperCamelCase_      ,   -self.config.clip_sample_range      ,   self.config.clip_sample_range	)

        # 4. Compute coefficients for pred_original_sample x_0 and current sample x_t
        # See formula (7) from https://arxiv.org/pdf/2006.11239.pdf
        A_       :					List[str]        =      (alpha_prod_t_prev ** 0.5 * beta) / beta_prod_t
        A_       :					Tuple        =      alpha ** 0.5 * beta_prod_t_prev / beta_prod_t

        # 5. Compute predicted previous sample µ_t
        # See formula (7) from https://arxiv.org/pdf/2006.11239.pdf
        A_       :					Optional[Any]        =      pred_original_sample_coeff * pred_original_sample + current_sample_coeff * sample

        # 6. Add noise
        A_       :					str        =      0
        if t > 0:
           A_       :					Tuple        =      randn_tensor(
               model_output.shape      ,   dtype=model_output.dtype      ,   generator=UpperCamelCase_      ,   device=model_output.device	)

           A_       :					Optional[Any]        =      self._get_variance(
               UpperCamelCase_      ,   predicted_variance=UpperCamelCase_      ,   prev_timestep=UpperCamelCase_      ,   )

           if self.variance_type == "fixed_small_log":
              A_       :					List[Any]        =      variance
           elif self.variance_type == "learned_range":
              A_       :					Dict        =      (0.5 * variance).exp()
           else:
              raise ValueError(
                  f"variance_type given as {self.variance_type} must be one of `fixed_small_log` or `learned_range`"
                  " for the UnCLIPScheduler."	)

           A_       :					Optional[int]        =      variance * variance_noise

        A_       :					int        =      pred_prev_sample + variance

        if not return_dict:
           return (pred_prev_sample,)

        return UnCLIPSchedulerOutput(prev_sample=UpperCamelCase_      ,   pred_original_sample=UpperCamelCase_	)





     def     SCREAMING_SNAKE_CASE		(				self							:Optional[Any]      ,   snake_case							:torch.FloatTensor      ,   snake_case							:torch.FloatTensor      ,   snake_case							:torch.IntTensor      ,   ):





        '''simple docstring'''




        A_       :					Dict        =      self.alphas_cumprod.to(device=original_samples.device      ,   dtype=original_samples.dtype	)
        A_       :					Union[str, Any]        =      timesteps.to(original_samples.device	)

        A_       :					List[Any]        =      alphas_cumprod[timesteps] ** 0.5
        A_       :					Any        =      sqrt_alpha_prod.flatten()
        while len(sqrt_alpha_prod.shape	) < len(original_samples.shape	):
           A_       :					List[str]        =      sqrt_alpha_prod.unsqueeze(-1	)

        A_       :					Optional[Any]        =      (1 - alphas_cumprod[timesteps]) ** 0.5
        A_       :					Optional[Any]        =      sqrt_one_minus_alpha_prod.flatten()
        while len(sqrt_one_minus_alpha_prod.shape	) < len(original_samples.shape	):
           A_       :					Dict        =      sqrt_one_minus_alpha_prod.unsqueeze(-1	)

        A_       :					List[Any]        =      sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise
        return noisy_samples








import unittest

from transformers import SqueezeBertConfig, is_torch_available
from transformers.testing_utils import require_sentencepiece, require_tokenizers, require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      SQUEEZEBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      SqueezeBertForMaskedLM,
      SqueezeBertForMultipleChoice,
      SqueezeBertForQuestionAnswering,
      SqueezeBertForSequenceClassification,
      SqueezeBertForTokenClassification,
      SqueezeBertModel,
  )


class    __lowercase   (						lowercase_	):







     '''simple docstring'''




     def __init__(					self							:      Optional[int]			,						UpperCamelCase_							:      Tuple			,						UpperCamelCase_							:      List[str]=13			,						UpperCamelCase_							:      int=7			,						UpperCamelCase_							:      Dict=True			,						UpperCamelCase_							:      int=True			,						UpperCamelCase_							:      str=False			,						UpperCamelCase_							:      Dict=True			,						UpperCamelCase_							:      Union[str, Any]=99			,						UpperCamelCase_							:      Any=32			,						UpperCamelCase_							:      Union[str, Any]=5			,						UpperCamelCase_							:      List[str]=4			,						UpperCamelCase_							:      Union[str, Any]=64			,						UpperCamelCase_							:      Any="gelu"			,						UpperCamelCase_							:      str=0.1			,						UpperCamelCase_							:      List[Any]=0.1			,						UpperCamelCase_							:      int=512			,						UpperCamelCase_							:      Dict=16			,						UpperCamelCase_							:      List[str]=2			,						UpperCamelCase_							:      int=0.02			,						UpperCamelCase_							:      Dict=3			,						UpperCamelCase_							:      List[Any]=4			,						UpperCamelCase_							:      int=None			,						UpperCamelCase_							:      Any=2			,						UpperCamelCase_							:      Any=2			,						UpperCamelCase_							:      int=2			,						UpperCamelCase_							:      Optional[Any]=2			,						UpperCamelCase_							:      str=4			,						UpperCamelCase_							:      List[str]=1			,						):


         """simple docstring"""



         __A       =						parent
         __A       =						batch_size
         __A       =						seq_length
         __A       =						is_training
         __A       =						use_input_mask
         __A       =						use_token_type_ids
         __A       =						use_labels
         __A       =						vocab_size
         __A       =						hidden_size
         __A       =						num_hidden_layers
         __A       =						num_attention_heads
         __A       =						intermediate_size
         __A       =						hidden_act
         __A       =						hidden_dropout_prob
         __A       =						attention_probs_dropout_prob
         __A       =						max_position_embeddings
         __A       =						type_vocab_size
         __A       =						type_sequence_label_size
         __A       =						initializer_range
         __A       =						num_labels
         __A       =						num_choices
         __A       =						scope
         __A       =						q_groups
         __A       =						k_groups
         __A       =						v_groups
         __A       =						post_attention_groups
         __A       =						intermediate_groups
         __A       =						output_groups




     def 		lowerCAmelCase_  (					self							:      Dict					):


         """simple docstring"""



         __A       =						ids_tensor([self.batch_size, self.seq_length]			,						self.vocab_size					)

         __A       =						None
         if self.use_input_mask:
             __A       =						random_attention_mask([self.batch_size, self.seq_length]					)

         __A       =						None
         __A       =						None
         __A       =						None
         if self.use_labels:
             __A       =						ids_tensor([self.batch_size]			,						self.type_sequence_label_size					)
             __A       =						ids_tensor([self.batch_size, self.seq_length]			,						self.num_labels					)
             __A       =						ids_tensor([self.batch_size]			,						self.num_choices					)

         __A       =						self.get_config()

         return config, input_ids, input_mask, sequence_labels, token_labels, choice_labels




     def 		lowerCAmelCase_  (					self							:      Any					):


         """simple docstring"""



         return SqueezeBertConfig(
             embedding_size=self.hidden_size			,						vocab_size=self.vocab_size			,						hidden_size=self.hidden_size			,						num_hidden_layers=self.num_hidden_layers			,						num_attention_heads=self.num_attention_heads			,						intermediate_size=self.intermediate_size			,						hidden_act=self.hidden_act			,						attention_probs_dropout_prob=self.hidden_dropout_prob			,						attention_dropout=self.attention_probs_dropout_prob			,						max_position_embeddings=self.max_position_embeddings			,						initializer_range=self.initializer_range			,						q_groups=self.q_groups			,						k_groups=self.k_groups			,						v_groups=self.v_groups			,						post_attention_groups=self.post_attention_groups			,						intermediate_groups=self.intermediate_groups			,						output_groups=self.output_groups			,						)




     def 		lowerCAmelCase_  (					self							:      Tuple			,						UpperCamelCase_							:      List[Any]			,						UpperCamelCase_							:      int			,						UpperCamelCase_							:      int			,						UpperCamelCase_							:      List[Any]			,						UpperCamelCase_							:      int			,						UpperCamelCase_							:      Dict					):


         """simple docstring"""



         __A       =						SqueezeBertModel(config=UpperCamelCase_					)
         model.to(UpperCamelCase_					)
         model.eval()
         __A       =						model(UpperCamelCase_			,						UpperCamelCase_					)
         __A       =						model(UpperCamelCase_					)
         self.parent.assertEqual(result.last_hidden_state.shape			,						(self.batch_size, self.seq_length, self.hidden_size)					)




     def 		lowerCAmelCase_  (					self							:      List[str]			,						UpperCamelCase_							:      Optional[int]			,						UpperCamelCase_							:      str			,						UpperCamelCase_							:      Optional[int]			,						UpperCamelCase_							:      List[str]			,						UpperCamelCase_							:      Union[str, Any]			,						UpperCamelCase_							:      Optional[int]					):


         """simple docstring"""



         __A       =						SqueezeBertForMaskedLM(config=UpperCamelCase_					)
         model.to(UpperCamelCase_					)
         model.eval()
         __A       =						model(UpperCamelCase_			,						attention_mask=UpperCamelCase_			,						labels=UpperCamelCase_					)
         self.parent.assertEqual(result.logits.shape			,						(self.batch_size, self.seq_length, self.vocab_size)					)




     def 		lowerCAmelCase_  (					self							:      Optional[int]			,						UpperCamelCase_							:      int			,						UpperCamelCase_							:      Optional[Any]			,						UpperCamelCase_							:      Optional[Any]			,						UpperCamelCase_							:      Optional[Any]			,						UpperCamelCase_							:      Union[str, Any]			,						UpperCamelCase_							:      Optional[Any]					):


         """simple docstring"""



         __A       =						SqueezeBertForQuestionAnswering(config=UpperCamelCase_					)
         model.to(UpperCamelCase_					)
         model.eval()
         __A       =						model(
             UpperCamelCase_			,						attention_mask=UpperCamelCase_			,						start_positions=UpperCamelCase_			,						end_positions=UpperCamelCase_					)
         self.parent.assertEqual(result.start_logits.shape			,						(self.batch_size, self.seq_length)					)
         self.parent.assertEqual(result.end_logits.shape			,						(self.batch_size, self.seq_length)					)




     def 		lowerCAmelCase_  (					self							:      str			,						UpperCamelCase_							:      Tuple			,						UpperCamelCase_							:      List[Any]			,						UpperCamelCase_							:      Optional[int]			,						UpperCamelCase_							:      Any			,						UpperCamelCase_							:      List[Any]			,						UpperCamelCase_							:      Any					):


         """simple docstring"""



         __A       =						self.num_labels
         __A       =						SqueezeBertForSequenceClassification(UpperCamelCase_					)
         model.to(UpperCamelCase_					)
         model.eval()
         __A       =						model(UpperCamelCase_			,						attention_mask=UpperCamelCase_			,						labels=UpperCamelCase_					)
         self.parent.assertEqual(result.logits.shape			,						(self.batch_size, self.num_labels)					)




     def 		lowerCAmelCase_  (					self							:      Any			,						UpperCamelCase_							:      Optional[Any]			,						UpperCamelCase_							:      str			,						UpperCamelCase_							:      Dict			,						UpperCamelCase_							:      List[Any]			,						UpperCamelCase_							:      Dict			,						UpperCamelCase_							:      int					):


         """simple docstring"""



         __A       =						self.num_labels
         __A       =						SqueezeBertForTokenClassification(config=UpperCamelCase_					)
         model.to(UpperCamelCase_					)
         model.eval()

         __A       =						model(UpperCamelCase_			,						attention_mask=UpperCamelCase_			,						labels=UpperCamelCase_					)
         self.parent.assertEqual(result.logits.shape			,						(self.batch_size, self.seq_length, self.num_labels)					)




     def 		lowerCAmelCase_  (					self							:      Tuple			,						UpperCamelCase_							:      List[str]			,						UpperCamelCase_							:      List[Any]			,						UpperCamelCase_							:      List[Any]			,						UpperCamelCase_							:      Tuple			,						UpperCamelCase_							:      Optional[Any]			,						UpperCamelCase_							:      Dict					):


         """simple docstring"""



         __A       =						self.num_choices
         __A       =						SqueezeBertForMultipleChoice(config=UpperCamelCase_					)
         model.to(UpperCamelCase_					)
         model.eval()
         __A       =						input_ids.unsqueeze(1					).expand(-1			,						self.num_choices			,						-1					).contiguous()
         __A       =						input_mask.unsqueeze(1					).expand(-1			,						self.num_choices			,						-1					).contiguous()
         __A       =						model(
             UpperCamelCase_			,						attention_mask=UpperCamelCase_			,						labels=UpperCamelCase_			,						)
         self.parent.assertEqual(result.logits.shape			,						(self.batch_size, self.num_choices)					)





     def 		lowerCAmelCase_  (					self							:      int					):


         """simple docstring"""



         __A       =						self.prepare_config_and_inputs()
         ((__A)     ,						(__A)     ,						(__A)     ,						(__A)     ,						(__A)     ,						(__A))       =						config_and_inputs
         __A       =						{"""input_ids""": input_ids, """attention_mask""": input_mask}
         return config, inputs_dict


@require_torch
class    __lowercase   (						lowercase_      ,   lowercase_      ,   unittest.TestCase	):







     '''simple docstring'''
     SCREAMING_SNAKE_CASE     =					(
         (
             SqueezeBertModel,
             SqueezeBertForMaskedLM,
             SqueezeBertForMultipleChoice,
             SqueezeBertForQuestionAnswering,
             SqueezeBertForSequenceClassification,
             SqueezeBertForTokenClassification,
         )
         if is_torch_available()
         else None
     )
     SCREAMING_SNAKE_CASE     =					(
         {
             "feature-extraction": SqueezeBertModel,
             "fill-mask": SqueezeBertForMaskedLM,
             "question-answering": SqueezeBertForQuestionAnswering,
             "text-classification": SqueezeBertForSequenceClassification,
             "token-classification": SqueezeBertForTokenClassification,
             "zero-shot": SqueezeBertForSequenceClassification,
         }
         if is_torch_available()
         else {}
     )
     SCREAMING_SNAKE_CASE     =					False
     SCREAMING_SNAKE_CASE     =					True
     SCREAMING_SNAKE_CASE     =					False




     def 		lowerCAmelCase_  (					self							:      Tuple					):


         """simple docstring"""



         __A       =						SqueezeBertModelTester(self					)
         __A       =						ConfigTester(self			,						config_class=UpperCamelCase_			,						dim=37					)




     def 		lowerCAmelCase_  (					self							:      Optional[Any]					):


         """simple docstring"""



         self.config_tester.run_common_tests()




     def 		lowerCAmelCase_  (					self							:      str					):


         """simple docstring"""



         __A       =						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_squeezebert_model(*UpperCamelCase_					)




     def 		lowerCAmelCase_  (					self							:      Union[str, Any]					):


         """simple docstring"""



         __A       =						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_squeezebert_for_masked_lm(*UpperCamelCase_					)




     def 		lowerCAmelCase_  (					self							:      Union[str, Any]					):


         """simple docstring"""



         __A       =						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_squeezebert_for_question_answering(*UpperCamelCase_					)




     def 		lowerCAmelCase_  (					self							:      Any					):


         """simple docstring"""



         __A       =						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_squeezebert_for_sequence_classification(*UpperCamelCase_					)




     def 		lowerCAmelCase_  (					self							:      Union[str, Any]					):


         """simple docstring"""



         __A       =						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_squeezebert_for_token_classification(*UpperCamelCase_					)




     def 		lowerCAmelCase_  (					self							:      Dict					):


         """simple docstring"""



         __A       =						self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_squeezebert_for_multiple_choice(*UpperCamelCase_					)





     @slow
     def 		lowerCAmelCase_  (					self							:      int					):


         """simple docstring"""



         for model_name in SQUEEZEBERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             __A       =						SqueezeBertModel.from_pretrained(UpperCamelCase_					)
             self.assertIsNotNone(UpperCamelCase_					)


@require_sentencepiece
@require_tokenizers
@require_torch
class    __lowercase   (						unittest.TestCase	):







     '''simple docstring'''




     @slow
     def 		lowerCAmelCase_  (					self							:      Optional[int]					):


         """simple docstring"""



         __A       =						SqueezeBertForSequenceClassification.from_pretrained("""squeezebert/squeezebert-mnli"""					)

         __A       =						torch.tensor([[1, 29_414, 232, 328, 740, 1_140, 12_695, 69, 13, 1_588, 2]]					)
         __A       =						model(UpperCamelCase_					)[0]
         __A       =						torch.Size((1, 3)					)
         self.assertEqual(output.shape			,						UpperCamelCase_					)
         __A       =						torch.tensor([[0.6401, -0.0349, -0.6041]]					)
         self.assertTrue(torch.allclose(UpperCamelCase_			,						UpperCamelCase_			,						atol=1e-4					)					)




