






"""simple docstring"""




import gc
import importlib.metadata
import tempfile
import unittest

from packaging import version

from transformers import (
    AutoModel,
    AutoModelForCausalLM,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoTokenizer,
    BitsAndBytesConfig,
    pipeline,
)
from transformers.testing_utils import (
    is_torch_available,
    require_accelerate,
    require_bitsandbytes,
    require_torch,
    require_torch_gpu,
    require_torch_multi_gpu,
    slow,
)

def 			lowerCAmelCase__     (     UpperCamelCase__				):



  '''simple docstring'''
  if model.config.model_type == "gpt2":
    return model.transformer.h[0].mlp.c_fc
  return model.transformer.h[0].mlp.dense_ah_to_h


if is_torch_available():
 import torch
 import torch.nn as nn






 class        UpperCamelCase			(	nn.Module      ):


  def __init__(     self		:						Union[str, Any]							,						UpperCAmelCase__		:						nn.Module							,						UpperCAmelCase__		:						int							)					->       str:
    super().__init__()
    _a  :       Dict             =	module
    _a  :       Optional[int]             =	nn.Sequential(
        nn.Linear(module.in_features							,						UpperCAmelCase__							,						bias=UpperCAmelCase__							)							,						nn.Linear(UpperCAmelCase__							,						module.out_features							,						bias=UpperCAmelCase__							)							,						)
    _a  :       Union[str, Any]             =	(2.0 / (5 * min(module.in_features							,						module.out_features							))) ** 0.5
    nn.init.normal_(self.adapter[0].weight							,						std=UpperCAmelCase__							)
    nn.init.zeros_(self.adapter[1].weight							)
    self.adapter.to(module.weight.device							)







  def _lowercase	(     self		:						Tuple							,						UpperCAmelCase__		:						Tuple							,						*UpperCAmelCase__		:						Union[str, Any]							,						**UpperCAmelCase__		:						Optional[Any]							)					->       Optional[int]:
    return self.module(UpperCAmelCase__							,						*UpperCAmelCase__							,						**UpperCAmelCase__							) + self.adapter(UpperCAmelCase__							)






@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class        UpperCamelCase			(	unittest.TestCase      ):
 # We keep the constants inside the init function and model loading inside setUp function

 # We need to test on relatively large models (aka >1b parameters otherwise the quantiztion may not work as expected)
 # Therefore here we use only bloom-1b3 to test our module
 UpperCamelCase					:       Optional[Any]			=					'''bigscience/bloom-1b7'''

 # Constant values
 UpperCamelCase					:       List[str]			=					2.1_0_9_6_5_9_5_5_2_6_9_2_5_7_4

 UpperCamelCase					:       int			=					'''Hello my name is'''
 UpperCamelCase					:       Dict			=					set()
 EXPECTED_OUTPUTS.add('''Hello my name is John and I am a professional photographer. I'''      )
 EXPECTED_OUTPUTS.add('''Hello my name is John.\nI am a friend of your father.\n'''      )
 EXPECTED_OUTPUTS.add('''Hello my name is John Doe, I am a student at the University'''      )
 UpperCamelCase					:       Optional[Any]			=					10


 def _lowercase	(     self		:						Union[str, Any]							)					->       List[str]:
   # Models and tokenizer
   _a  :       Dict             =	AutoTokenizer.from_pretrained(self.model_name							)







class        UpperCamelCase			(	snake_case_      ):


 def _lowercase	(     self		:						int							)					->       Optional[Any]:
   super().setUp()

   # Models and tokenizer
   _a  :       Optional[int]             =	AutoModelForCausalLM.from_pretrained(
       self.model_name							,						torch_dtype=torch.floataa							,						device_map="""auto"""							)
   _a  :       Optional[int]             =	AutoModelForCausalLM.from_pretrained(self.model_name							,						load_in_abit=UpperCAmelCase__							,						device_map="""auto"""							)


 def _lowercase	(     self		:						Any							)					->       Optional[Any]:
   del self.model_fpaa
   del self.model_abit

   gc.collect()
   torch.cuda.empty_cache()


 def _lowercase	(     self		:						List[str]							)					->       Union[str, Any]:
   _a  :       Dict             =	self.model_abit.config

   self.assertTrue(hasattr(UpperCAmelCase__							,						"""quantization_config"""							)							)

   _a  :       Optional[Any]             =	config.to_dict()
   _a  :       List[str]             =	config.to_diff_dict()

   _a  :       Tuple             =	config.to_json_string()


 def _lowercase	(     self		:						Dict							)					->       List[str]:
   from bitsandbytes.nn import Paramsabit

   _a  :       Optional[Any]             =	self.model_fpaa.get_memory_footprint()
   _a  :       Optional[int]             =	self.model_abit.get_memory_footprint()

   self.assertAlmostEqual(mem_fpaa / mem_abit							,						self.EXPECTED_RELATIVE_DIFFERENCE							)
   _a  :       Optional[Any]             =	get_some_linear_layer(self.model_abit							)
   self.assertTrue(linear.weight.__class__ == Paramsabit							)


 def _lowercase	(     self		:						Union[str, Any]							)					->       List[Any]:
   from transformers import TaPreTrainedModel

   self.model_fpaa.get_memory_footprint()
   self.model_abit.get_memory_footprint()

   for name, module in self.model_abit.named_modules():
     if isinstance(UpperCAmelCase__							,						torch.nn.Linear							):
       if name not in ["lm_head"] + TaPreTrainedModel._keep_in_fpaa_modules:
         # 4-bit parameters are packed in uint8 variables
         self.assertTrue(module.weight.dtype == torch.uinta							)


 def _lowercase	(     self		:						Union[str, Any]							)					->       List[str]:
   _a  :       Optional[int]             =	self.tokenizer(self.input_text							,						return_tensors="""pt"""							)
   _a  :       Optional[int]             =	self.model_abit.generate(input_ids=encoded_input["""input_ids"""].to(0							)							,						max_new_tokens=10							)

   self.assertIn(self.tokenizer.decode(output_sequences[0]							,						skip_special_tokens=UpperCAmelCase__							)							,						self.EXPECTED_OUTPUTS							)


 def _lowercase	(     self		:						Optional[int]							)					->       str:
   _a  :       str             =	BitsAndBytesConfig()
   _a  :       Any             =	True

   _a  :       int             =	AutoModelForCausalLM.from_pretrained(
       self.model_name							,						quantization_config=UpperCAmelCase__							,						device_map="""auto"""							)

   _a  :       Optional[Any]             =	self.tokenizer(self.input_text							,						return_tensors="""pt"""							)
   _a  :       Union[str, Any]             =	model_abit_from_config.generate(
       input_ids=encoded_input["""input_ids"""].to(0							)							,						max_new_tokens=10							)

   self.assertIn(self.tokenizer.decode(output_sequences[0]							,						skip_special_tokens=UpperCAmelCase__							)							,						self.EXPECTED_OUTPUTS							)


 def _lowercase	(     self		:						List[str]							)					->       Tuple:
   with self.assertRaises(UpperCAmelCase__							), tempfile.TemporaryDirectory() as tmpdirname:
     self.model_abit.save_pretrained(UpperCAmelCase__							)


 def _lowercase	(     self		:						int							)					->       int:
   _a  :       Any             =	BitsAndBytesConfig()

   with self.assertRaises(UpperCAmelCase__							):
     _a  :       List[Any]             =	AutoModelForCausalLM.from_pretrained(
         self.model_name							,						quantization_config=UpperCAmelCase__							,						load_in_abit=UpperCAmelCase__							,						device_map="""auto"""							,						bnb_abit_quant_type="""nf4"""							,						)


 def _lowercase	(     self		:						Dict							)					->       Any:
   with self.assertRaises(UpperCAmelCase__							):
     # Tries with `str`
     self.model_abit.to("""cpu"""							)

   with self.assertRaises(UpperCAmelCase__							):
     # Tries with a `dtype``
     self.model_abit.to(torch.floataa							)

   with self.assertRaises(UpperCAmelCase__							):
     # Tries with a `device`
     self.model_abit.to(torch.device("""cuda:0"""							)							)

   with self.assertRaises(UpperCAmelCase__							):
     # Tries with a `device`
     self.model_abit.float()

   with self.assertRaises(UpperCAmelCase__							):
     # Tries with a `device`
     self.model_abit.half()

   # Test if we did not break anything
   _a  :       Optional[int]             =	self.tokenizer(self.input_text							,						return_tensors="""pt"""							)

   _a  :       Any             =	self.model_fpaa.to(torch.floataa							)
   _a  :       Union[str, Any]             =	self.model_fpaa.generate(input_ids=encoded_input["""input_ids"""].to(0							)							,						max_new_tokens=10							)

   # Check this does not throw an error
   _a  :       int             =	self.model_fpaa.to("""cpu"""							)

   # Check this does not throw an error
   _a  :       Optional[int]             =	self.model_fpaa.half()

   # Check this does not throw an error
   _a  :       int             =	self.model_fpaa.float()







 def _lowercase	(     self		:						Union[str, Any]							)					->       Optional[int]:
   _a  :       Optional[Any]             =	AutoModelForSeqaSeqLM.from_pretrained("""t5-small"""							,						load_in_abit=UpperCAmelCase__							,						device_map="""auto"""							)
   self.assertTrue(model.decoder.block[0].layer[2].DenseReluDense.wo.weight.dtype == torch.floataa							)







@require_bitsandbytes
@require_accelerate
@require_torch
@require_torch_gpu
@slow
class        UpperCamelCase			(	unittest.TestCase      ):


 @classmethod
 def _lowercase	(     cls		:						Optional[int]							)					->       Any:
   _a  :       Any             =	"""t5-small"""
   _a  :       Any             =	"""google/flan-t5-small"""  # flan-t5 uses dense-act instead of dense-relu-dense
   _a  :       str             =	AutoTokenizer.from_pretrained(cls.model_name							)
   _a  :       Tuple             =	"""Translate in German: Hello, my dog is cute"""


 def _lowercase	(     self		:						List[Any]							)					->       Optional[int]:
   gc.collect()
   torch.cuda.empty_cache()


 def _lowercase	(     self		:						List[str]							)					->       Optional[int]:
   from transformers import TaForConditionalGeneration

   _a  :       List[Any]             =	TaForConditionalGeneration._keep_in_fpaa_modules
   _a  :       Dict             =	None

   # test with `t5-small`
   _a  :       Union[str, Any]             =	TaForConditionalGeneration.from_pretrained(self.model_name							,						load_in_abit=UpperCAmelCase__							,						device_map="""auto"""							)
   _a  :       List[str]             =	self.tokenizer(self.input_text							,						return_tensors="""pt"""							).to(0							)
   _a  :       str             =	model.generate(**UpperCAmelCase__							)

   # test with `flan-t5-small`
   _a  :       List[str]             =	TaForConditionalGeneration.from_pretrained(
       self.dense_act_model_name							,						load_in_abit=UpperCAmelCase__							,						device_map="""auto"""							)
   _a  :       str             =	self.tokenizer(self.input_text							,						return_tensors="""pt"""							).to(0							)
   _a  :       int             =	model.generate(**UpperCAmelCase__							)
   _a  :       Dict             =	modules







 def _lowercase	(     self		:						Any							)					->       Optional[Any]:
   import bitsandbytes as bnb

   from transformers import TaForConditionalGeneration

   # test with `t5-small`
   _a  :       List[Any]             =	TaForConditionalGeneration.from_pretrained(self.model_name							,						load_in_abit=UpperCAmelCase__							,						device_map="""auto"""							)

   # there was a bug with decoders - this test checks that it is fixed
   self.assertTrue(isinstance(model.decoder.block[0].layer[0].SelfAttention.q							,						bnb.nn.Linearabit							)							)

   _a  :       str             =	self.tokenizer(self.input_text							,						return_tensors="""pt"""							).to(0							)
   _a  :       List[str]             =	model.generate(**UpperCAmelCase__							)

   # test with `flan-t5-small`
   _a  :       str             =	TaForConditionalGeneration.from_pretrained(
       self.dense_act_model_name							,						load_in_abit=UpperCAmelCase__							,						device_map="""auto"""							)
   _a  :       str             =	self.tokenizer(self.input_text							,						return_tensors="""pt"""							).to(0							)
   _a  :       Optional[int]             =	model.generate(**UpperCAmelCase__							)







class        UpperCamelCase			(	snake_case_      ):


 def _lowercase	(     self		:						Optional[int]							)					->       str:
   super().setUp()
   # model_name
   _a  :       str             =	"""bigscience/bloom-560m"""
   _a  :       str             =	"""t5-small"""

   # Different types of model

   _a  :       Optional[Any]             =	AutoModel.from_pretrained(self.model_name							,						load_in_abit=UpperCAmelCase__							,						device_map="""auto"""							)
   # Sequence classification model
   _a  :       List[str]             =	AutoModelForSequenceClassification.from_pretrained(
       self.model_name							,						load_in_abit=UpperCAmelCase__							,						device_map="""auto"""							)
   # CausalLM model
   _a  :       Optional[Any]             =	AutoModelForCausalLM.from_pretrained(self.model_name							,						load_in_abit=UpperCAmelCase__							,						device_map="""auto"""							)
   # Seq2seq model
   _a  :       Optional[int]             =	AutoModelForSeqaSeqLM.from_pretrained(
       self.seq_to_seq_name							,						load_in_abit=UpperCAmelCase__							,						device_map="""auto"""							)


 def _lowercase	(     self		:						Union[str, Any]							)					->       int:
   del self.base_model
   del self.sequence_model
   del self.model_abit
   del self.seq_to_seq_model

   gc.collect()
   torch.cuda.empty_cache()







 def _lowercase	(     self		:						int							)					->       Tuple:
   from bitsandbytes.nn import Paramsabit

   self.assertTrue(self.base_model.h[-1].mlp.dense_ah_to_h.weight.__class__ == Paramsabit							)

   # Other heads should be nn.Parameter
   self.assertTrue(self.model_abit.lm_head.weight.__class__ == torch.nn.Parameter							)
   self.assertTrue(self.sequence_model.score.weight.__class__ == torch.nn.Parameter							)
   self.assertTrue(self.seq_to_seq_model.lm_head.weight.__class__ == torch.nn.Parameter							)







class        UpperCamelCase			(	snake_case_      ):


 def _lowercase	(     self		:						Dict							)					->       Optional[int]:
   super().setUp()


 def _lowercase	(     self		:						List[Any]							)					->       Dict:
   del self.pipe

   gc.collect()
   torch.cuda.empty_cache()







 def _lowercase	(     self		:						int							)					->       List[str]:
   _a  :       Union[str, Any]             =	pipeline(
       """text-generation"""							,						model=self.model_name							,						model_kwargs={"""device_map""": """auto""", """load_in_4bit""": True, """torch_dtype""": torch.floataa}							,						max_new_tokens=self.MAX_NEW_TOKENS							,						)

   # Real second forward pass
   _a  :       Dict             =	self.pipe(self.input_text							)
   self.assertIn(pipeline_output[0]["""generated_text"""]							,						self.EXPECTED_OUTPUTS							)







@require_torch_multi_gpu
class        UpperCamelCase			(	snake_case_      ):


 def _lowercase	(     self		:						int							)					->       Tuple:
   super().setUp()







 def _lowercase	(     self		:						List[str]							)					->       Union[str, Any]:
   _a  :       List[Any]             =	AutoModelForCausalLM.from_pretrained(
       self.model_name							,						load_in_abit=UpperCAmelCase__							,						device_map="""balanced"""							)

   # Check correct device map
   self.assertEqual(set(model_parallel.hf_device_map.values()							)							,						{0, 1}							)

   # Check that inference pass works on the model
   _a  :       Union[str, Any]             =	self.tokenizer(self.input_text							,						return_tensors="""pt"""							)

   # Second real batch
   _a  :       List[str]             =	model_parallel.generate(input_ids=encoded_input["""input_ids"""].to(0							)							,						max_new_tokens=10							)
   self.assertIn(self.tokenizer.decode(output_parallel[0]							,						skip_special_tokens=UpperCAmelCase__							)							,						self.EXPECTED_OUTPUTS							)







class        UpperCamelCase			(	snake_case_      ):


 def _lowercase	(     self		:						Tuple							)					->       Tuple:
   _a  :       Any             =	"""facebook/opt-350m"""
   super().setUp()







 def _lowercase	(     self		:						Tuple							)					->       str:
   if version.parse(importlib.metadata.version("""bitsandbytes"""							)							) < version.parse("""0.37.0"""							):
     return

   # Step 1: freeze all parameters
   _a  :       Optional[Any]             =	AutoModelForCausalLM.from_pretrained(self.model_name							,						load_in_abit=UpperCAmelCase__							)

   self.assertEqual(set(model.hf_device_map.values()							)							,						{torch.cuda.current_device()}							)

   for param in model.parameters():
     _a  :       Optional[int]             =	False  # freeze the model - train adapters later
     if param.ndim == 1:
       # cast the small parameters (e.g. layernorm) to fp32 for stability
       _a  :       Tuple             =	param.data.to(torch.floataa							)

        # Step 2: add adapters
   for _, module in model.named_modules():
     if "OPTAttention" in repr(type(UpperCAmelCase__							)							):
       _a  :       Union[str, Any]             =	LoRALayer(module.q_proj							,						rank=16							)
       _a  :       List[str]             =	LoRALayer(module.k_proj							,						rank=16							)
       _a  :       Dict             =	LoRALayer(module.v_proj							,						rank=16							)

        # Step 3: dummy batch
   _a  :       Union[str, Any]             =	self.tokenizer("""Test batch """							,						return_tensors="""pt"""							).to(0							)

   # Step 4: Check if the gradient is not None
   with torch.cuda.amp.autocast():
     _a  :       Optional[Any]             =	model.forward(**UpperCAmelCase__							)
     out.logits.norm().backward()

   for module in model.modules():
     if isinstance(UpperCAmelCase__							,						UpperCAmelCase__							):
       self.assertTrue(module.adapter[1].weight.grad is not None							)
       self.assertTrue(module.adapter[1].weight.grad.norm().item() > 0							)
     elif isinstance(UpperCAmelCase__							,						nn.Embedding							):
       self.assertTrue(module.weight.grad is None							)







class        UpperCamelCase			(	snake_case_      ):
 UpperCamelCase					:       Optional[int]			=					'''gpt2-xl'''
 UpperCamelCase					:       Union[str, Any]			=					3.3_1_9_1_8_5_4_8_5_4_1_5_2_1_8_7









"""simple docstring"""




from __future__ import annotations

def 			lowerCAmelCase__     (     UpperCamelCase__     ,   UpperCamelCase__				):



  '''simple docstring'''
  print(F"""Vertex\tShortest Distance from vertex {src}"""				)
  for i, d in enumerate(UpperCamelCase__				):
    print(F"""{i}\t\t{d}"""				)

def 			lowerCAmelCase__     (     UpperCamelCase__     ,   UpperCamelCase__     ,   UpperCamelCase__				):



  '''simple docstring'''
  for j in range(UpperCamelCase__				):
    _a      ,  _a      ,  _a  :       List[str]             =	(graph[j][k] for k in ["""src""", """dst""", """weight"""])
    if distance[u] != float("""inf"""				) and distance[u] + w < distance[v]:
      return True
  return False

def 			lowerCAmelCase__     (     UpperCamelCase__     ,   UpperCamelCase__     ,   UpperCamelCase__     ,   UpperCamelCase__				):



  '''simple docstring'''
  _a  :       Dict             =	[float("""inf"""				)] * vertex_count
  _a  :       Any             =	0.0

  for _ in range(vertex_count - 1				):
    for j in range(UpperCamelCase__				):
      _a      ,  _a      ,  _a  :       List[Any]             =	(graph[j][k] for k in ["""src""", """dst""", """weight"""])

      if distance[u] != float("""inf"""				) and distance[u] + w < distance[v]:
        _a  :       Any             =	distance[u] + w

  _a  :       Union[str, Any]             =	check_negative_cycle(UpperCamelCase__     ,   UpperCamelCase__     ,   UpperCamelCase__				)
  if negative_cycle_exists:
    raise Exception("""Negative cycle found"""				)

  return distance


if __name__ == "__main__":
 import doctest

 doctest.testmod()

 _snake_case          =   int(input('Enter number of vertices: ').strip())
 _snake_case          =   int(input('Enter number of edges: ').strip())

 _snake_case          =   [{} for _ in range(E)]

 for i in range(E):
  print('Edge ', i + 1)
  _snake_case      , _snake_case      , _snake_case          =   (
      int(x)
      for x in input('Enter source, destination, weight: ').strip().split(' ')
  )
  _snake_case          =   {'src': src, 'dst': dest, 'weight': weight}

 _snake_case          =   int(input('\nEnter shortest path source:').strip())
 _snake_case          =   bellman_ford(graph, V, E, source)
 print_distance(shortest_distance, 0)

