





import logging
import re

import pytorch_quantization
import pytorch_quantization.nn as quant_nn
import torch
from pytorch_quantization import calib
from pytorch_quantization.tensor_quant import QuantDescriptor


snake_case_				=			logging.getLogger(__name__)

snake_case_				=			50  # max width of layer names
snake_case_				=			70  # max width of quantizer names






def lowerCamelCase__    (							snake_case_	:  List[str]       )      ->				Union[str, Any]:
   __snake_case							       =   parser.add_argument_group('''quant_trainer arguments'''       )
   group.add_argument('''--wprec'''       ,      type=lowerCAmelCase_       ,      default=8       ,      help='''weight precision'''       )
   group.add_argument('''--aprec'''       ,      type=lowerCAmelCase_       ,      default=8       ,      help='''activation precision'''       )
   group.add_argument('''--quant-per-tensor'''       ,      action='''store_true'''       ,      help='''per tensor weight scaling'''       )
   group.add_argument('''--quant-disable'''       ,      action='''store_true'''       ,      help='''disable all quantizers'''       )
   group.add_argument('''--quant-disable-embeddings'''       ,      action='''store_true'''       ,      help='''disable all embeddings quantizers'''       )
   group.add_argument('''--quant-disable-keyword'''       ,      type=lowerCAmelCase_       ,      nargs='''+'''       ,      help='''disable quantizers by keyword'''       )
   group.add_argument('''--quant-disable-layer-module'''       ,      type=lowerCAmelCase_       ,      help='''disable quantizers by keyword under layer.'''       )
   group.add_argument('''--quant-enable-layer-module'''       ,      type=lowerCAmelCase_       ,      help='''enable quantizers by keyword under layer'''       )
   group.add_argument('''--calibrator'''       ,      default='''max'''       ,      help='''which quantization range calibrator to use'''       )
   group.add_argument('''--percentile'''       ,      default=lowerCAmelCase_       ,      type=lowerCAmelCase_       ,      help='''percentile for PercentileCalibrator'''       )
   group.add_argument('''--fuse-qkv'''       ,      action='''store_true'''       ,      help='''use the same scale factor for qkv'''       )
   group.add_argument('''--clip-gelu'''       ,      metavar='''N'''       ,      type=lowerCAmelCase_       ,      help='''clip gelu output maximum value to N'''       )
   group.add_argument(
       '''--recalibrate-weights'''       ,      action='''store_true'''       ,      help=(
           '''recalibrate weight amaxes by taking the max of the weights.'''
           ''' amaxes will be computed with the current quantization granularity (axis).'''
       )       ,      )






def lowerCamelCase__    (							snake_case_	:  Union[str, Any]       )      ->				int:

   if args.calibrator == "max":
      __snake_case							       =   '''max'''
   elif args.calibrator == "percentile":
      if args.percentile is None:
         raise ValueError('''Specify --percentile when using percentile calibrator'''       )
      __snake_case							       =   '''histogram'''
   elif args.calibrator == "mse":
      __snake_case							       =   '''histogram'''
   else:
      raise ValueError(f"""Invalid calibrator {args.calibrator}"""       )

   __snake_case							       =   QuantDescriptor(num_bits=args.aprec       ,      calib_method=lowerCAmelCase_       )
   __snake_case							       =   QuantDescriptor(num_bits=args.wprec       ,      axis=(None if args.quant_per_tensor else (0,))       )
   quant_nn.QuantLinear.set_default_quant_desc_input(lowerCAmelCase_       )
   quant_nn.QuantLinear.set_default_quant_desc_weight(lowerCAmelCase_       )






def lowerCamelCase__    (							snake_case_	:  Any       ,      snake_case_	:  List[str]       ,      snake_case_	:  Union[str, Any]=False       ,      snake_case_	:  Union[str, Any]=False       )      ->				Any:
   logger.info('''Configuring Model for Quantization'''       )
   logger.info(f"""using quantization package {pytorch_quantization.__file__}"""       )

   if not calib:
      if args.quant_disable_embeddings:
         set_quantizer_by_name(lowerCAmelCase_       ,      ['''embeddings''']       ,      which='''weight'''       ,      _disabled=lowerCAmelCase_       )

      if args.quant_disable:
         set_quantizer_by_name(lowerCAmelCase_       ,      ['''''']       ,      _disabled=lowerCAmelCase_       )

      if args.quant_disable_keyword:
         set_quantizer_by_name(lowerCAmelCase_       ,      args.quant_disable_keyword       ,      _disabled=lowerCAmelCase_       )

      if args.quant_disable_layer_module:
         set_quantizer_by_name(lowerCAmelCase_       ,      [R'''layer.\d+.''' + args.quant_disable_layer_module]       ,      _disabled=lowerCAmelCase_       )

      if args.quant_enable_layer_module:
         set_quantizer_by_name(lowerCAmelCase_       ,      [R'''layer.\d+.''' + args.quant_enable_layer_module]       ,      _disabled=lowerCAmelCase_       )

      if args.recalibrate_weights:
         recalibrate_weights(lowerCAmelCase_       )

      if args.fuse_qkv:
         fuse_qkv(lowerCAmelCase_       ,      lowerCAmelCase_       )

   if args.clip_gelu:
      clip_gelu(lowerCAmelCase_       ,      args.clip_gelu       )

   # if args.local_rank in [-1, 0] and not calib:
   print_quant_summary(lowerCAmelCase_       )






def lowerCamelCase__    (							snake_case_	:  Union[str, Any]       )      ->				List[str]:
   logger.info('''Enabling Calibration'''       )
   for name, module in model.named_modules():
      if name.endswith('''_quantizer'''       ):
         if module._calibrator is not None:
            module.disable_quant()
            module.enable_calib()
         else:
            module.disable()
         logger.info(f"""{name:80}: {module}"""       )






def lowerCamelCase__    (							snake_case_	:  Optional[Any]       ,      snake_case_	:  Any       )      ->				Any:
   logger.info('''Loading calibrated amax'''       )
   for name, module in model.named_modules():
      if name.endswith('''_quantizer'''       ):
         if module._calibrator is not None:
            if isinstance(module._calibrator       ,      calib.MaxCalibrator       ):
               module.load_calib_amax()
            else:
               module.load_calib_amax('''percentile'''       ,      percentile=args.percentile       )
            module.enable_quant()
            module.disable_calib()
         else:
            module.enable()
   model.cuda()
   print_quant_summary(lowerCAmelCase_       )






def lowerCamelCase__    (							snake_case_	:  Optional[int]       ,      snake_case_	:  Dict       )      ->				Any:

   def fusea(snake_case_	:  Optional[Any]       ,      snake_case_	:  Dict       ,      snake_case_	:  str       ):
      for mod in [qq, qk, qv]:
         if not hasattr(lowerCAmelCase_       ,      '''_amax'''       ):
            print('''          WARNING: NO AMAX BUFFER'''       )
            return
      __snake_case							       =   qq._amax.detach().item()
      __snake_case							       =   qk._amax.detach().item()
      __snake_case							       =   qv._amax.detach().item()

      __snake_case							       =   max(lowerCAmelCase_       ,      lowerCAmelCase_       ,      lowerCAmelCase_       )
      qq._amax.fill_(lowerCAmelCase_       )
      qk._amax.fill_(lowerCAmelCase_       )
      qv._amax.fill_(lowerCAmelCase_       )
      logger.info(f"""          q={q:5.2f} k={k:5.2f} v={v:5.2f} -> {amax:5.2f}"""       )

   for name, mod in model.named_modules():
      if name.endswith('''.attention.self'''       ):
         logger.info(f"""FUSE_QKV: {name:{name_width}}"""       )
         fusea(mod.matmul_q_input_quantizer       ,      mod.matmul_k_input_quantizer       ,      mod.matmul_v_input_quantizer       )
         if args.quant_per_tensor:
            fusea(mod.query._weight_quantizer       ,      mod.key._weight_quantizer       ,      mod.value._weight_quantizer       )






def lowerCamelCase__    (							snake_case_	:  List[Any]       ,      snake_case_	:  List[str]       )      ->				str:

   for name, mod in model.named_modules():
      if name.endswith('''.output.dense'''       ) and not name.endswith('''attention.output.dense'''       ):
         __snake_case							       =   mod._input_quantizer._amax.data.detach().item()
         mod._input_quantizer._amax.data.detach().clamp_(max=lowerCAmelCase_       )
         __snake_case							       =   mod._input_quantizer._amax.data.detach().item()
         logger.info(f"""CLIP_GELU: {name:{name_width}} amax: {amax_init:5.2f} -> {amax:5.2f}"""       )






def lowerCamelCase__    (							snake_case_	:  Union[str, Any]       )      ->				Optional[Any]:

   for name, mod in model.named_modules():
      if hasattr(lowerCAmelCase_       ,      '''_weight_quantizer'''       ) and mod._weight_quantizer.axis is not None:
         __snake_case							       =   mod.weight.shape[0]
         __snake_case							       =   mod._weight_quantizer._amax.detach()
         __snake_case							       =   torch.ones(lowerCAmelCase_       ,      dtype=amax.dtype       ,      device=amax.device       ) * amax
         print(f"""expanding {name} {amax} -> {mod._weight_quantizer._amax}"""       )






def lowerCamelCase__    (							snake_case_	:  Union[str, Any]       )      ->				str:

   for name, mod in model.named_modules():
      if hasattr(lowerCAmelCase_       ,      '''_weight_quantizer'''       ):
         if not hasattr(mod.weight_quantizer       ,      '''_amax'''       ):
            print('''RECALIB: {name:{name_width}} WARNING: NO AMAX BUFFER'''       )
            continue

         # determine which axes to reduce across
         # e.g. a 4D tensor quantized per axis 0 should reduce over (1,2,3)
         __snake_case							       =   set() if mod._weight_quantizer.axis is None else set(mod._weight_quantizer.axis       )
         __snake_case							       =   set(range(len(mod.weight.size()       )       )       ) - axis_set
         __snake_case							       =   pytorch_quantization.utils.reduce_amax(mod.weight       ,      axis=lowerCAmelCase_       ,      keepdims=lowerCAmelCase_       ).detach()
         logger.info(f"""RECALIB: {name:{name_width}} {mod._weight_quantizer._amax.flatten()} -> {amax.flatten()}"""       )
         __snake_case							       =   amax






def lowerCamelCase__    (							snake_case_	:  Union[str, Any]       ,      snake_case_	:  str=25       ,      snake_case_	:  Tuple=180       ,      snake_case_	:  str=None       )      ->				Dict:

   if ignore is None:
      __snake_case							       =   []
   elif not isinstance(lowerCAmelCase_       ,      lowerCAmelCase_       ):
      __snake_case							       =   [ignore]

   __snake_case							       =   0
   for name, mod in model.named_modules():
      if not hasattr(lowerCAmelCase_       ,      '''weight'''       ):
         continue
      __snake_case							       =   max(lowerCAmelCase_       ,      len(lowerCAmelCase_       )       )

   for name, mod in model.named_modules():
      __snake_case							       =   getattr(lowerCAmelCase_       ,      '''_input_quantizer'''       ,      lowerCAmelCase_       )
      __snake_case							       =   getattr(lowerCAmelCase_       ,      '''_weight_quantizer'''       ,      lowerCAmelCase_       )
      if not hasattr(lowerCAmelCase_       ,      '''weight'''       ):
         continue
      if type(lowerCAmelCase_       ) in ignore:
         continue
      if [True for s in ignore if type(lowerCAmelCase_       ) is str and s in name]:
         continue
      __snake_case							       =   f"""Act:{input_q.extra_repr()}"""
      __snake_case							       =   f"""Wgt:{weight_q.extra_repr()}"""
      __snake_case							       =   f"""{name:{name_width}} {act_str} {wgt_str}"""
      if len(lowerCAmelCase_       ) <= line_width:
         logger.info(lowerCAmelCase_       )
      else:
         logger.info(f"""{name:{name_width}} {act_str}"""       )
         logger.info(f"""{'  ':{name_width}} {wgt_str}"""       )






def lowerCamelCase__    (							snake_case_	:  Any       )      ->				int:
   __snake_case							       =   0
   for name, mod in model.named_modules():
      if isinstance(lowerCAmelCase_       ,      pytorch_quantization.nn.TensorQuantizer       ):
         print(f"""{name:80} {mod}"""       )
         count += 1
   print(f"""{count} TensorQuantizers found in model"""       )






def lowerCamelCase__    (							snake_case_	:  int       ,      snake_case_	:  Any       ,      snake_case_	:  List[Any]       ,      snake_case_	:  Optional[Any]       ,      snake_case_	:  Dict       )      ->				Any:
   __snake_case							       =   getattr(lowerCAmelCase_       ,      lowerCAmelCase_       ,      lowerCAmelCase_       )
   if quantizer_mod is not None:
      assert hasattr(lowerCAmelCase_       ,      lowerCAmelCase_       )
      setattr(lowerCAmelCase_       ,      lowerCAmelCase_       ,      lowerCAmelCase_       )
   else:
      logger.warning(f"""{name} has no {quantizer}"""       )






def lowerCamelCase__    (							snake_case_	:  Dict       ,      snake_case_	:  Dict       ,      snake_case_	:  Optional[int]="both"       ,      **snake_case_	:  List[Any]       )      ->				List[Any]:
   __snake_case							       =   f"""Warning: changing {which} quantizers of {name:{qname_width}}"""
   for k, v in kwargs.items():
      s += f""" {k}={v}"""
      if which in ["input", "both"]:
         set_quantizer(lowerCAmelCase_       ,      lowerCAmelCase_       ,      '''_input_quantizer'''       ,      lowerCAmelCase_       ,      lowerCAmelCase_       )
      if which in ["weight", "both"]:
         set_quantizer(lowerCAmelCase_       ,      lowerCAmelCase_       ,      '''_weight_quantizer'''       ,      lowerCAmelCase_       ,      lowerCAmelCase_       )
   logger.info(lowerCAmelCase_       )







def lowerCamelCase__    (							snake_case_	:  int       ,      snake_case_	:  Optional[Any]       ,      **snake_case_	:  Dict       )      ->				Any:

   for name, mod in model.named_modules():
      if hasattr(lowerCAmelCase_       ,      '''_input_quantizer'''       ) or hasattr(lowerCAmelCase_       ,      '''_weight_quantizer'''       ):
         for n in names:
            if re.search(lowerCAmelCase_       ,      lowerCAmelCase_       ):
               set_quantizers(lowerCAmelCase_       ,      lowerCAmelCase_       ,      **lowerCAmelCase_       )
      elif name.endswith('''_quantizer'''       ):
         for n in names:
            if re.search(lowerCAmelCase_       ,      lowerCAmelCase_       ):
               __snake_case							       =   f"""Warning: changing {name:{name_width}}"""
               for k, v in kwargs.items():
                  s += f""" {k}={v}"""
                  setattr(lowerCAmelCase_       ,      lowerCAmelCase_       ,      lowerCAmelCase_       )
               logger.info(lowerCAmelCase_       )





def 	lowerCAmelCase       (						lowerCAmelCase_ = 10**9       )->       int:
				lowerCAmelCase_       :					List[Any]			 = 1
				lowerCAmelCase_       :					Optional[int]			 = 2

				lowerCAmelCase_       :					Any			 = 0
				lowerCAmelCase_       :					str			 = 0
				lowerCAmelCase_       :					str			 = 0
				while perimeter <= max_perimeter:
								perimeters_sum += perimeter

								prev_value += 2 * value
								value += prev_value

								lowerCAmelCase_       :					Any			 = 2 * value + 2 if i % 2 == 0 else 2 * value - 2
								i += 1

				return perimeters_sum


if __name__ == "__main__":
						print(f"""{solution() = }""")