

'''simple docstring'''






import logging
import re

import pytorch_quantization
import pytorch_quantization.nn as quant_nn
import torch
from pytorch_quantization import calib
from pytorch_quantization.tensor_quant import QuantDescriptor


lowerCAmelCase__											=	logging.getLogger(__name__)

lowerCAmelCase__											=	50  # max width of layer names
lowerCAmelCase__											=	70  # max width of quantizer names


def 		_A     (    A__      ):






 """simple docstring"""



 __lowercase							  = parser.add_argument_group('''quant_trainer arguments'''      )
 group.add_argument('''--wprec'''     ,				type=A__     ,				default=8     ,				help='''weight precision'''      )
 group.add_argument('''--aprec'''     ,				type=A__     ,				default=8     ,				help='''activation precision'''      )
 group.add_argument('''--quant-per-tensor'''     ,				action='''store_true'''     ,				help='''per tensor weight scaling'''      )
 group.add_argument('''--quant-disable'''     ,				action='''store_true'''     ,				help='''disable all quantizers'''      )
 group.add_argument('''--quant-disable-embeddings'''     ,				action='''store_true'''     ,				help='''disable all embeddings quantizers'''      )
 group.add_argument('''--quant-disable-keyword'''     ,				type=A__     ,				nargs='''+'''     ,				help='''disable quantizers by keyword'''      )
 group.add_argument('''--quant-disable-layer-module'''     ,				type=A__     ,				help='''disable quantizers by keyword under layer.'''      )
 group.add_argument('''--quant-enable-layer-module'''     ,				type=A__     ,				help='''enable quantizers by keyword under layer'''      )
 group.add_argument('''--calibrator'''     ,				default='''max'''     ,				help='''which quantization range calibrator to use'''      )
 group.add_argument('''--percentile'''     ,				default=A__     ,				type=A__     ,				help='''percentile for PercentileCalibrator'''      )
 group.add_argument('''--fuse-qkv'''     ,				action='''store_true'''     ,				help='''use the same scale factor for qkv'''      )
 group.add_argument('''--clip-gelu'''     ,				metavar='''N'''     ,				type=A__     ,				help='''clip gelu output maximum value to N'''      )
 group.add_argument(
     '''--recalibrate-weights'''     ,				action='''store_true'''     ,				help=(
         '''recalibrate weight amaxes by taking the max of the weights.'''
         ''' amaxes will be computed with the current quantization granularity (axis).'''
     )     ,				)


def 		_A     (    A__      ):






 """simple docstring"""



 if args.calibrator == "max":
  __lowercase							  = '''max'''
 elif args.calibrator == "percentile":
  if args.percentile is None:
   raise ValueError('''Specify --percentile when using percentile calibrator'''      )
  __lowercase							  = '''histogram'''
 elif args.calibrator == "mse":
  __lowercase							  = '''histogram'''
 else:
  raise ValueError(F"Invalid calibrator {args.calibrator}"      )

 __lowercase							  = QuantDescriptor(num_bits=args.aprec     ,				calib_method=A__      )
 __lowercase							  = QuantDescriptor(num_bits=args.wprec     ,				axis=(None if args.quant_per_tensor else (0,))      )
 quant_nn.QuantLinear.set_default_quant_desc_input(A__      )
 quant_nn.QuantLinear.set_default_quant_desc_weight(A__      )


def 		_A     (    A__     ,				A__     ,				A__=False     ,				A__=False      ):






 """simple docstring"""



 logger.info('''Configuring Model for Quantization'''      )
 logger.info(F"using quantization package {pytorch_quantization.__file__}"      )

 if not calib:
  if args.quant_disable_embeddings:
   set_quantizer_by_name(A__     ,				['''embeddings''']     ,				which='''weight'''     ,				_disabled=A__      )

  if args.quant_disable:
   set_quantizer_by_name(A__     ,				['''''']     ,				_disabled=A__      )

  if args.quant_disable_keyword:
   set_quantizer_by_name(A__     ,				args.quant_disable_keyword     ,				_disabled=A__      )

  if args.quant_disable_layer_module:
   set_quantizer_by_name(A__     ,				[R'''layer.\d+.''' + args.quant_disable_layer_module]     ,				_disabled=A__      )

  if args.quant_enable_layer_module:
   set_quantizer_by_name(A__     ,				[R'''layer.\d+.''' + args.quant_enable_layer_module]     ,				_disabled=A__      )

  if args.recalibrate_weights:
   recalibrate_weights(A__      )

  if args.fuse_qkv:
   fuse_qkv(A__     ,				A__      )

 if args.clip_gelu:
  clip_gelu(A__     ,				args.clip_gelu      )

 # if args.local_rank in [-1, 0] and not calib:
 print_quant_summary(A__      )


def 		_A     (    A__      ):






 """simple docstring"""



 logger.info('''Enabling Calibration'''      )
 for name, module in model.named_modules():
  if name.endswith('''_quantizer'''      ):
   if module._calibrator is not None:
    module.disable_quant()
    module.enable_calib()
   else:
    module.disable()
   logger.info(F"{name:80}: {module}"      )


def 		_A     (    A__     ,				A__      ):






 """simple docstring"""



 logger.info('''Loading calibrated amax'''      )
 for name, module in model.named_modules():
  if name.endswith('''_quantizer'''      ):
   if module._calibrator is not None:
    if isinstance(module._calibrator     ,				calib.MaxCalibrator      ):
     module.load_calib_amax()
    else:
     module.load_calib_amax('''percentile'''     ,				percentile=args.percentile      )
    module.enable_quant()
    module.disable_calib()
   else:
    module.enable()
 model.cuda()
 print_quant_summary(A__      )


def 		_A     (    A__     ,				A__      ):






 """simple docstring"""



 def fusea(A__     ,				A__     ,				A__      ):
  for mod in [qq, qk, qv]:
   if not hasattr(A__     ,				'''_amax'''      ):
    print('''          WARNING: NO AMAX BUFFER'''      )
    return
  __lowercase							  = qq._amax.detach().item()
  __lowercase							  = qk._amax.detach().item()
  __lowercase							  = qv._amax.detach().item()

  __lowercase							  = max(A__     ,				A__     ,				A__      )
  qq._amax.fill_(A__      )
  qk._amax.fill_(A__      )
  qv._amax.fill_(A__      )
  logger.info(F"          q={q:5.2f} k={k:5.2f} v={v:5.2f} -> {amax:5.2f}"      )

 for name, mod in model.named_modules():
  if name.endswith('''.attention.self'''      ):
   logger.info(F"FUSE_QKV: {name:{name_width}}"      )
   fusea(mod.matmul_q_input_quantizer     ,				mod.matmul_k_input_quantizer     ,				mod.matmul_v_input_quantizer      )
   if args.quant_per_tensor:
    fusea(mod.query._weight_quantizer     ,				mod.key._weight_quantizer     ,				mod.value._weight_quantizer      )


def 		_A     (    A__     ,				A__      ):






 """simple docstring"""



 for name, mod in model.named_modules():
  if name.endswith('''.output.dense'''      ) and not name.endswith('''attention.output.dense'''      ):
   __lowercase							  = mod._input_quantizer._amax.data.detach().item()
   mod._input_quantizer._amax.data.detach().clamp_(max=A__      )
   __lowercase							  = mod._input_quantizer._amax.data.detach().item()
   logger.info(F"CLIP_GELU: {name:{name_width}} amax: {amax_init:5.2f} -> {amax:5.2f}"      )


def 		_A     (    A__      ):






 """simple docstring"""



 for name, mod in model.named_modules():
  if hasattr(A__     ,				'''_weight_quantizer'''      ) and mod._weight_quantizer.axis is not None:
   __lowercase							  = mod.weight.shape[0]
   __lowercase							  = mod._weight_quantizer._amax.detach()
   __lowercase							  = torch.ones(A__     ,				dtype=amax.dtype     ,				device=amax.device      ) * amax
   print(F"expanding {name} {amax} -> {mod._weight_quantizer._amax}"      )


def 		_A     (    A__      ):






 """simple docstring"""



 for name, mod in model.named_modules():
  if hasattr(A__     ,				'''_weight_quantizer'''      ):
   if not hasattr(mod.weight_quantizer     ,				'''_amax'''      ):
    print('''RECALIB: {name:{name_width}} WARNING: NO AMAX BUFFER'''      )
    continue

   # determine which axes to reduce across
   # e.g. a 4D tensor quantized per axis 0 should reduce over (1,2,3)
   __lowercase							  = set() if mod._weight_quantizer.axis is None else set(mod._weight_quantizer.axis      )
   __lowercase							  = set(range(len(mod.weight.size()      )      )      ) - axis_set
   __lowercase							  = pytorch_quantization.utils.reduce_amax(mod.weight     ,				axis=A__     ,				keepdims=A__      ).detach()
   logger.info(F"RECALIB: {name:{name_width}} {mod._weight_quantizer._amax.flatten()} -> {amax.flatten()}"      )
   __lowercase							  = amax


def 		_A     (    A__     ,				A__=25     ,				A__=180     ,				A__=None      ):






 """simple docstring"""



 if ignore is None:
  __lowercase							  = []
 elif not isinstance(A__     ,				A__      ):
  __lowercase							  = [ignore]

 __lowercase							  = 0
 for name, mod in model.named_modules():
  if not hasattr(A__     ,				'''weight'''      ):
   continue
  __lowercase							  = max(A__     ,				len(A__      )      )

 for name, mod in model.named_modules():
  __lowercase							  = getattr(A__     ,				'''_input_quantizer'''     ,				A__      )
  __lowercase							  = getattr(A__     ,				'''_weight_quantizer'''     ,				A__      )
  if not hasattr(A__     ,				'''weight'''      ):
   continue
  if type(A__      ) in ignore:
   continue
  if [True for s in ignore if type(A__      ) is str and s in name]:
   continue
  __lowercase							  = F"Act:{input_q.extra_repr()}"
  __lowercase							  = F"Wgt:{weight_q.extra_repr()}"
  __lowercase							  = F"{name:{name_width}} {act_str} {wgt_str}"
  if len(A__      ) <= line_width:
   logger.info(A__      )
  else:
   logger.info(F"{name:{name_width}} {act_str}"      )
   logger.info(F"{'  ':{name_width}} {wgt_str}"      )


def 		_A     (    A__      ):






 """simple docstring"""



 __lowercase							  = 0
 for name, mod in model.named_modules():
  if isinstance(A__     ,				pytorch_quantization.nn.TensorQuantizer      ):
   print(F"{name:80} {mod}"      )
   count += 1
 print(F"{count} TensorQuantizers found in model"      )


def 		_A     (    A__     ,				A__     ,				A__     ,				A__     ,				A__      ):






 """simple docstring"""



 __lowercase							  = getattr(A__     ,				A__     ,				A__      )
 if quantizer_mod is not None:
  assert hasattr(A__     ,				A__      )
  setattr(A__     ,				A__     ,				A__      )
 else:
  logger.warning(F"{name} has no {quantizer}"      )


def 		_A     (    A__     ,				A__     ,				A__="both"     ,				**A__      ):






 """simple docstring"""



 __lowercase							  = F"Warning: changing {which} quantizers of {name:{qname_width}}"
 for k, v in kwargs.items():
  s += F" {k}={v}"
  if which in ["input", "both"]:
   set_quantizer(A__     ,				A__     ,				'''_input_quantizer'''     ,				A__     ,				A__      )
  if which in ["weight", "both"]:
   set_quantizer(A__     ,				A__     ,				'''_weight_quantizer'''     ,				A__     ,				A__      )
 logger.info(A__      )






def 		_A     (    A__     ,				A__     ,				**A__      ):






 """simple docstring"""



 for name, mod in model.named_modules():
  if hasattr(A__     ,				'''_input_quantizer'''      ) or hasattr(A__     ,				'''_weight_quantizer'''      ):
   for n in names:
    if re.search(A__     ,				A__      ):
     set_quantizers(A__     ,				A__     ,				**A__      )
  elif name.endswith('''_quantizer'''      ):
   for n in names:
    if re.search(A__     ,				A__      ):
     __lowercase							  = F"Warning: changing {name:{name_width}}"
     for k, v in kwargs.items():
      s += F" {k}={v}"
      setattr(A__     ,				A__     ,				A__      )
     logger.info(A__      )




'''simple docstring'''






def 		_A     (    A__ = 10     ,				A__ = 22      ):






 """simple docstring"""



 __lowercase							  = range(1     ,				A__      )
 __lowercase							  = range(1     ,				A__      )
 return sum(
     1 for power in powers for base in bases if len(str(base**power      )      ) == power      )


if __name__ == "__main__":
 print(f'{solution(10, 22) = }')

