




"""simple docstring"""






import logging
import re

import pytorch_quantization
import pytorch_quantization.nn as quant_nn
import torch
from pytorch_quantization import calib
from pytorch_quantization.tensor_quant import QuantDescriptor


lowercase__     						=						logging.getLogger(__name__)

lowercase__     						=						50  # max width of layer names
lowercase__     						=						70  # max width of quantizer names







def      _snake_case			(				lowercase__		):
				_lowerCamelCase     :			Dict    =    parser.add_argument_group('quant_trainer arguments'		)
				group.add_argument('--wprec'							,    type=lowercase__							,    default=8							,    help='weight precision'		)
				group.add_argument('--aprec'							,    type=lowercase__							,    default=8							,    help='activation precision'		)
				group.add_argument('--quant-per-tensor'							,    action='store_true'							,    help='per tensor weight scaling'		)
				group.add_argument('--quant-disable'							,    action='store_true'							,    help='disable all quantizers'		)
				group.add_argument('--quant-disable-embeddings'							,    action='store_true'							,    help='disable all embeddings quantizers'		)
				group.add_argument('--quant-disable-keyword'							,    type=lowercase__							,    nargs='+'							,    help='disable quantizers by keyword'		)
				group.add_argument('--quant-disable-layer-module'							,    type=lowercase__							,    help='disable quantizers by keyword under layer.'		)
				group.add_argument('--quant-enable-layer-module'							,    type=lowercase__							,    help='enable quantizers by keyword under layer'		)
				group.add_argument('--calibrator'							,    default='max'							,    help='which quantization range calibrator to use'		)
				group.add_argument('--percentile'							,    default=lowercase__							,    type=lowercase__							,    help='percentile for PercentileCalibrator'		)
				group.add_argument('--fuse-qkv'							,    action='store_true'							,    help='use the same scale factor for qkv'		)
				group.add_argument('--clip-gelu'							,    metavar='N'							,    type=lowercase__							,    help='clip gelu output maximum value to N'		)
				group.add_argument(
				    '--recalibrate-weights'							,    action='store_true'							,    help=(
				        'recalibrate weight amaxes by taking the max of the weights.'
				        ' amaxes will be computed with the current quantization granularity (axis).'
				    )							,    )







def      _snake_case			(				lowercase__		):

				if args.calibrator == "max":
								_lowerCamelCase     :			Union[str, Any]    =    'max'
				elif args.calibrator == "percentile":
								if args.percentile is None:
												raise ValueError('Specify --percentile when using percentile calibrator'		)
								_lowerCamelCase     :			Tuple    =    'histogram'
				elif args.calibrator == "mse":
								_lowerCamelCase     :			Optional[Any]    =    'histogram'
				else:
								raise ValueError(f'''Invalid calibrator {args.calibrator}'''		)

				_lowerCamelCase     :			List[Any]    =    QuantDescriptor(num_bits=args.aprec							,    calib_method=lowercase__		)
				_lowerCamelCase     :			int    =    QuantDescriptor(num_bits=args.wprec							,    axis=(None if args.quant_per_tensor else (0,))		)
				quant_nn.QuantLinear.set_default_quant_desc_input(lowercase__		)
				quant_nn.QuantLinear.set_default_quant_desc_weight(lowercase__		)







def      _snake_case			(				lowercase__							,    lowercase__							,    lowercase__=False							,    lowercase__=False		):
				logger.info('Configuring Model for Quantization'		)
				logger.info(f'''using quantization package {pytorch_quantization.__file__}'''		)

				if not calib:
								if args.quant_disable_embeddings:
												set_quantizer_by_name(lowercase__							,    ['embeddings']							,    which='weight'							,    _disabled=lowercase__		)

								if args.quant_disable:
												set_quantizer_by_name(lowercase__							,    ['']							,    _disabled=lowercase__		)

								if args.quant_disable_keyword:
												set_quantizer_by_name(lowercase__							,    args.quant_disable_keyword							,    _disabled=lowercase__		)

								if args.quant_disable_layer_module:
												set_quantizer_by_name(lowercase__							,    [r'layer.\d+.' + args.quant_disable_layer_module]							,    _disabled=lowercase__		)

								if args.quant_enable_layer_module:
												set_quantizer_by_name(lowercase__							,    [r'layer.\d+.' + args.quant_enable_layer_module]							,    _disabled=lowercase__		)

								if args.recalibrate_weights:
												recalibrate_weights(lowercase__		)

								if args.fuse_qkv:
												fuse_qkv(lowercase__							,    lowercase__		)

				if args.clip_gelu:
								clip_gelu(lowercase__							,    args.clip_gelu		)

				# if args.local_rank in [-1, 0] and not calib:
				print_quant_summary(lowercase__		)







def      _snake_case			(				lowercase__		):
				logger.info('Enabling Calibration'		)
				for name, module in model.named_modules():
								if name.endswith('_quantizer'		):
												if module._calibrator is not None:
																module.disable_quant()
																module.enable_calib()
												else:
																module.disable()
												logger.info(f'''{name:80}: {module}'''		)







def      _snake_case			(				lowercase__							,    lowercase__		):
				logger.info('Loading calibrated amax'		)
				for name, module in model.named_modules():
								if name.endswith('_quantizer'		):
												if module._calibrator is not None:
																if isinstance(module._calibrator							,    calib.MaxCalibrator		):
																				module.load_calib_amax()
																else:
																				module.load_calib_amax('percentile'							,    percentile=args.percentile		)
																module.enable_quant()
																module.disable_calib()
												else:
																module.enable()
				model.cuda()
				print_quant_summary(lowercase__		)







def      _snake_case			(				lowercase__							,    lowercase__		):

				def fusea(lowercase__							,    lowercase__							,    lowercase__		):
								for mod in [qq, qk, qv]:
												if not hasattr(lowercase__							,    '_amax'		):
																print('          WARNING: NO AMAX BUFFER'		)
																return
								_lowerCamelCase     :			Dict    =    qq._amax.detach().item()
								_lowerCamelCase     :			Union[str, Any]    =    qk._amax.detach().item()
								_lowerCamelCase     :			int    =    qv._amax.detach().item()

								_lowerCamelCase     :			Optional[int]    =    max(lowercase__							,    lowercase__							,    lowercase__		)
								qq._amax.fill_(lowercase__		)
								qk._amax.fill_(lowercase__		)
								qv._amax.fill_(lowercase__		)
								logger.info(f'''          q={q:5.2f} k={k:5.2f} v={v:5.2f} -> {amax:5.2f}'''		)

				for name, mod in model.named_modules():
								if name.endswith('.attention.self'		):
												logger.info(f'''FUSE_QKV: {name:{name_width}}'''		)
												fusea(mod.matmul_q_input_quantizer							,    mod.matmul_k_input_quantizer							,    mod.matmul_v_input_quantizer		)
												if args.quant_per_tensor:
																fusea(mod.query._weight_quantizer							,    mod.key._weight_quantizer							,    mod.value._weight_quantizer		)







def      _snake_case			(				lowercase__							,    lowercase__		):

				for name, mod in model.named_modules():
								if name.endswith('.output.dense'		) and not name.endswith('attention.output.dense'		):
												_lowerCamelCase     :			Optional[int]    =    mod._input_quantizer._amax.data.detach().item()
												mod._input_quantizer._amax.data.detach().clamp_(max=lowercase__		)
												_lowerCamelCase     :			List[Any]    =    mod._input_quantizer._amax.data.detach().item()
												logger.info(f'''CLIP_GELU: {name:{name_width}} amax: {amax_init:5.2f} -> {amax:5.2f}'''		)







def      _snake_case			(				lowercase__		):

				for name, mod in model.named_modules():
								if hasattr(lowercase__							,    '_weight_quantizer'		) and mod._weight_quantizer.axis is not None:
												_lowerCamelCase     :			Union[str, Any]    =    mod.weight.shape[0]
												_lowerCamelCase     :			int    =    mod._weight_quantizer._amax.detach()
												_lowerCamelCase     :			str    =    torch.ones(lowercase__							,    dtype=amax.dtype							,    device=amax.device		) * amax
												print(f'''expanding {name} {amax} -> {mod._weight_quantizer._amax}'''		)







def      _snake_case			(				lowercase__		):

				for name, mod in model.named_modules():
								if hasattr(lowercase__							,    '_weight_quantizer'		):
												if not hasattr(mod.weight_quantizer							,    '_amax'		):
																print('RECALIB: {name:{name_width}} WARNING: NO AMAX BUFFER'		)
																continue

												# determine which axes to reduce across
												# e.g. a 4D tensor quantized per axis 0 should reduce over (1,2,3)
												_lowerCamelCase     :			List[str]    =    set() if mod._weight_quantizer.axis is None else set(mod._weight_quantizer.axis		)
												_lowerCamelCase     :			str    =    set(range(len(mod.weight.size()		)		)		) - axis_set
												_lowerCamelCase     :			int    =    pytorch_quantization.utils.reduce_amax(mod.weight							,    axis=lowercase__							,    keepdims=lowercase__		).detach()
												logger.info(f'''RECALIB: {name:{name_width}} {mod._weight_quantizer._amax.flatten()} -> {amax.flatten()}'''		)
												_lowerCamelCase     :			Any    =    amax







def      _snake_case			(				lowercase__							,    lowercase__=25							,    lowercase__=180							,    lowercase__=None		):

				if ignore is None:
								_lowerCamelCase     :			str    =    []
				elif not isinstance(lowercase__							,    lowercase__		):
								_lowerCamelCase     :			int    =    [ignore]

				_lowerCamelCase     :			Any    =    0
				for name, mod in model.named_modules():
								if not hasattr(lowercase__							,    'weight'		):
												continue
								_lowerCamelCase     :			Optional[Any]    =    max(lowercase__							,    len(lowercase__		)		)

				for name, mod in model.named_modules():
								_lowerCamelCase     :			Any    =    getattr(lowercase__							,    '_input_quantizer'							,    lowercase__		)
								_lowerCamelCase     :			Dict    =    getattr(lowercase__							,    '_weight_quantizer'							,    lowercase__		)
								if not hasattr(lowercase__							,    'weight'		):
												continue
								if type(lowercase__		) in ignore:
												continue
								if [True for s in ignore if type(lowercase__		) is str and s in name]:
												continue
								_lowerCamelCase     :			List[str]    =    f'''Act:{input_q.extra_repr()}'''
								_lowerCamelCase     :			Dict    =    f'''Wgt:{weight_q.extra_repr()}'''
								_lowerCamelCase     :			str    =    f'''{name:{name_width}} {act_str} {wgt_str}'''
								if len(lowercase__		) <= line_width:
												logger.info(lowercase__		)
								else:
												logger.info(f'''{name:{name_width}} {act_str}'''		)
												logger.info(f'''{'  ':{name_width}} {wgt_str}'''		)







def      _snake_case			(				lowercase__		):
				_lowerCamelCase     :			Dict    =    0
				for name, mod in model.named_modules():
								if isinstance(lowercase__							,    pytorch_quantization.nn.TensorQuantizer		):
												print(f'''{name:80} {mod}'''		)
												count += 1
				print(f'''{count} TensorQuantizers found in model'''		)







def      _snake_case			(				lowercase__							,    lowercase__							,    lowercase__							,    lowercase__							,    lowercase__		):
				_lowerCamelCase     :			Optional[int]    =    getattr(lowercase__							,    lowercase__							,    lowercase__		)
				if quantizer_mod is not None:
								assert hasattr(lowercase__							,    lowercase__		)
								setattr(lowercase__							,    lowercase__							,    lowercase__		)
				else:
								logger.warning(f'''{name} has no {quantizer}'''		)







def      _snake_case			(				lowercase__							,    lowercase__							,    lowercase__="both"							,    **lowercase__		):
				_lowerCamelCase     :			str    =    f'''Warning: changing {which} quantizers of {name:{qname_width}}'''
				for k, v in kwargs.items():
								s += f''' {k}={v}'''
								if which in ["input", "both"]:
												set_quantizer(lowercase__							,    lowercase__							,    '_input_quantizer'							,    lowercase__							,    lowercase__		)
								if which in ["weight", "both"]:
												set_quantizer(lowercase__							,    lowercase__							,    '_weight_quantizer'							,    lowercase__							,    lowercase__		)
				logger.info(lowercase__		)







def      _snake_case			(				lowercase__							,    lowercase__							,    **lowercase__		):

				for name, mod in model.named_modules():
								if hasattr(lowercase__							,    '_input_quantizer'		) or hasattr(lowercase__							,    '_weight_quantizer'		):
												for n in names:
																if re.search(lowercase__							,    lowercase__		):
																				set_quantizers(lowercase__							,    lowercase__							,    **lowercase__		)
								elif name.endswith('_quantizer'		):
												for n in names:
																if re.search(lowercase__							,    lowercase__		):
																				_lowerCamelCase     :			Optional[int]    =    f'''Warning: changing {name:{name_width}}'''
																				for k, v in kwargs.items():
																								s += f''' {k}={v}'''
																								setattr(lowercase__							,    lowercase__							,    lowercase__		)
																				logger.info(lowercase__		)





"""simple docstring"""






import argparse
import json
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType
from accelerate.utils.deepspeed import DummyOptim, DummyScheduler


lowercase__     						=						16
lowercase__     						=						32







def      _snake_case			(				lowercase__							,    lowercase__ = 16							,    lowercase__ = "bert-base-cased"		):
				_lowerCamelCase     :			List[Any]    =    AutoTokenizer.from_pretrained(lowercase__		)
				_lowerCamelCase     :			Tuple    =    load_dataset('glue'							,    'mrpc'		)

				def tokenize_function(lowercase__		):
								# max_length=None => use the model max length (it's actually the default)
								_lowerCamelCase     :			Union[str, Any]    =    tokenizer(examples['sentence1']							,    examples['sentence2']							,    truncation=lowercase__							,    max_length=lowercase__		)
								return outputs

				# Apply the method we just defined to all the examples in all the splits of the dataset
				_lowerCamelCase     :			int    =    datasets.map(
				    lowercase__							,    batched=lowercase__							,    remove_columns=['idx', 'sentence1', 'sentence2']							,    load_from_cache_file=lowercase__		)

				# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
				# transformers library
				_lowerCamelCase     :			Optional[int]    =    tokenized_datasets.rename_column('label'							,    'labels'		)

				def collate_fn(lowercase__		):
								# On TPU it's best to pad everything to the same length or training will be very slow.
								if accelerator.distributed_type == DistributedType.TPU:
												return tokenizer.pad(lowercase__							,    padding='max_length'							,    max_length=128							,    return_tensors='pt'		)
								return tokenizer.pad(lowercase__							,    padding='longest'							,    return_tensors='pt'		)

				# Instantiate dataloaders.
				_lowerCamelCase     :			List[str]    =    DataLoader(
				    tokenized_datasets['train']							,    shuffle=lowercase__							,    collate_fn=lowercase__							,    batch_size=lowercase__		)
				_lowerCamelCase     :			int    =    DataLoader(
				    tokenized_datasets['validation']							,    shuffle=lowercase__							,    collate_fn=lowercase__							,    batch_size=lowercase__		)

				return train_dataloader, eval_dataloader







def      _snake_case			(				lowercase__							,    lowercase__		):
				# Initialize accelerator
				_lowerCamelCase     :			Optional[int]    =    Accelerator()

				# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
				_lowerCamelCase     :			Optional[int]    =    config['lr']
				_lowerCamelCase     :			Optional[int]    =    int(config['num_epochs']		)
				_lowerCamelCase     :			Union[str, Any]    =    int(config['seed']		)
				_lowerCamelCase     :			Optional[int]    =    int(config['batch_size']		)
				_lowerCamelCase     :			Dict    =    args.model_name_or_path

				set_seed(lowercase__		)
				_lowerCamelCase, _lowerCamelCase     :			Optional[int]    =    get_dataloaders(lowercase__							,    lowercase__							,    lowercase__		)

				# Instantiate the model (we build the model here so that the seed also control new weights initialization)
				_lowerCamelCase     :			int    =    AutoModelForSequenceClassification.from_pretrained(lowercase__							,    return_dict=lowercase__		)

				# Instantiate optimizer
				_lowerCamelCase     :			Optional[int]    =    (
				    AdamW
				    if accelerator.state.deepspeed_plugin is None
				    or 'optimizer' not in accelerator.state.deepspeed_plugin.deepspeed_config
				    else DummyOptim
				)
				_lowerCamelCase     :			Union[str, Any]    =    optimizer_cls(params=model.parameters()							,    lr=lowercase__		)

				if accelerator.state.deepspeed_plugin is not None:
								_lowerCamelCase     :			str    =    accelerator.state.deepspeed_plugin.deepspeed_config[
								    'gradient_accumulation_steps'
								]
				else:
								_lowerCamelCase     :			Tuple    =    1
				_lowerCamelCase     :			List[Any]    =    (len(lowercase__		) * num_epochs) // gradient_accumulation_steps

				# Instantiate scheduler
				if (
				    accelerator.state.deepspeed_plugin is None
				    or "scheduler" not in accelerator.state.deepspeed_plugin.deepspeed_config
				):
								_lowerCamelCase     :			Tuple    =    get_linear_schedule_with_warmup(
								    optimizer=lowercase__							,    num_warmup_steps=0							,    num_training_steps=lowercase__							,    )
				else:
								_lowerCamelCase     :			Any    =    DummyScheduler(lowercase__							,    total_num_steps=lowercase__							,    warmup_num_steps=0		)

				# Prepare everything
				# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
				# prepare method.
				_lowerCamelCase, _lowerCamelCase, _lowerCamelCase, _lowerCamelCase, _lowerCamelCase     :			Union[str, Any]    =    accelerator.prepare(
				    lowercase__							,    lowercase__							,    lowercase__							,    lowercase__							,    lowercase__		)

				# We need to keep track of how many total steps we have iterated over
				_lowerCamelCase     :			Union[str, Any]    =    0
				# We also need to keep track of the stating epoch so files are named properly
				_lowerCamelCase     :			Dict    =    0

				# Now we train the model
				_lowerCamelCase     :			Dict    =    evaluate.load('glue'							,    'mrpc'		)
				_lowerCamelCase     :			Optional[int]    =    0
				_lowerCamelCase     :			str    =    {}
				for epoch in range(lowercase__							,    lowercase__		):
								model.train()
								for step, batch in enumerate(lowercase__		):
												_lowerCamelCase     :			List[Any]    =    model(**lowercase__		)
												_lowerCamelCase     :			int    =    outputs.loss
												_lowerCamelCase     :			Dict    =    loss / gradient_accumulation_steps
												accelerator.backward(lowercase__		)
												if step % gradient_accumulation_steps == 0:
																optimizer.step()
																lr_scheduler.step()
																optimizer.zero_grad()

												overall_step += 1

								model.eval()
								_lowerCamelCase     :			Union[str, Any]    =    0
								for step, batch in enumerate(lowercase__		):
												# We could avoid this line since we set the accelerator with `device_placement=True`.
												batch.to(accelerator.device		)
												with torch.no_grad():
																_lowerCamelCase     :			Optional[int]    =    model(**lowercase__		)
												_lowerCamelCase     :			Dict    =    outputs.logits.argmax(dim=-1		)
												# It is slightly faster to call this once, than multiple times
												_lowerCamelCase, _lowerCamelCase     :			List[str]    =    accelerator.gather(
												    (predictions, batch['labels'])		)  # If we are in a multiprocess environment, the last batch has duplicates
												if accelerator.use_distributed:
																if step == len(lowercase__		) - 1:
																				_lowerCamelCase     :			Optional[Any]    =    predictions[: len(eval_dataloader.dataset		) - samples_seen]
																				_lowerCamelCase     :			Dict    =    references[: len(eval_dataloader.dataset		) - samples_seen]
																else:
																				samples_seen += references.shape[0]
												metric.add_batch(
												    predictions=lowercase__							,    references=lowercase__							,    )

								_lowerCamelCase     :			List[Any]    =    metric.compute()
								# Use accelerator.print to print only on the main process.
								accelerator.print(f'''epoch {epoch}:'''							,    lowercase__		)
								_lowerCamelCase     :			Tuple    =    eval_metric['accuracy']

								if best_performance < eval_metric["accuracy"]:
												_lowerCamelCase     :			str    =    eval_metric['accuracy']

				if args.performance_lower_bound is not None:
								assert (
								    args.performance_lower_bound <= best_performance
								), f'''Best performance metric {best_performance} is lower than the lower bound {args.performance_lower_bound}'''

				accelerator.wait_for_everyone()
				if accelerator.is_main_process:
								with open(os.path.join(args.output_dir							,    'all_results.json'		)							,    'w'		) as f:
												json.dump(lowercase__							,    lowercase__		)







def      _snake_case			(				):
				_lowerCamelCase     :			Any    =    argparse.ArgumentParser(description='Simple example of training script tracking peak GPU memory usage.'		)
				parser.add_argument(
				    '--model_name_or_path'							,    type=lowercase__							,    default='bert-base-cased'							,    help='Path to pretrained model or model identifier from huggingface.co/models.'							,    required=lowercase__							,    )
				parser.add_argument(
				    '--output_dir'							,    type=lowercase__							,    default='.'							,    help='Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'							,    )
				parser.add_argument(
				    '--performance_lower_bound'							,    type=lowercase__							,    default=lowercase__							,    help='Optional lower bound for the performance metric. If set, the training will throw error when the performance metric drops below this value.'							,    )
				parser.add_argument(
				    '--num_epochs'							,    type=lowercase__							,    default=3							,    help='Number of train epochs.'							,    )
				_lowerCamelCase     :			Optional[Any]    =    parser.parse_args()
				_lowerCamelCase     :			str    =    {'lr': 2E-5, 'num_epochs': args.num_epochs, 'seed': 42, 'batch_size': 16}
				training_function(lowercase__							,    lowercase__		)


if __name__ == "__main__":
			main()