





import argparse

import torch

from transformers import GPTaLMHeadModel, RobertaForMaskedLM


if __name__ == "__main__":
				UpperCAmelCase						:   List[Any]			  =     argparse.ArgumentParser(
				    description=(
				        '''Extraction some layers of the full RobertaForMaskedLM or GPT2LMHeadModel for Transfer Learned'''
				        ''' Distillation'''
				    )
				)
				parser.add_argument('''--model_type''', default='''roberta''', choices=['''roberta''', '''gpt2'''])
				parser.add_argument('''--model_name''', default='''roberta-large''', type=str)
				parser.add_argument('''--dump_checkpoint''', default='''serialization_dir/tf_roberta_048131723.pth''', type=str)
				parser.add_argument('''--vocab_transform''', action='''store_true''')
				UpperCAmelCase						:   str			  =     parser.parse_args()

				if args.model_type == "roberta":
								UpperCAmelCase						:   List[Any]			  =     RobertaForMaskedLM.from_pretrained(args.model_name)
								UpperCAmelCase						:   Optional[int]			  =     'roberta'
				elif args.model_type == "gpt2":
								UpperCAmelCase						:   List[Any]			  =     GPTaLMHeadModel.from_pretrained(args.model_name)
								UpperCAmelCase						:   int			  =     'transformer'

				UpperCAmelCase						:   Optional[Any]			  =     model.state_dict()
				UpperCAmelCase						:   Tuple			  =     {}

				# Embeddings #
				if args.model_type == "gpt2":
								for param_name in ["wte.weight", "wpe.weight"]:
												UpperCAmelCase						:   int			  =     state_dict[F"""{prefix}.{param_name}"""]
				else:
								for w in ["word_embeddings", "position_embeddings", "token_type_embeddings"]:
												UpperCAmelCase						:   Dict			  =     F"""{prefix}.embeddings.{w}.weight"""
												UpperCAmelCase						:   Optional[int]			  =     state_dict[param_name]
								for w in ["weight", "bias"]:
												UpperCAmelCase						:   Tuple			  =     F"""{prefix}.embeddings.LayerNorm.{w}"""
												UpperCAmelCase						:   Optional[Any]			  =     state_dict[param_name]

    # Transformer Blocks #
				UpperCAmelCase						:   Optional[Any]			  =     0
				for teacher_idx in [0, 2, 4, 7, 9, 11]:
								if args.model_type == "gpt2":
												for layer in ["ln_1", "attn.c_attn", "attn.c_proj", "ln_2", "mlp.c_fc", "mlp.c_proj"]:
																for w in ["weight", "bias"]:
																				UpperCAmelCase						:   str			  =     state_dict[
																				    F"""{prefix}.h.{teacher_idx}.{layer}.{w}"""
																				]
												UpperCAmelCase						:   Union[str, Any]			  =     state_dict[F"""{prefix}.h.{teacher_idx}.attn.bias"""]
								else:
												for layer in [
												    "attention.self.query",
												    "attention.self.key",
												    "attention.self.value",
												    "attention.output.dense",
												    "attention.output.LayerNorm",
												    "intermediate.dense",
												    "output.dense",
												    "output.LayerNorm",
												]:
																for w in ["weight", "bias"]:
																				UpperCAmelCase						:   List[Any]			  =     state_dict[
																				    F"""{prefix}.encoder.layer.{teacher_idx}.{layer}.{w}"""
																				]
								std_idx += 1

				# Language Modeling Head ###s
				if args.model_type == "roberta":
								for layer in ["lm_head.decoder.weight", "lm_head.bias"]:
												UpperCAmelCase						:   str			  =     state_dict[F"""{layer}"""]
								if args.vocab_transform:
												for w in ["weight", "bias"]:
																UpperCAmelCase						:   Dict			  =     state_dict[F"""lm_head.dense.{w}"""]
																UpperCAmelCase						:   int			  =     state_dict[F"""lm_head.layer_norm.{w}"""]
				elif args.model_type == "gpt2":
								for w in ["weight", "bias"]:
												UpperCAmelCase						:   Optional[Any]			  =     state_dict[F"""{prefix}.ln_f.{w}"""]
								UpperCAmelCase						:   Optional[Any]			  =     state_dict['lm_head.weight']

				print(F"""N layers selected for distillation: {std_idx}""")
				print(F"""Number of params transferred for distillation: {len(compressed_sd.keys())}""")

				print(F"""Save transferred checkpoint to {args.dump_checkpoint}.""")
				torch.save(compressed_sd, args.dump_checkpoint)











from itertools import permutations



def _a				(	lowerCamelCase:   tuple )	->						bool:







      '''simple docstring'''






      if num[3] % 2 != 0:
            return False

      if (num[2] + num[3] + num[4]) % 3 != 0:
            return False

      if num[5] % 5 != 0:
            return False

      __A          =				[7, 11, 13, 17]
      for i, test in enumerate(lowerCamelCase ):
            if (num[i + 4] * 1_00 + num[i + 5] * 10 + num[i + 6]) % test != 0:
                  return False
      return True



def _a				(	lowerCamelCase:   int = 10 )	->						int:







      '''simple docstring'''






      return sum(
          int(''''''.join(map(lowerCamelCase					,				lowerCamelCase ) ) )
          for num in permutations(range(lowerCamelCase ) )
          if is_substring_divisible(lowerCamelCase ) )


if __name__ == "__main__":
       print(f'{solution() = }')





