configuration load codeparrot tokenizer trained for python code tokenization config scaleattnbylayeridx and reorderandupcastattn are mistral stability tweaks load model config gpt2 large in this case initialize new model with config save model to the hub configuration load codeparrot tokenizer trained for python code tokenization config scale_attn_by_layer_idx and reorder_and_upcast_attn are mistral stability tweaks load model config gpt 2 large in this case initialize new model with config save model to the hub
from arguments import InitializationArguments

from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer, HfArgumentParser



parser = HfArgumentParser(InitializationArguments)
args = parser.parse_args()


tokenizer = AutoTokenizer.from_pretrained(args.tokenizer_name)


config_kwargs = {
    "vocab_size": len(tokenizer),
    "scale_attn_by_inverse_layer_idx": True,
    "reorder_and_upcast_attn": True,
}


config = AutoConfig.from_pretrained(args.config_name, **config_kwargs)


model = AutoModelForCausalLM.from_config(config)


model.save_pretrained(args.model_name, push_to_hub=args.push_to_hub)