Upload folder using huggingface_hub

README.md

@@ -1,16 +1,34 @@
 ---
-datasets:
-- yahma/alpaca-cleaned
-language:
-- en
-license: apache-2.0
-tags:
-- pytorch
-- causal-lm
-- llama2
-- fine-tuning
-- alpaca
-
+library_name: peft
 ---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: True
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: True
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+### Framework versions
+
+- PEFT 0.5.0.dev0
 
-# Llama-2-7b fine-tuned on LoRA alpaca-cleaned
+- PEFT 0.5.0.dev0

flyte_training_config.json

@@ -1 +1 @@
-{"base_model": "meta-llama/Llama-2-7b-hf", "data_path": "yahma/alpaca-cleaned", "instruction_key": "instruction", "input_key": "input", "output_key": "output", "output_dir": "./output", "device_map": "auto", "batch_size": 32, "micro_batch_size": 8, "num_epochs": 1, "max_steps": 300, "eval_steps": 200, "save_steps": 50, "learning_rate": 0.0003, "cutoff_len": 512, "val_set_size": 0, "lora_r": 8, "lora_alpha": 16, "lora_dropout": 0.05, "weight_decay": 0.02, "warmup_ratio": 0.03, "lr_scheduler_type": "cosine", "lora_target_modules": ["q_proj", "k_proj", "v_proj"], "train_on_inputs": true, "add_eos_token": true, "group_by_length": true, "resume_from_checkpoint": null, "wandb_project": "unionai-llm-fine-tuning", "wandb_run_name": "", "wandb_watch": "", "wandb_log_model": "", "debug_mode": false, "debug_train_data_size": 1024}
+{"base_model": "meta-llama/Llama-2-7b-hf", "data_path": "yahma/alpaca-cleaned", "instruction_key": "instruction", "input_key": "input", "output_key": "output", "output_dir": "./output", "device_map": "auto", "batch_size": 32, "micro_batch_size": 16, "num_epochs": 1, "max_steps": 50, "eval_steps": 5, "save_steps": 500, "learning_rate": 3e-08, "cutoff_len": 512, "val_set_size": 1000, "lora_r": 8, "lora_alpha": 16, "lora_dropout": 0.05, "weight_decay": 0.02, "warmup_ratio": 0.03, "lr_scheduler_type": "cosine", "lora_target_modules": ["q_proj", "k_proj", "v_proj"], "train_on_inputs": true, "add_eos_token": true, "group_by_length": false, "resume_from_checkpoint": null, "wandb_project": "unionai-llm-fine-tuning", "wandb_run_name": "", "wandb_watch": "", "wandb_log_model": "", "debug_mode": false, "debug_train_data_size": 1024}

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c7404ccd2db3be9b05a3dc59889c92771351aab687ad0c14878fca9a50c2c646
+oid sha256:0018f2fe57030840f1adb0dd94cc28a1803425bcce15c72278fbfb5f07984334
 size 4027