Upload 3 files

adapter/adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "meta-llama/Meta-Llama-3-8B-Instruct",
+  "bias": "none",
+  "fan_in_fan_out": null,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.05,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "k_proj",
+    "up_proj",
+    "v_proj",
+    "gate_proj",
+    "o_proj",
+    "down_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

adapter/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0d5017dc4515f66cb5143ef88278786539fe73248af0757a649ba0ca61c96aac
+size 42002584

axolotl/training.yaml

@@ -0,0 +1,93 @@
+base_model: meta-llama/Meta-Llama-3-8B-Instruct
+model_type: LlamaForCausalLM
+tokenizer_type: AutoTokenizer  # PreTrainedTokenizerFast
+
+load_in_8bit: false
+load_in_4bit: true
+strict: false
+
+datasets:
+  - path: taddeusb90/finbro-v0.1.0
+    type: alpaca
+dataset_prepared_path: last_run_prepared
+val_set_size: 0.01
+output_dir: ./out/finbro-v0.1.0-llama-3-8B-instruct-131k
+
+adapter: qlora
+lora_model_dir:
+
+sequence_len: 8192
+sample_packing: false
+pad_to_sequence_len: true
+
+lora_r: 8
+lora_alpha: 16
+lora_dropout: 0.05
+lora_target_modules:
+lora_target_linear: true
+lora_fan_in_fan_out:
+
+wandb_project: finbro-v0.1.0-llama-3-8B-instruct-131k
+wandb_entity: sigmance
+wandb_watch: "true"
+wandb_name: finbro-v0.1.0-llama-3-8B-instruct-131k
+wandb_log_model: "true"
+
+use_pose: true
+pose_max_context_len: 131072
+
+# lora_on_cpu: 
+overrides_of_model_config:
+  rope_theta: 500000.0
+  max_position_embeddings: 131072
+  rope_scaling: 
+
+gradient_accumulation_steps: 4
+micro_batch_size: 1
+num_epochs: 4
+optimizer: adamw_torch
+lr_scheduler: cosine
+learning_rate: 0.0002
+
+train_on_inputs: false
+group_by_length: false
+bf16: auto
+fp16:
+tf32: false
+
+gradient_checkpointing: true
+gradient_checkpointing_kwargs:
+  use_reentrant: true
+early_stopping_patience: 50
+resume_from_checkpoint: ./out/finbro-v0.1.0-llama-3-8B-instruct-131k/checkpoint-3600
+local_rank:
+logging_steps: 1
+xformers_attention:
+flash_attention: true
+
+
+warmup_steps: 1
+save_steps: 100
+eval_steps: 100
+# evals_per_epoch: 4
+eval_table_size:
+# saves_per_epoch: 1
+debug:
+deepspeed:
+weight_decay: 0.0
+fsdp:
+  - full_shard
+  - auto_wrap
+fsdp_config:
+  fsdp_limit_all_gathers: true
+  fsdp_sync_module_states: true
+  fsdp_offload_params: true
+  fsdp_use_orig_params: false
+  fsdp_cpu_ram_efficient_loading: true
+  fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
+  fsdp_transformer_layer_cls_to_wrap: LlamaDecoderLayer
+  fsdp_state_dict_type: FULL_STATE_DICT
+  fsdp_sharding_strategy: FULL_SHARD
+special_tokens:
+  pad_token: <|end_of_text|>
+