checkpoint-200 for llama2-13b lazy lora

README.md

@@ -8,7 +8,17 @@ license: llama2
 
 0. using the updated [Meta's LLaMA-2 models](https://huggingface.co/meta-llama/Llama-2-13b-hf).
 1. support [4-bit qlora](https://arxiv.org/abs/2305.14314), extreme GPU memory and inference time saving;
-2. better MMLU evaluation dataset results, llama2-13b's 54.8% to our 57.16% (+2.36%).
+2. comparable MMLU evaluation dataset results, llama2-13b:
+
+ |               | eval   | test   | comp-eval | comp-test |
+ |---------------|--------|--------|-----------|-----------|
+ |llama2-13b     | 56.57% | -      |           |           |
+ |ckpt-200       | 56.19% | 54.75% | -0.38%    | -         |
+ |ckpt-800       | 53.16% | 55.06% | -3.41%    | -         |
+
+ llama2-13b: "31820c723bc0368c339e067553021dc6a8bf4375"
+
+
 3. This lazy-lora adapter is based on [Meta's LLaMA-2-13b-hf](https://huggingface.co/meta-llama/Llama-2-13b-hf), and using the [oasst1 dataset](https://huggingface.co/datasets/OpenAssistant/oasst1), following [Guanaco](https://huggingface.co/timdettmers/guanaco-65b).
 
 ### Introduction
@@ -84,67 +94,131 @@ model.print_trainable_parameters()
 
 ## MMLU result:
 
+### MMLU eval result:
 ```json
-{"mmlu_loss": 1.7513423996918502,
- "mmlu_eval_accuracy_high_school_microeconomics": 0.6538461538461539,
- "mmlu_eval_accuracy_human_aging": 0.6956521739130435,
- "mmlu_eval_accuracy_high_school_biology": 0.5625,
- "mmlu_eval_accuracy_machine_learning": 0.45454545454545453,
- "mmlu_eval_accuracy_moral_scenarios": 0.31,
- "mmlu_eval_accuracy_astronomy": 0.6875,
- "mmlu_eval_accuracy_medical_genetics": 0.8181818181818182,
+{"mmlu_loss": 1.459045737611944,
+ "mmlu_eval_accuracy_sociology": 0.8181818181818182,
  "mmlu_eval_accuracy_high_school_government_and_politics": 0.7142857142857143,
- "mmlu_eval_accuracy_anatomy": 0.35714285714285715,
- "mmlu_eval_accuracy_prehistory": 0.5714285714285714,
- "mmlu_eval_accuracy_high_school_mathematics": 0.20689655172413793,
- "mmlu_eval_accuracy_philosophy": 0.6176470588235294,
- "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
- "mmlu_eval_accuracy_high_school_statistics": 0.30434782608695654,
- "mmlu_eval_accuracy_conceptual_physics": 0.38461538461538464,
- "mmlu_eval_accuracy_global_facts": 0.5,
- "mmlu_eval_accuracy_high_school_us_history": 0.7727272727272727,
- "mmlu_eval_accuracy_professional_accounting": 0.41935483870967744,
- "mmlu_eval_accuracy_college_computer_science": 0.5454545454545454,
- "mmlu_eval_accuracy_econometrics": 0.4166666666666667,
- "mmlu_eval_accuracy_high_school_physics": 0.23529411764705882,
- "mmlu_eval_accuracy_public_relations": 0.6666666666666666,
+ "mmlu_eval_accuracy_marketing": 0.88,
  "mmlu_eval_accuracy_us_foreign_policy": 0.9090909090909091,
- "mmlu_eval_accuracy_miscellaneous": 0.6744186046511628,
- "mmlu_eval_accuracy_college_mathematics": 0.45454545454545453,
+ "mmlu_eval_accuracy_high_school_world_history": 0.5,
  "mmlu_eval_accuracy_management": 0.7272727272727273,
- "mmlu_eval_accuracy_college_biology": 0.625,
- "mmlu_eval_accuracy_high_school_world_history": 0.5384615384615384,
- "mmlu_eval_accuracy_electrical_engineering": 0.5,
+ "mmlu_eval_accuracy_college_mathematics": 0.36363636363636365,
+ "mmlu_eval_accuracy_high_school_us_history": 0.8181818181818182,
+ "mmlu_eval_accuracy_high_school_computer_science": 0.5555555555555556,
+ "mmlu_eval_accuracy_elementary_mathematics": 0.3170731707317073,
+ "mmlu_eval_accuracy_high_school_chemistry": 0.36363636363636365,
  "mmlu_eval_accuracy_computer_security": 0.8181818181818182,
- "mmlu_eval_accuracy_clinical_knowledge": 0.4482758620689655,
- "mmlu_eval_accuracy_professional_psychology": 0.5507246376811594,
- "mmlu_eval_accuracy_high_school_geography": 0.8181818181818182,
- "mmlu_eval_accuracy_high_school_psychology": 0.8333333333333334,
- "mmlu_eval_accuracy_moral_disputes": 0.631578947368421,
- "mmlu_eval_accuracy_formal_logic": 0.2857142857142857,
- "mmlu_eval_accuracy_international_law": 0.8461538461538461,
- "mmlu_eval_accuracy_nutrition": 0.7878787878787878,
- "mmlu_eval_accuracy_marketing": 0.88,
- "mmlu_eval_accuracy_high_school_chemistry": 0.4090909090909091,
  "mmlu_eval_accuracy_college_chemistry": 0.375,
- "mmlu_eval_accuracy_professional_medicine": 0.5483870967741935,
- "mmlu_eval_accuracy_virology": 0.3888888888888889,
- "mmlu_eval_accuracy_logical_fallacies": 0.7222222222222222,
- "mmlu_eval_accuracy_high_school_european_history": 0.7777777777777778,
+ "mmlu_eval_accuracy_econometrics": 0.4166666666666667,
+ "mmlu_eval_accuracy_professional_psychology": 0.5362318840579711,
+ "mmlu_eval_accuracy_high_school_statistics": 0.30434782608695654,
+ "mmlu_eval_accuracy_high_school_macroeconomics": 0.5581395348837209,
+ "mmlu_eval_accuracy_moral_scenarios": 0.36,
+ "mmlu_eval_accuracy_moral_disputes": 0.6578947368421053,
+ "mmlu_eval_accuracy_astronomy": 0.5625,
+ "mmlu_eval_accuracy_professional_accounting": 0.41935483870967744,
+ "mmlu_eval_accuracy_human_sexuality": 0.5,
+ "mmlu_eval_accuracy_world_religions": 0.7894736842105263,
+ "mmlu_eval_accuracy_high_school_psychology": 0.8166666666666667,
+ "mmlu_eval_accuracy_nutrition": 0.7575757575757576,
+ "mmlu_eval_accuracy_high_school_geography": 0.8181818181818182,
+ "mmlu_eval_accuracy_global_facts": 0.4,
+ "mmlu_eval_accuracy_high_school_mathematics": 0.20689655172413793,
+ "mmlu_eval_accuracy_college_medicine": 0.4090909090909091,
+ "mmlu_eval_accuracy_electrical_engineering": 0.5,
  "mmlu_eval_accuracy_jurisprudence": 0.6363636363636364,
- "mmlu_eval_accuracy_human_sexuality": 0.5833333333333334,
+ "mmlu_eval_accuracy_public_relations": 0.5833333333333334,
+ "mmlu_eval_accuracy_virology": 0.4444444444444444,
+ "mmlu_eval_accuracy_high_school_physics": 0.29411764705882354,
+ "mmlu_eval_accuracy_college_physics": 0.6363636363636364,
+ "mmlu_eval_accuracy_miscellaneous": 0.686046511627907,
+ "mmlu_eval_accuracy_international_law": 0.8461538461538461,
+ "mmlu_eval_accuracy_human_aging": 0.6956521739130435,
+ "mmlu_eval_accuracy_conceptual_physics": 0.38461538461538464,
+ "mmlu_eval_accuracy_formal_logic": 0.2857142857142857,
+ "mmlu_eval_accuracy_medical_genetics": 0.7272727272727273,
+ "mmlu_eval_accuracy_philosophy": 0.6470588235294118,
+ "mmlu_eval_accuracy_professional_medicine": 0.5806451612903226,
+ "mmlu_eval_accuracy_security_studies": 0.6666666666666666,
+ "mmlu_eval_accuracy_college_biology": 0.625,
+ "mmlu_eval_accuracy_high_school_biology": 0.53125,
+ "mmlu_eval_accuracy_machine_learning": 0.45454545454545453,
+ "mmlu_eval_accuracy_high_school_microeconomics": 0.7307692307692307,
+ "mmlu_eval_accuracy_professional_law": 0.38823529411764707,
+ "mmlu_eval_accuracy_logical_fallacies": 0.7777777777777778,
+ "mmlu_eval_accuracy_high_school_european_history": 0.7222222222222222,
+ "mmlu_eval_accuracy_prehistory": 0.6,
  "mmlu_eval_accuracy_abstract_algebra": 0.2727272727272727,
+ "mmlu_eval_accuracy_clinical_knowledge": 0.5517241379310345,
+ "mmlu_eval_accuracy_anatomy": 0.35714285714285715,
  "mmlu_eval_accuracy_business_ethics": 0.2727272727272727,
- "mmlu_eval_accuracy_security_studies": 0.7037037037037037,
- "mmlu_eval_accuracy_professional_law": 0.4,
- "mmlu_eval_accuracy_college_medicine": 0.5,
- "mmlu_eval_accuracy_elementary_mathematics": 0.34146341463414637,
- "mmlu_eval_accuracy_high_school_macroeconomics": 0.5813953488372093,
- "mmlu_eval_accuracy_sociology": 0.8636363636363636,
- "mmlu_eval_accuracy_world_religions": 0.7894736842105263,
- "mmlu_eval_accuracy_college_physics": 0.6363636363636364,
- "mmlu_eval_accuracy": 0.5716083571911647,
- "epoch": 0.68}
+ "mmlu_eval_accuracy_college_computer_science": 0.45454545454545453,
+ "mmlu_eval_accuracy": 0.5618637260755334}
+```
+
+### MMLU test result:
+
+```json
+{"mmlu_loss": 1.4737504210172938,
+ "mmlu_test_accuracy_professional_law": 0.4172099087353325,
+ "mmlu_test_accuracy_formal_logic": 0.3412698412698413,
+ "mmlu_test_accuracy_anatomy": 0.4962962962962963,
+ "mmlu_test_accuracy_professional_medicine": 0.5404411764705882,
+ "mmlu_test_accuracy_high_school_mathematics": 0.2851851851851852,
+ "mmlu_test_accuracy_high_school_government_and_politics": 0.8134715025906736,
+ "mmlu_test_accuracy_electrical_engineering": 0.5448275862068965,
+ "mmlu_test_accuracy_conceptual_physics": 0.4595744680851064,
+ "mmlu_test_accuracy_jurisprudence": 0.7407407407407407,
+ "mmlu_test_accuracy_college_chemistry": 0.4,
+ "mmlu_test_accuracy_high_school_computer_science": 0.56,
+ "mmlu_test_accuracy_college_computer_science": 0.44,
+ "mmlu_test_accuracy_marketing": 0.7863247863247863,
+ "mmlu_test_accuracy_high_school_psychology": 0.7376146788990826,
+ "mmlu_test_accuracy_professional_psychology": 0.5522875816993464,
+ "mmlu_test_accuracy_logical_fallacies": 0.6748466257668712,
+ "mmlu_test_accuracy_high_school_microeconomics": 0.5672268907563025,
+ "mmlu_test_accuracy_international_law": 0.7107438016528925,
+ "mmlu_test_accuracy_high_school_world_history": 0.7088607594936709,
+ "mmlu_test_accuracy_medical_genetics": 0.57,
+ "mmlu_test_accuracy_professional_accounting": 0.41843971631205673,
+ "mmlu_test_accuracy_business_ethics": 0.57,
+ "mmlu_test_accuracy_moral_scenarios": 0.293854748603352,
+ "mmlu_test_accuracy_abstract_algebra": 0.34,
+ "mmlu_test_accuracy_elementary_mathematics": 0.328042328042328,
+ "mmlu_test_accuracy_high_school_biology": 0.6580645161290323,
+ "mmlu_test_accuracy_public_relations": 0.6090909090909091,
+ "mmlu_test_accuracy_high_school_physics": 0.31788079470198677,
+ "mmlu_test_accuracy_astronomy": 0.4934210526315789,
+ "mmlu_test_accuracy_high_school_us_history": 0.75,
+ "mmlu_test_accuracy_college_medicine": 0.5028901734104047,
+ "mmlu_test_accuracy_college_mathematics": 0.34,
+ "mmlu_test_accuracy_econometrics": 0.2894736842105263,
+ "mmlu_test_accuracy_clinical_knowledge": 0.6150943396226415,
+ "mmlu_test_accuracy_moral_disputes": 0.630057803468208,
+ "mmlu_test_accuracy_machine_learning": 0.26785714285714285,
+ "mmlu_test_accuracy_human_sexuality": 0.5801526717557252,
+ "mmlu_test_accuracy_virology": 0.4457831325301205,
+ "mmlu_test_accuracy_high_school_chemistry": 0.43349753694581283,
+ "mmlu_test_accuracy_high_school_geography": 0.6515151515151515,
+ "mmlu_test_accuracy_high_school_european_history": 0.6363636363636364,
+ "mmlu_test_accuracy_nutrition": 0.5849673202614379,
+ "mmlu_test_accuracy_human_aging": 0.6502242152466368,
+ "mmlu_test_accuracy_philosophy": 0.639871382636656,
+ "mmlu_test_accuracy_global_facts": 0.34,
+ "mmlu_test_accuracy_high_school_statistics": 0.4722222222222222,
+ "mmlu_test_accuracy_college_biology": 0.5486111111111112,
+ "mmlu_test_accuracy_management": 0.6893203883495146,
+ "mmlu_test_accuracy_sociology": 0.7213930348258707,
+ "mmlu_test_accuracy_security_studies": 0.6326530612244898,
+ "mmlu_test_accuracy_college_physics": 0.24509803921568626,
+ "mmlu_test_accuracy_high_school_macroeconomics": 0.5,
+ "mmlu_test_accuracy_us_foreign_policy": 0.83,
+ "mmlu_test_accuracy_miscellaneous": 0.7343550446998723,
+ "mmlu_test_accuracy_prehistory": 0.6512345679012346,
+ "mmlu_test_accuracy_computer_security": 0.69,
+ "mmlu_test_accuracy_world_religions": 0.7602339181286549,
+ "mmlu_test_accuracy": 0.5475190434068002}
 ```
 
 ## License and intended use 

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:946885901731680cd58d8c7d6b65785bd7c8b8463e7e9078f69daf3b91672099
+oid sha256:326f478b0100dbf136cd83e9493d97fd1d65c2c7f4fd91bf93040e000ca69308
 size 500836813