Training in progress, step 10

26_10_23_config_test.csv

@@ -5,4 +5,5 @@ run_number,comment,peformed_already,num_train_epochs,max_tokens,temperature,stop
 4,classification_of_valuems set True,True,2,100,0.8,False,True,False,False
 5,classification_of_valuems set True,True,2,100,0.8,False,True,False,False
 6,classification_of_valuems set True,True,2,100,0.8,False,True,False,False
-7,classification_of_valuems set True,False,2,100,0.8,False,True,False,False
+7,classification_of_valuems set True,True,2,100,0.8,False,True,False,False
+8,classification_of_valuems set True,False,2,100,0.8,False,True,False,False

26_10_23_results_real_3.csv

@@ -5,4 +5,4 @@ run_number,items_per_minute,changed_settings,total_time_taken,rouge_scores_unnes
 4,1525.602524101584,classification_of_valuems set True,17.304638385772705,0,0.3449559795118336,0.3423341666457499,0.3223067218438548,0.3962323417858379,0.3934280156800367,0.3705820791169619,0.4480554100301472,0.4472379094634592,0.4191781582462838,0.1924640322240557,0.1864412220397102,0.1791588870738077,0.2322731554369229,0.2249807377601852,0.2130668974234288,0.275524472174082,0.267281230315844,0.2520826707151871,0.25,0.25,0.25
 5,1526.1514943613795,lemmatization set True    stemming set True    classification_of_valuems set True,17.29841375350952,0,0.2740090800292036,0.3255265414032295,0.2775650458916542,0.3242606066397121,0.382074138505741,0.3267769081728278,0.3772870132781871,0.4458722947774239,0.3791647882107381,0.1589738558026723,0.1948121530040918,0.163589425650448,0.197346401556911,0.2400173323974756,0.2012985648757239,0.2417203208425002,0.291225773302693,0.2448890209857868,0.5,0.5,0.5
 6,1530.684509829563,lemmatization set True    classification_of_valuems set True,17.247185707092285,0,0.2089967577635823,0.2539863482828974,0.2180593819184433,0.2566736342083835,0.3067268601430931,0.2634860133658033,0.3083760408920908,0.3641939637052689,0.3143688188949664,0.1022732513904408,0.1231935696888519,0.1061505173202719,0.1366654941744046,0.1619876288426343,0.1387847257382091,0.1783473709510714,0.2101453928900783,0.1797982054917218,0.4999999999999998,0.4999999999999998,0.4999999999999998
-9,1537.4830854419854,lemmatization set True    stemming set True    classification_of_valuems set True,17.170920610427856,0,0.3137446610565907,0.346938031739325,0.3116890258549815,0.36716366175121,0.4078239534610949,0.3658593356405938,0.4216956378639542,0.4672338475399563,0.4174884317080075,0.1868318428239036,0.2090565062942769,0.1872856926409339,0.2333035163819085,0.2570762720696271,0.2307891585347144,0.2805117716175197,0.3026827885812185,0.2738993050878683,0.25,0.25,0.25
+10,1537.4830854419854,lemmatization set True    stemming set True    classification_of_valuems set True,17.170920610427856,0,0.3137446610565907,0.346938031739325,0.3116890258549815,0.36716366175121,0.4078239534610949,0.3658593356405938,0.4216956378639542,0.4672338475399563,0.4174884317080075,0.1868318428239036,0.2090565062942769,0.1872856926409339,0.2333035163819085,0.2570762720696271,0.2307891585347144,0.2805117716175197,0.3026827885812185,0.2738993050878683,0.25,0.25,0.25

26_10_23_results_test.csv

@@ -7,4 +7,5 @@ run_number,items_per_minute,changed_settings,total_time_taken,rouge_scores_unnes
 0,1524.8745306877388,classification_of_valuems set True,17.312899827957153,0,0.1523559158621903,0.2402063075738076,0.1711786154828841,0.1879905787977005,0.2725423044286807,0.2052492305829349,0.2249142701073362,0.3101320775218208,0.2376186625911425,0.0456612013757211,0.061517705140545,0.0486773374221998,0.064538975391185,0.08457436273416,0.0659237042923236,0.0864961253662038,0.1081372741329041,0.0849530450555745,0.5,0.5,0.5
 0,1530.8284721565442,classification_of_valuems set True,17.245563745498657,0,0.1835189445382876,0.2841207438961418,0.2112056166511617,0.2178189965285001,0.3192970863671794,0.2472962612883028,0.2545664417741938,0.3541988923842352,0.2811301451542974,0.0548828059844037,0.0718096062377719,0.060413755710315,0.0712374843623777,0.0926737949778724,0.0779697186650809,0.0904927939124252,0.1151478474543225,0.0988436189356827,0.4999999999999999,0.4999999999999999,0.4999999999999999
 0,1519.4253778075708,classification_of_valuems set True,17.37498950958252,0,0.2448062646628152,0.3469024393843046,0.2689767055556361,0.2839292845493338,0.3847744667974158,0.3046270148335429,0.3231436457836793,0.4233519343131728,0.3403500268435545,0.0988042176523105,0.1314341849392717,0.1053886873309759,0.1221408031923088,0.161322194104573,0.1288816368026844,0.1480071736130433,0.1914677474964019,0.1538736199513103,0.5000000000000002,0.5000000000000002,0.5000000000000002
+0,1534.4169599085549,classification_of_valuems set True,17.2052321434021,0,0.2302730213229578,0.3219961939226703,0.2531060186731637,0.2717769025648489,0.3726734239350072,0.2946711204360011,0.3123765700487656,0.4172674335113968,0.3315270276187541,0.0942940049125073,0.1271350898134716,0.101126848822042,0.122824934183027,0.157562640377479,0.129838907479759,0.151774580889469,0.1937226258515307,0.158954825883685,0.5,0.5,0.5
 0,0.0,0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

README.md

@@ -964,6 +964,18 @@ The following `bitsandbytes` quantization config was used during training:
 - bnb_4bit_use_double_quant: True
 - bnb_4bit_compute_dtype: bfloat16
 
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+
 The following `bitsandbytes` quantization config was used during training:
 - quant_method: bitsandbytes
 - load_in_8bit: False
@@ -1057,5 +1069,6 @@ The following `bitsandbytes` quantization config was used during training:
 - PEFT 0.5.0
 - PEFT 0.5.0
 - PEFT 0.5.0
+- PEFT 0.5.0
 
 - PEFT 0.5.0

adapter_config.json

@@ -1,6 +1,6 @@
 {
   "auto_mapping": null,
-  "base_model_name_or_path": "tiiuae/falcon-rw-1b",
+  "base_model_name_or_path": "tiiuae/falcon-7b",
   "bias": "none",
   "fan_in_fan_out": false,
   "inference_mode": true,

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c7c413039edc99f37322a0fb24b61baa3ac4182a50ce358ae8df84cc0b1df58e
-size 100733709
+oid sha256:0d5521d0dc5a0cbd51ef766a435be5a79dc06ddc478a35eae0550dc713af1c93
+size 261189453

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a0db11243e5213810db7b77dc7935e9a727eea52a14f6a40e0ce27679d1e654c
+oid sha256:ae5f7148db11fd54683f3c602f52e502a9d8996da1d3dc603e68626fb1a0651b
 size 100690288

special_tokens_map.json

@@ -1,6 +1,17 @@
 {
-  "bos_token": "<|endoftext|>",
+  "additional_special_tokens": [
+    ">>TITLE<<",
+    ">>ABSTRACT<<",
+    ">>INTRODUCTION<<",
+    ">>SUMMARY<<",
+    ">>COMMENT<<",
+    ">>ANSWER<<",
+    ">>QUESTION<<",
+    ">>DOMAIN<<",
+    ">>PREFIX<<",
+    ">>SUFFIX<<",
+    ">>MIDDLE<<"
+  ],
   "eos_token": "<|endoftext|>",
-  "pad_token": "<|endoftext|>",
-  "unk_token": "<|endoftext|>"
+  "pad_token": "<|endoftext|>"
 }

tokenizer_config.json

@@ -1,7 +1,95 @@
 {
   "add_prefix_space": false,
   "added_tokens_decoder": {
-    "50256": {
+    "0": {
+      "content": ">>TITLE<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": ">>ABSTRACT<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": ">>INTRODUCTION<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": ">>SUMMARY<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": ">>COMMENT<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": ">>ANSWER<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": ">>QUESTION<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": ">>DOMAIN<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": ">>PREFIX<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": ">>SUFFIX<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": ">>MIDDLE<<",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
       "content": "<|endoftext|>",
       "lstrip": false,
       "normalized": false,
@@ -10,11 +98,26 @@
       "special": true
     }
   },
-  "bos_token": "<|endoftext|>",
+  "additional_special_tokens": [
+    ">>TITLE<<",
+    ">>ABSTRACT<<",
+    ">>INTRODUCTION<<",
+    ">>SUMMARY<<",
+    ">>COMMENT<<",
+    ">>ANSWER<<",
+    ">>QUESTION<<",
+    ">>DOMAIN<<",
+    ">>PREFIX<<",
+    ">>SUFFIX<<",
+    ">>MIDDLE<<"
+  ],
   "clean_up_tokenization_spaces": true,
   "eos_token": "<|endoftext|>",
-  "model_max_length": 1024,
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 2048,
   "pad_token": "<|endoftext|>",
-  "tokenizer_class": "GPT2Tokenizer",
-  "unk_token": "<|endoftext|>"
+  "tokenizer_class": "PreTrainedTokenizerFast"
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:de81a9b53f31479f45e0d3951a33c2bff7ee7362fe634aac9c0438b9164db6ce
+oid sha256:d44904d4f28af8a9f09198e70fae4bb2617a43ca6ca5c1dabf4b9f141cad2df8
 size 4283