Training in progress, step 10

26_10_23_config_test.csv

@@ -10,3 +10,4 @@ run_number,comment,peformed_already,num_train_epochs,max_tokens,temperature,stop
 9,classification_of_valuems set True,False,2,100,0.8,False,True,False,False
 10,classification_of_valuems set True,False,2,100,0.8,False,True,False,False
 11,classification_of_valuems set True,False,2,100,0.8,False,True,False,False
+12,classification_of_valuems set True,False,2,100,0.8,False,True,False,False

26_10_23_results_real_3.csv

@@ -5,4 +5,4 @@ run_number,items_per_minute,changed_settings,total_time_taken,rouge_scores_unnes
 4,1525.602524101584,classification_of_valuems set True,17.304638385772705,0,0.3449559795118336,0.3423341666457499,0.3223067218438548,0.3962323417858379,0.3934280156800367,0.3705820791169619,0.4480554100301472,0.4472379094634592,0.4191781582462838,0.1924640322240557,0.1864412220397102,0.1791588870738077,0.2322731554369229,0.2249807377601852,0.2130668974234288,0.275524472174082,0.267281230315844,0.2520826707151871,0.25,0.25,0.25
 5,1526.1514943613795,lemmatization set True    stemming set True    classification_of_valuems set True,17.29841375350952,0,0.2740090800292036,0.3255265414032295,0.2775650458916542,0.3242606066397121,0.382074138505741,0.3267769081728278,0.3772870132781871,0.4458722947774239,0.3791647882107381,0.1589738558026723,0.1948121530040918,0.163589425650448,0.197346401556911,0.2400173323974756,0.2012985648757239,0.2417203208425002,0.291225773302693,0.2448890209857868,0.5,0.5,0.5
 6,1530.684509829563,lemmatization set True    classification_of_valuems set True,17.247185707092285,0,0.2089967577635823,0.2539863482828974,0.2180593819184433,0.2566736342083835,0.3067268601430931,0.2634860133658033,0.3083760408920908,0.3641939637052689,0.3143688188949664,0.1022732513904408,0.1231935696888519,0.1061505173202719,0.1366654941744046,0.1619876288426343,0.1387847257382091,0.1783473709510714,0.2101453928900783,0.1797982054917218,0.4999999999999998,0.4999999999999998,0.4999999999999998
-14,1537.4830854419854,lemmatization set True    stemming set True    classification_of_valuems set True,17.170920610427856,0,0.3137446610565907,0.346938031739325,0.3116890258549815,0.36716366175121,0.4078239534610949,0.3658593356405938,0.4216956378639542,0.4672338475399563,0.4174884317080075,0.1868318428239036,0.2090565062942769,0.1872856926409339,0.2333035163819085,0.2570762720696271,0.2307891585347144,0.2805117716175197,0.3026827885812185,0.2738993050878683,0.25,0.25,0.25
+15,1537.4830854419854,lemmatization set True    stemming set True    classification_of_valuems set True,17.170920610427856,0,0.3137446610565907,0.346938031739325,0.3116890258549815,0.36716366175121,0.4078239534610949,0.3658593356405938,0.4216956378639542,0.4672338475399563,0.4174884317080075,0.1868318428239036,0.2090565062942769,0.1872856926409339,0.2333035163819085,0.2570762720696271,0.2307891585347144,0.2805117716175197,0.3026827885812185,0.2738993050878683,0.25,0.25,0.25

26_10_23_results_test.csv

@@ -13,3 +13,4 @@ run_number,items_per_minute,changed_settings,total_time_taken,rouge_scores_unnes
 0,0.0,0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
 0,0.0,0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
 0,0.0,0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0
+0,0.0,0,0.0,0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0

README.md

@@ -976,6 +976,18 @@ The following `bitsandbytes` quantization config was used during training:
 - bnb_4bit_use_double_quant: True
 - bnb_4bit_compute_dtype: bfloat16
 
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: False
+- load_in_4bit: True
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: nf4
+- bnb_4bit_use_double_quant: True
+- bnb_4bit_compute_dtype: bfloat16
+
 The following `bitsandbytes` quantization config was used during training:
 - quant_method: bitsandbytes
 - load_in_8bit: False
@@ -1070,5 +1082,6 @@ The following `bitsandbytes` quantization config was used during training:
 - PEFT 0.5.0
 - PEFT 0.5.0
 - PEFT 0.5.0
+- PEFT 0.5.0
 
 - PEFT 0.5.0

adapter_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9950b97f63bb37f53555b0552d50ea07c37bfd47bf87179b3d3d44fac6d798c9
+oid sha256:b536a1d4926ea279ed7f2999a3e87c440f1d8d077c4438f523a01fb41ff2a737
 size 261189453

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae5f7148db11fd54683f3c602f52e502a9d8996da1d3dc603e68626fb1a0651b
-size 100690288
+oid sha256:f619bac221ad47d5e75d145e26ade5d18998b4360842d8357557853adf65ebdf
+size 261131840

config.json

@@ -1,33 +1,33 @@
 {
-  "_name_or_path": "tiiuae/falcon-rw-1b",
-  "alibi": true,
+  "_name_or_path": "tiiuae/falcon-7b",
+  "alibi": false,
   "apply_residual_connection_post_layernorm": false,
   "architectures": [
     "FalconForCausalLM"
   ],
   "attention_dropout": 0.0,
   "auto_map": {
-    "AutoConfig": "tiiuae/falcon-rw-1b--configuration_falcon.FalconConfig",
-    "AutoModel": "tiiuae/falcon-rw-1b--modeling_falcon.FalconModel",
-    "AutoModelForCausalLM": "tiiuae/falcon-rw-1b--modeling_falcon.FalconForCausalLM",
-    "AutoModelForQuestionAnswering": "tiiuae/falcon-rw-1b--modeling_falcon.FalconForQuestionAnswering",
-    "AutoModelForSequenceClassification": "tiiuae/falcon-rw-1b--modeling_falcon.FalconForSequenceClassification",
-    "AutoModelForTokenClassification": "tiiuae/falcon-rw-1b--modeling_falcon.FalconForTokenClassification"
+    "AutoConfig": "tiiuae/falcon-7b--configuration_falcon.FalconConfig",
+    "AutoModel": "tiiuae/falcon-7b--modeling_falcon.FalconModel",
+    "AutoModelForCausalLM": "tiiuae/falcon-7b--modeling_falcon.FalconForCausalLM",
+    "AutoModelForQuestionAnswering": "tiiuae/falcon-7b--modeling_falcon.FalconForQuestionAnswering",
+    "AutoModelForSequenceClassification": "tiiuae/falcon-7b--modeling_falcon.FalconForSequenceClassification",
+    "AutoModelForTokenClassification": "tiiuae/falcon-7b--modeling_falcon.FalconForTokenClassification"
   },
-  "bias": true,
-  "bos_token_id": 1,
-  "eos_token_id": 2,
+  "bias": false,
+  "bos_token_id": 11,
+  "eos_token_id": 11,
   "hidden_dropout": 0.0,
-  "hidden_size": 2048,
+  "hidden_size": 4544,
   "initializer_range": 0.02,
   "layer_norm_epsilon": 1e-05,
   "model_type": "falcon",
-  "multi_query": false,
+  "multi_query": true,
   "new_decoder_architecture": false,
-  "num_attention_heads": 32,
-  "num_hidden_layers": 24,
-  "num_kv_heads": 32,
-  "parallel_attn": false,
+  "num_attention_heads": 71,
+  "num_hidden_layers": 32,
+  "num_kv_heads": 71,
+  "parallel_attn": true,
   "quantization_config": {
     "bnb_4bit_compute_dtype": "bfloat16",
     "bnb_4bit_quant_type": "nf4",
@@ -43,5 +43,5 @@
   "torch_dtype": "bfloat16",
   "transformers_version": "4.34.1",
   "use_cache": false,
-  "vocab_size": 50304
+  "vocab_size": 65024
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bd41500e653f0b2a83d0a0787831fddbaf302b015d6871fb22cddf10e8e0a01b
+oid sha256:17e1d6ea179881703aebc9cf5ee9455199d40dcc06ec43acb88843d785a2245b
 size 4283