Training in progress, step 75

adapter_config.json

@@ -1,17 +1,26 @@
 {
+  "alpha_pattern": {},
   "auto_mapping": null,
-  "base_model_name_or_path": "deepseek-ai/deepseek-coder-1.3b-instruct",
-  "encoder_dropout": 0.0,
-  "encoder_hidden_size": 250,
-  "encoder_num_layers": 2,
-  "encoder_reparameterization_type": "MLP",
+  "base_model_name_or_path": "teknium/OpenHermes-2.5-Mistral-7B",
+  "bias": "none",
+  "fan_in_fan_out": false,
   "inference_mode": true,
-  "num_attention_heads": 16,
-  "num_layers": 24,
-  "num_transformer_submodules": 1,
-  "num_virtual_tokens": 50,
-  "peft_type": "P_TUNING",
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 32,
+  "lora_dropout": 0.1,
+  "modules_to_save": [
+    "embed_tokens"
+  ],
+  "peft_type": "LORA",
+  "r": 16,
+  "rank_pattern": {},
   "revision": null,
-  "task_type": "CAUSAL_LM",
-  "token_dim": 2048
+  "target_modules": [
+    "gate_proj",
+    "up_proj",
+    "down_proj"
+  ],
+  "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:79acdcc1b8e57db310c2b04aab07db81dd60615a67b42d7a105b2b40f171d1e8
-size 409720
+oid sha256:ab69c31afaffb1acf340f47e56d2f4deaa97626231f62effadcb10ae6d5c88b2
+size 637592768

added_tokens.json

@@ -0,0 +1,4 @@
+{
+  "<|im_end|>": 32000,
+  "<|im_start|>": 32001
+}

special_tokens_map.json

@@ -1,17 +1,6 @@
 {
-  "bos_token": {
-    "content": "<｜begin▁of▁sentence｜>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "<|EOT|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "pad_token": "Ġsq"
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "</s>",
+  "unk_token": "<unk>"
 }

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -2,192 +2,60 @@
   "add_bos_token": true,
   "add_eos_token": true,
   "added_tokens_decoder": {
-    "32000": {
-      "content": "õ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "32001": {
-      "content": "÷",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "32002": {
-      "content": "Á",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "32003": {
-      "content": "ý",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "32004": {
-      "content": "À",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "32005": {
-      "content": "ÿ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "32006": {
-      "content": "ø",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "32007": {
-      "content": "ú",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "32008": {
-      "content": "þ",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "32009": {
-      "content": "ü",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "32010": {
-      "content": "ù",
+    "0": {
+      "content": "<unk>",
       "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "32011": {
-      "content": "ö",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "32012": {
-      "content": "û",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "32013": {
-      "content": "<｜begin▁of▁sentence｜>",
-      "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "32014": {
-      "content": "<｜end▁of▁sentence｜>",
+    "1": {
+      "content": "<s>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     },
-    "32015": {
-      "content": "<｜fim▁hole｜>",
+    "2": {
+      "content": "</s>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
-    },
-    "32016": {
-      "content": "<｜fim▁begin｜>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "32017": {
-      "content": "<｜fim▁end｜>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "32018": {
-      "content": "<pad>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
-    },
-    "32019": {
-      "content": "<|User|>",
-      "lstrip": false,
-      "normalized": true,
-      "rstrip": false,
-      "single_word": false,
-      "special": false
+      "special": true
     },
-    "32020": {
-      "content": "<|Assistant|>",
+    "32000": {
+      "content": "<|im_end|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
-    "32021": {
-      "content": "<|EOT|>",
+    "32001": {
+      "content": "<|im_start|>",
       "lstrip": false,
-      "normalized": true,
+      "normalized": false,
       "rstrip": false,
       "single_word": false,
       "special": true
     }
   },
-  "bos_token": "<｜begin▁of▁sentence｜>",
-  "chat_template": "{% if not add_generation_prompt is defined %}\n{% set add_generation_prompt = false %}\n{% endif %}\n{%- set ns = namespace(found=false) -%}\n{%- for message in messages -%}\n    {%- if message['role'] == 'system' -%}\n        {%- set ns.found = true -%}\n    {%- endif -%}\n{%- endfor -%}\n{{bos_token}}{%- if not ns.found -%}\n{{'You are an AI programming assistant, utilizing the Deepseek Coder model, developed by Deepseek Company, and you only answer questions related to computer science. For politically sensitive questions, security and privacy issues, and other non-computer science questions, you will refuse to answer\\n'}}\n{%- endif %}\n{%- for message in messages %}\n    {%- if message['role'] == 'system' %}\n{{ message['content'] }}\n    {%- else %}\n        {%- if message['role'] == 'user' %}\n{{'### Instruction:\\n' + message['content'] + '\\n'}}\n        {%- else %}\n{{'### Response:\\n' + message['content'] + '\\n<|EOT|>\\n'}}\n        {%- endif %}\n    {%- endif %}\n{%- endfor %}\n{% if add_generation_prompt %}\n{{'### Response:'}}\n{% endif %}",
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
   "clean_up_tokenization_spaces": false,
-  "eos_token": "<|EOT|>",
+  "eos_token": "</s>",
   "legacy": true,
-  "model_max_length": 16384,
-  "pad_token": "Ġsq",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
   "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
-  "unk_token": null,
-  "use_default_system_prompt": false
+  "trust_remote_code": false,
+  "unk_token": "<unk>",
+  "use_default_system_prompt": true,
+  "use_fast": true
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:759434762d7bfb55a21f68b18a6e6125b5035fbd73274ea67425ff0b6cc062ad
+oid sha256:b239c79424204a0c8ffc8edf94952bccae3e9a354372d5dc4f94a81aad08e77e
 size 4664