yujiepan commited on
Commit
074f4c9
1 Parent(s): 7b752f3

Upload folder using huggingface_hub

Browse files
added_tokens.json CHANGED
@@ -1,5 +1,4 @@
1
  {
2
  "<|im_end|>": 100279,
3
- "<|im_start|>": 100278,
4
- "<|pad|>": 100277
5
  }
 
1
  {
2
  "<|im_end|>": 100279,
3
+ "<|im_start|>": 100278
 
4
  }
config.json CHANGED
@@ -9,10 +9,6 @@
9
  "model_type": "",
10
  "rope_theta": 500000
11
  },
12
- "auto_map": {
13
- "AutoConfig": "databricks/dbrx-instruct--configuration_dbrx.DbrxConfig",
14
- "AutoModelForCausalLM": "databricks/dbrx-instruct--modeling_dbrx.DbrxForCausalLM"
15
- },
16
  "d_model": 4,
17
  "emb_pdrop": 0.0,
18
  "ffn_config": {
@@ -30,10 +26,9 @@
30
  "n_layers": 2,
31
  "output_router_logits": false,
32
  "resid_pdrop": 0.0,
33
- "router_aux_loss_coef": 0.05,
34
  "tie_word_embeddings": false,
35
  "torch_dtype": "float16",
36
- "transformers_version": "4.38.1",
37
  "use_cache": true,
38
  "vocab_size": 100352
39
  }
 
9
  "model_type": "",
10
  "rope_theta": 500000
11
  },
 
 
 
 
12
  "d_model": 4,
13
  "emb_pdrop": 0.0,
14
  "ffn_config": {
 
26
  "n_layers": 2,
27
  "output_router_logits": false,
28
  "resid_pdrop": 0.0,
 
29
  "tie_word_embeddings": false,
30
  "torch_dtype": "float16",
31
+ "transformers_version": "4.40.0",
32
  "use_cache": true,
33
  "vocab_size": 100352
34
  }
generation_config.json CHANGED
@@ -1,4 +1,4 @@
1
  {
2
  "_from_model_config": true,
3
- "transformers_version": "4.38.1"
4
  }
 
1
  {
2
  "_from_model_config": true,
3
+ "transformers_version": "4.40.0"
4
  }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json CHANGED
@@ -1,8 +1,4 @@
1
  {
2
- "additional_special_tokens": [
3
- "<|im_start|>",
4
- "<|im_end|>"
5
- ],
6
  "bos_token": {
7
  "content": "<|endoftext|>",
8
  "lstrip": false,
 
1
  {
 
 
 
 
2
  "bos_token": {
3
  "content": "<|endoftext|>",
4
  "lstrip": false,
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json CHANGED
@@ -1,8 +1,14 @@
1
  {
2
- "add_bos_token": false,
3
- "add_eos_token": false,
4
  "add_prefix_space": false,
5
  "added_tokens_decoder": {
 
 
 
 
 
 
 
 
6
  "100257": {
7
  "content": "<|endoftext|>",
8
  "lstrip": false,
@@ -11,6 +17,158 @@
11
  "single_word": false,
12
  "special": true
13
  },
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
14
  "100277": {
15
  "content": "<|pad|>",
16
  "lstrip": false,
@@ -36,25 +194,12 @@
36
  "special": true
37
  }
38
  },
39
- "additional_special_tokens": [
40
- "<|im_start|>",
41
- "<|im_end|>"
42
- ],
43
- "auto_map": {
44
- "AutoTokenizer": [
45
- "databricks/dbrx-instruct--tiktoken.TiktokenTokenizerWrapper",
46
- null
47
- ]
48
- },
49
  "bos_token": "<|endoftext|>",
 
50
  "clean_up_tokenization_spaces": true,
51
- "encoding_name": null,
52
  "eos_token": "<|endoftext|>",
53
- "errors": "replace",
54
- "model_max_length": 1000000000000000019884624838656,
55
- "model_name": "gpt-4",
56
  "pad_token": "<|pad|>",
57
- "tokenizer_class": "TiktokenTokenizerWrapper",
58
- "unk_token": "<|endoftext|>",
59
- "use_default_system_prompt": true
60
  }
 
1
  {
 
 
2
  "add_prefix_space": false,
3
  "added_tokens_decoder": {
4
+ "100256": {
5
+ "content": "<||_unused_0_||>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
  "100257": {
13
  "content": "<|endoftext|>",
14
  "lstrip": false,
 
17
  "single_word": false,
18
  "special": true
19
  },
20
+ "100258": {
21
+ "content": "<|fim_prefix|>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
+ "100259": {
29
+ "content": "<|fim_middle|>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
+ },
36
+ "100260": {
37
+ "content": "<|fim_suffix|>",
38
+ "lstrip": false,
39
+ "normalized": false,
40
+ "rstrip": false,
41
+ "single_word": false,
42
+ "special": true
43
+ },
44
+ "100261": {
45
+ "content": "<||_unused_1_||>",
46
+ "lstrip": false,
47
+ "normalized": false,
48
+ "rstrip": false,
49
+ "single_word": false,
50
+ "special": true
51
+ },
52
+ "100262": {
53
+ "content": "<||_unused_2_||>",
54
+ "lstrip": false,
55
+ "normalized": false,
56
+ "rstrip": false,
57
+ "single_word": false,
58
+ "special": true
59
+ },
60
+ "100263": {
61
+ "content": "<||_unused_3_||>",
62
+ "lstrip": false,
63
+ "normalized": false,
64
+ "rstrip": false,
65
+ "single_word": false,
66
+ "special": true
67
+ },
68
+ "100264": {
69
+ "content": "<||_unused_4_||>",
70
+ "lstrip": false,
71
+ "normalized": false,
72
+ "rstrip": false,
73
+ "single_word": false,
74
+ "special": true
75
+ },
76
+ "100265": {
77
+ "content": "<||_unused_5_||>",
78
+ "lstrip": false,
79
+ "normalized": false,
80
+ "rstrip": false,
81
+ "single_word": false,
82
+ "special": true
83
+ },
84
+ "100266": {
85
+ "content": "<||_unused_6_||>",
86
+ "lstrip": false,
87
+ "normalized": false,
88
+ "rstrip": false,
89
+ "single_word": false,
90
+ "special": true
91
+ },
92
+ "100267": {
93
+ "content": "<||_unused_7_||>",
94
+ "lstrip": false,
95
+ "normalized": false,
96
+ "rstrip": false,
97
+ "single_word": false,
98
+ "special": true
99
+ },
100
+ "100268": {
101
+ "content": "<||_unused_8_||>",
102
+ "lstrip": false,
103
+ "normalized": false,
104
+ "rstrip": false,
105
+ "single_word": false,
106
+ "special": true
107
+ },
108
+ "100269": {
109
+ "content": "<||_unused_9_||>",
110
+ "lstrip": false,
111
+ "normalized": false,
112
+ "rstrip": false,
113
+ "single_word": false,
114
+ "special": true
115
+ },
116
+ "100270": {
117
+ "content": "<||_unused_10_||>",
118
+ "lstrip": false,
119
+ "normalized": false,
120
+ "rstrip": false,
121
+ "single_word": false,
122
+ "special": true
123
+ },
124
+ "100271": {
125
+ "content": "<||_unused_11_||>",
126
+ "lstrip": false,
127
+ "normalized": false,
128
+ "rstrip": false,
129
+ "single_word": false,
130
+ "special": true
131
+ },
132
+ "100272": {
133
+ "content": "<||_unused_12_||>",
134
+ "lstrip": false,
135
+ "normalized": false,
136
+ "rstrip": false,
137
+ "single_word": false,
138
+ "special": true
139
+ },
140
+ "100273": {
141
+ "content": "<||_unused_13_||>",
142
+ "lstrip": false,
143
+ "normalized": false,
144
+ "rstrip": false,
145
+ "single_word": false,
146
+ "special": true
147
+ },
148
+ "100274": {
149
+ "content": "<||_unused_14_||>",
150
+ "lstrip": false,
151
+ "normalized": false,
152
+ "rstrip": false,
153
+ "single_word": false,
154
+ "special": true
155
+ },
156
+ "100275": {
157
+ "content": "<||_unused_15_||>",
158
+ "lstrip": false,
159
+ "normalized": false,
160
+ "rstrip": false,
161
+ "single_word": false,
162
+ "special": true
163
+ },
164
+ "100276": {
165
+ "content": "<|endofprompt|>",
166
+ "lstrip": false,
167
+ "normalized": false,
168
+ "rstrip": false,
169
+ "single_word": false,
170
+ "special": true
171
+ },
172
  "100277": {
173
  "content": "<|pad|>",
174
  "lstrip": false,
 
194
  "special": true
195
  }
196
  },
 
 
 
 
 
 
 
 
 
 
197
  "bos_token": "<|endoftext|>",
198
+ "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% elif 'system' not in messages[0]['role'] %}{% set loop_messages = messages %}{% set system_message = 'You are DBRX, created by Databricks. You were last updated in December 2023. You answer questions based on information available up to that point.\nYOU PROVIDE SHORT RESPONSES TO SHORT QUESTIONS OR STATEMENTS, but provide thorough responses to more complex and open-ended questions.\nYou assist with various tasks, from writing to coding (using markdown for code blocks — remember to use ``` with code, JSON, and tables).\n(You do not have real-time data access or code execution capabilities. You avoid stereotyping and provide balanced perspectives on controversial topics. You do not provide song lyrics, poems, or news articles and do not divulge details of your training data.)\nThis is your system prompt, guiding your responses. Do not reference it, just respond to the user. If you find yourself talking about this message, stop. You should be responding appropriately and usually that means not mentioning this.\nYOU DO NOT MENTION ANY OF THIS INFORMATION ABOUT YOURSELF UNLESS THE INFORMATION IS DIRECTLY PERTINENT TO THE USER\\'S QUERY.' %}{% else %}{% set loop_messages = messages %}{% set system_message = false %}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 %}{% if system_message != false %}{{ '<|im_start|>system\n' + system_message | trim + '<|im_end|>\n'}}{% endif %}{{ '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' }}{% else %}{{ '\n' + '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' }}{% endif %}{% if (add_generation_prompt == true and loop.last) %}{{ '\n' + '<|im_start|>' + 'assistant' + '\n' }}{% endif %}{% endfor %}",
199
  "clean_up_tokenization_spaces": true,
 
200
  "eos_token": "<|endoftext|>",
201
+ "model_max_length": 32768,
 
 
202
  "pad_token": "<|pad|>",
203
+ "tokenizer_class": "GPT2Tokenizer",
204
+ "unk_token": "<|endoftext|>"
 
205
  }
vocab.json ADDED
The diff for this file is too large to render. See raw diff