Hugging Face's logo

microsoft
/
Phi-3-mini-4k-instruct

Text Generation
Transformers
Safetensors
English
French
phi3
nlp
code
conversational
custom_code
text-generation-inference
Inference Endpoints
Model card Files Community
96
gugarosa commited on
Commit
653ee82
•
1 Parent(s): 4eea1a7

fix(tokenizer): Also updates tokenizer.json to prevent any mismatch.

Browse files
Files changed (1) hide show
  1. tokenizer.json +12 -12
tokenizer.json CHANGED
@@ -26,9 +26,9 @@
26
  "content": "</s>",
27
  "single_word": false,
28
  "lstrip": false,
29
- "rstrip": false,
30
  "normalized": false,
31
- "special": true
32
  },
33
  {
34
  "id": 32000,
@@ -44,7 +44,7 @@
44
  "content": "<|assistant|>",
45
  "single_word": false,
46
  "lstrip": false,
47
- "rstrip": false,
48
  "normalized": false,
49
  "special": true
50
  },
@@ -53,7 +53,7 @@
53
  "content": "<|placeholder1|>",
54
  "single_word": false,
55
  "lstrip": false,
56
- "rstrip": false,
57
  "normalized": false,
58
  "special": true
59
  },
@@ -62,7 +62,7 @@
62
  "content": "<|placeholder2|>",
63
  "single_word": false,
64
  "lstrip": false,
65
- "rstrip": false,
66
  "normalized": false,
67
  "special": true
68
  },
@@ -71,7 +71,7 @@
71
  "content": "<|placeholder3|>",
72
  "single_word": false,
73
  "lstrip": false,
74
- "rstrip": false,
75
  "normalized": false,
76
  "special": true
77
  },
@@ -80,7 +80,7 @@
80
  "content": "<|placeholder4|>",
81
  "single_word": false,
82
  "lstrip": false,
83
- "rstrip": false,
84
  "normalized": false,
85
  "special": true
86
  },
@@ -89,7 +89,7 @@
89
  "content": "<|system|>",
90
  "single_word": false,
91
  "lstrip": false,
92
- "rstrip": false,
93
  "normalized": false,
94
  "special": true
95
  },
@@ -98,7 +98,7 @@
98
  "content": "<|end|>",
99
  "single_word": false,
100
  "lstrip": false,
101
- "rstrip": false,
102
  "normalized": false,
103
  "special": true
104
  },
@@ -107,7 +107,7 @@
107
  "content": "<|placeholder5|>",
108
  "single_word": false,
109
  "lstrip": false,
110
- "rstrip": false,
111
  "normalized": false,
112
  "special": true
113
  },
@@ -116,7 +116,7 @@
116
  "content": "<|placeholder6|>",
117
  "single_word": false,
118
  "lstrip": false,
119
- "rstrip": false,
120
  "normalized": false,
121
  "special": true
122
  },
@@ -125,7 +125,7 @@
125
  "content": "<|user|>",
126
  "single_word": false,
127
  "lstrip": false,
128
- "rstrip": false,
129
  "normalized": false,
130
  "special": true
131
  }
 
26
  "content": "</s>",
27
  "single_word": false,
28
  "lstrip": false,
29
+ "rstrip": true,
30
  "normalized": false,
31
+ "special": false
32
  },
33
  {
34
  "id": 32000,
 
44
  "content": "<|assistant|>",
45
  "single_word": false,
46
  "lstrip": false,
47
+ "rstrip": true,
48
  "normalized": false,
49
  "special": true
50
  },
 
53
  "content": "<|placeholder1|>",
54
  "single_word": false,
55
  "lstrip": false,
56
+ "rstrip": true,
57
  "normalized": false,
58
  "special": true
59
  },
 
62
  "content": "<|placeholder2|>",
63
  "single_word": false,
64
  "lstrip": false,
65
+ "rstrip": true,
66
  "normalized": false,
67
  "special": true
68
  },
 
71
  "content": "<|placeholder3|>",
72
  "single_word": false,
73
  "lstrip": false,
74
+ "rstrip": true,
75
  "normalized": false,
76
  "special": true
77
  },
 
80
  "content": "<|placeholder4|>",
81
  "single_word": false,
82
  "lstrip": false,
83
+ "rstrip": true,
84
  "normalized": false,
85
  "special": true
86
  },
 
89
  "content": "<|system|>",
90
  "single_word": false,
91
  "lstrip": false,
92
+ "rstrip": true,
93
  "normalized": false,
94
  "special": true
95
  },
 
98
  "content": "<|end|>",
99
  "single_word": false,
100
  "lstrip": false,
101
+ "rstrip": true,
102
  "normalized": false,
103
  "special": true
104
  },
 
107
  "content": "<|placeholder5|>",
108
  "single_word": false,
109
  "lstrip": false,
110
+ "rstrip": true,
111
  "normalized": false,
112
  "special": true
113
  },
 
116
  "content": "<|placeholder6|>",
117
  "single_word": false,
118
  "lstrip": false,
119
+ "rstrip": true,
120
  "normalized": false,
121
  "special": true
122
  },
 
125
  "content": "<|user|>",
126
  "single_word": false,
127
  "lstrip": false,
128
+ "rstrip": true,
129
  "normalized": false,
130
  "special": true
131
  }