xu-song commited on
Commit
5db13e0
1 Parent(s): f0f84b2

add amber and crystal_coder

Browse files
vocab/__init__.py CHANGED
@@ -135,7 +135,8 @@ all_tokenizers = [
135
  ("gpt_4", "", "tiktoken"),
136
 
137
  # 未分类
138
-
 
139
  ("mistral_7b",),
140
  ("mixtral_8_7b",),
141
 
 
135
  ("gpt_4", "", "tiktoken"),
136
 
137
  # 未分类
138
+ ("amber", ""),
139
+ ("crystal_coder", ""),
140
  ("mistral_7b",),
141
  ("mixtral_8_7b",),
142
 
vocab/amber/__init__.py ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ from transformers import LlamaTokenizer
2
+
3
+ tokenizer = LlamaTokenizer.from_pretrained("LLM360/Amber", revision="ckpt_356")
vocab/crystal_coder/__init__.py ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ from transformers import AutoTokenizer
2
+
3
+ tokenizer = AutoTokenizer.from_pretrained(
4
+ "LLM360/CrystalCoder",
5
+ revision="CrystalCoder_phase1_checkpoint_055500",
6
+ trust_remote_code=True
7
+ )