Update model.py
Browse files
model.py
CHANGED
@@ -35,15 +35,7 @@ def load_embedder_and_tokenizer(name: str) -> Tuple[
|
|
35 |
transformers.PreTrainedTokenizer
|
36 |
]:
|
37 |
if name.startswith("nomic") or (name == "bert-base-uncased"):
|
38 |
-
|
39 |
-
if name.endswith("--from-scratch"):
|
40 |
-
name = name.replace("--from-scratch", "")
|
41 |
-
config = transformers.AutoConfig.from_pretrained(name, trust_remote_code=True)
|
42 |
-
model = NomicBertModel._from_config(config)
|
43 |
-
else:
|
44 |
-
model = NomicBertModel.from_pretrained(
|
45 |
-
name, add_pooling_layer=False
|
46 |
-
)
|
47 |
tokenizer = transformers.AutoTokenizer.from_pretrained(name)
|
48 |
elif name in ["gtr-base", "gtr_base"]:
|
49 |
model = transformers.AutoModel.from_pretrained(
|
|
|
35 |
transformers.PreTrainedTokenizer
|
36 |
]:
|
37 |
if name.startswith("nomic") or (name == "bert-base-uncased"):
|
38 |
+
model = transfromers.AutoModel.from_pretrained(name)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
39 |
tokenizer = transformers.AutoTokenizer.from_pretrained(name)
|
40 |
elif name in ["gtr-base", "gtr_base"]:
|
41 |
model = transformers.AutoModel.from_pretrained(
|