File size: 1,425 Bytes
14e4843 d6d7ec6 14e4843 d6d7ec6 14e4843 d6d7ec6 14e4843 d6d7ec6 14e4843 d6d7ec6 14e4843 d6d7ec6 14e4843 d6d7ec6 14e4843 d6d7ec6 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 |
#!/usr/bin/env python
import os
import fnmatch
import json
from huggingface_hub import HfApi
def find_json_files(directory):
matches = []
for root, dirnames, filenames in os.walk(directory):
for filename in fnmatch.filter(filenames, "*.json"):
matches.append(os.path.join(root, filename))
return matches
directory_path = "/Users/pasquale/workspace/eval/requests"
json_files = find_json_files(directory_path)
api = HfApi()
model_lst = api.list_models()
model_lst = [m for m in model_lst]
id_to_model = {m.id: m for m in model_lst}
for path in json_files:
with open(path, "r") as fr:
data = json.load(fr)
model_id = data["model"]
if model_id in id_to_model:
model = id_to_model[model_id]
to_overwrite = False
is_finetuned = any(tag.startswith("base_model:") for tag in id_to_model[data["model"]].tags)
if is_finetuned:
data["model_type"] = "fine-tuned"
to_overwrite = True
is_instruction_tuned = ("nstruct" in model_id) or ("chat" in model_id)
if is_instruction_tuned:
data["model_type"] = "instruction-tuned"
to_overwrite = True
if to_overwrite is True:
with open(path, "w") as fw:
json.dump(data, fw)
else:
print(f"Model {model_id} not found")
|