<jupyter_start><jupyter_text>Subset to frequent concepts - for testing<jupyter_code>MIN_FREQ = 300
for p in data['projects']:
    for d in p['documents']:
        anns = []
        for a in d['annotations']:
            if cnts[a['cui']] > MIN_FREQ:
                anns.append(a)
        d['annotations'] = anns
DATA_PATH = BASE_PATH + "data/medmentions/medmentions_only_above_300.json"
json.dump(data, open(DATA_PATH, 'w'))
dataset = datasets.load_dataset(os.path.abspath(medcat_ner.__file__), 
                                data_files=DATA_PATH, 
                                split=datasets.Split.TRAIN,
                                cache_dir='/tmp/')
dataset
hf_tokenizer = AutoTokenizer.from_pretrained("emilyalsentzer/Bio_ClinicalBERT")
id2type = {}
for i in range(hf_tokenizer.vocab_size):
    id2type[i] = 'sub' if hf_tokenizer.convert_ids_to_tokens(i).startswith("##") else 'start'
tokenizer = TokenizerNER(hf_tokenizer, id2type=id2type)
encoded_dataset = dataset.map(
        lambda examples: tokenizer.encode(examples, ignore_subwords=True),
        batched=True,
        remove_columns=['ent_cuis', 'ent_ends', 'ent_starts', 'text'])
encoded_dataset
model = AutoModelForTokenClassification.from_pretrained("emilyalsentzer/Bio_ClinicalBERT", num_labels=len(tokenizer.label_map))
encoded_dataset = encoded_dataset.train_test_split(test_size = 0.2)
collate_fn = CollateAndPadNER(hf_tokenizer.pad_token_id)
training_args = TrainingArguments(
    output_dir='./results',          # output directory
    num_train_epochs=10,              # total number of training epochs
    per_device_train_batch_size=4,  # batch size per device during training
    per_device_eval_batch_size=4,   # batch size for evaluation
    weight_decay=0.1431478776404838,               # strength of weight decay
    logging_dir='./logs',            # directory for storing logs
    logging_steps=200,
    eval_steps=500,
    warmup_steps=15,
    learning_rate= 4.4670352057797207e-05,
    eval_accumulation_steps=1,
    do_eval=True,
    evaluation_strategy='steps',
    load_best_model_at_end=True,
)
from sklearn.metrics import classification_report
def metrics(p):
    preds = np.argmax(p.predictions, axis=2)
    # Ignore predictions where label == -100, padding
    preds[np.where(p.label_ids == -100)] = -100
    print(classification_report(np.reshape(p.label_ids, -1), np.reshape(preds, -1)))
    return {'none': 0}
trainer = Trainer(
    model=model,                         
    args=training_args,                 
    train_dataset=encoded_dataset['train'],       
    eval_dataset=encoded_dataset['test'],     
    compute_metrics=metrics,
    data_collator=collate_fn,
    tokenizer=None
)
trainer.train()
p = trainer.predict(encoded_dataset['test'])
preds = np.argmax(p.predictions, axis=2)
# Ignore predictions where label == -100, padding
preds[np.where(p.label_ids == -100)] = -100
report = classification_report(np.reshape(p.label_ids, -1), np.reshape(preds, -1), output_dict=True)
r_label_map = {v:k for k,v in tokenizer.label_map.items()}
for key in report.keys():
    if key.isdigit():
        cui = r_label_map.get(int(key), key)
    else:
        cui = key
    
    name = cdb.get_name(cui)
    print(name)
    print(report[key])
    print()<jupyter_output><empty_output><jupyter_text>Test MedCAT on the same data<jupyter_code>data = json.load(open(DATA_PATH))
cnts = {}
for p in data['projects']:
    for d in p['documents']:
        for a in d['annotations']:
            cnts[a['cui']] = cnts.get(a['cui'], 0) + 1
cnts
cat.config.linking['filters']['cuis'] = set(cnts.keys())
cat.config.linking['similarity_threshold'] = 0.25
# Print stats before training
fp, fn, tp, p, r, f1, cui_counts, examples = cat._print_stats(data)
np.average(list(f1.values()))
np.average(list(p.values()))
np.average(list(r.values()))
fp, fn, tp, p, r, f1, cui_counts, examples = cat.train_supervised(data_path=DATA_PATH, print_stats=1, nepochs=5, test_size=0.2, devalue_others=True,
                         train_from_false_positives=True)
np.average(list(f1.values()))
np.average(list(p.values()))
np.average(list(r.values()))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12855079441502168, 'fraction_numerical': 0.023832450649975925, 'mean_word_length': 3.7002262443438916, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '456341', 'license': 'MIT', 'n_tokens_mistral': 1499, 'n_tokens_neox': 1425, 'n_words': 255, 'path': 'notebooks/BERT for NER.ipynb', 'repo_name': 'tomolopolis/MedCAT-1', 'size': '4154'}