Update README.md

README.md

@@ -62,10 +62,11 @@ from medkit.core.text import TextDocument
 from medkit.text.ner.hf_entity_matcher import HFEntityMatcher
 
 matcher = HFEntityMatcher(model="camila-ud/DrBERT-CASM2")
+
 test_doc = TextDocument("Elle souffre d'asthme mais n'a pas besoin d'Allegra")
+detected_entities = matcher.run([test_doc.raw_segment])
 
-# detect entities in the raw segment
-detected_entities = matcher.run([test_doc.raw_segment]) 
+# show information
 msg = "|".join(f"'{entity.label}':{entity.text}" for entity in detected_entities)
 print(f"Text: '{test_doc.text}'\n{msg}")
 ```
@@ -119,10 +120,18 @@ from medkit.text.metrics.ner import SeqEvalEvaluator
 matcher = HFEntityMatcher(model="camila-ud/DrBERT-CASM2")
 predicted_entities = [matcher.run([doc.raw_segment]) for doc in test_documents]
 
-# define seqeval evaluator
 evaluator  = SeqEvalEvaluator(tagging_scheme="iob2")
 evaluator.compute(test_documents,predicted_entities=predicted_entities)
 ```
+You can use the tokenizer from HF to evaluate by tokens instead of characters 
+```python
+from transformers import AutoTokenizer
+
+tokenizer_drbert = AutoTokenizer.from_pretrained("camila-ud/DrBERT-CASM2", use_fast=True)
+
+evaluator  = SeqEvalEvaluator(tokenizer=tokenizer_drbert,tagging_scheme="iob2")
+evaluator.compute(test_documents,predicted_entities=predicted_entities)
+```
 
 # Citation