jeduardogruiz
commited on
Commit
•
33a234a
1
Parent(s):
545871f
Update README.md
Browse files
README.md
CHANGED
@@ -34,6 +34,9 @@ tags:
|
|
34 |
|
35 |
### Tokenization with `mistral-common`
|
36 |
|
|
|
|
|
|
|
37 |
```py
|
38 |
from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
|
39 |
from mistral_common.protocol.instruct.messages import UserMessage
|
|
|
34 |
|
35 |
### Tokenization with `mistral-common`
|
36 |
|
37 |
+
|
38 |
+
// utilizando todos los files del repositorio:
|
39 |
+
|
40 |
```py
|
41 |
from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
|
42 |
from mistral_common.protocol.instruct.messages import UserMessage
|