update
Browse files
README.md
CHANGED
@@ -27,9 +27,7 @@ Download your tokenizer, model checkpoints, and optionally the training logs (`e
|
|
27 |
Optionally, test model using the MLM task:
|
28 |
```sh
|
29 |
pip install pya0
|
30 |
-
python test.py
|
31 |
-
--model_name_or_path ./ckpt/to/tokenizer \
|
32 |
-
--tokenizer_name_or_path ./ckpt/to/tokenizer
|
33 |
```
|
34 |
> **Note**
|
35 |
> Modify the test examples in `test.txt` to play with it.
|
|
|
27 |
Optionally, test model using the MLM task:
|
28 |
```sh
|
29 |
pip install pya0
|
30 |
+
python test.py ./ckpt/to/tokenizer ./ckpt/to/model
|
|
|
|
|
31 |
```
|
32 |
> **Note**
|
33 |
> Modify the test examples in `test.txt` to play with it.
|
test.py
CHANGED
@@ -25,7 +25,7 @@ def classifier_hook(tokenizer, tokens, topk, module, inputs, outputs):
|
|
25 |
str(tokenizer.convert_ids_to_tokens(top_cands)))
|
26 |
|
27 |
|
28 |
-
def test(
|
29 |
|
30 |
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path)
|
31 |
model = AutoModelForPreTraining.from_pretrained(model_name_or_path,
|
|
|
25 |
str(tokenizer.convert_ids_to_tokens(top_cands)))
|
26 |
|
27 |
|
28 |
+
def test(tokenizer_name_or_path, model_name_or_path, test_file='test.txt'):
|
29 |
|
30 |
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path)
|
31 |
model = AutoModelForPreTraining.from_pretrained(model_name_or_path,
|