update
Browse files
README.md
CHANGED
|
@@ -27,9 +27,7 @@ Download your tokenizer, model checkpoints, and optionally the training logs (`e
|
|
| 27 |
Optionally, test model using the MLM task:
|
| 28 |
```sh
|
| 29 |
pip install pya0
|
| 30 |
-
python test.py
|
| 31 |
-
--model_name_or_path ./ckpt/to/tokenizer \
|
| 32 |
-
--tokenizer_name_or_path ./ckpt/to/tokenizer
|
| 33 |
```
|
| 34 |
> **Note**
|
| 35 |
> Modify the test examples in `test.txt` to play with it.
|
|
|
|
| 27 |
Optionally, test model using the MLM task:
|
| 28 |
```sh
|
| 29 |
pip install pya0
|
| 30 |
+
python test.py ./ckpt/to/tokenizer ./ckpt/to/model
|
|
|
|
|
|
|
| 31 |
```
|
| 32 |
> **Note**
|
| 33 |
> Modify the test examples in `test.txt` to play with it.
|
test.py
CHANGED
|
@@ -25,7 +25,7 @@ def classifier_hook(tokenizer, tokens, topk, module, inputs, outputs):
|
|
| 25 |
str(tokenizer.convert_ids_to_tokens(top_cands)))
|
| 26 |
|
| 27 |
|
| 28 |
-
def test(
|
| 29 |
|
| 30 |
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path)
|
| 31 |
model = AutoModelForPreTraining.from_pretrained(model_name_or_path,
|
|
|
|
| 25 |
str(tokenizer.convert_ids_to_tokens(top_cands)))
|
| 26 |
|
| 27 |
|
| 28 |
+
def test(tokenizer_name_or_path, model_name_or_path, test_file='test.txt'):
|
| 29 |
|
| 30 |
tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path)
|
| 31 |
model = AutoModelForPreTraining.from_pretrained(model_name_or_path,
|