Update README.md
Browse files
README.md
CHANGED
@@ -37,7 +37,6 @@ This llama model was trained 2x faster with [Unsloth](https://github.com/unsloth
|
|
37 |
### 2. モデル・トークナイザの読み込み
|
38 |
|
39 |
```python
|
40 |
-
|
41 |
from transformers import (
|
42 |
AutoModelForCausalLM,
|
43 |
AutoTokenizer,
|
@@ -74,7 +73,6 @@ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True, to
|
|
74 |
### 3. 入力データの準備
|
75 |
|
76 |
```python
|
77 |
-
|
78 |
# Load dataset
|
79 |
datasets = []
|
80 |
with open("./elyza-tasks-100-TV_0.jsonl", "r") as f:
|
@@ -85,13 +83,11 @@ with open("./elyza-tasks-100-TV_0.jsonl", "r") as f:
|
|
85 |
if item.endswith("}"):
|
86 |
datasets.append(json.loads(item))
|
87 |
item = ""
|
88 |
-
|
89 |
```
|
90 |
|
91 |
### 4. 推論実行
|
92 |
|
93 |
```python
|
94 |
-
|
95 |
results = []
|
96 |
for data in tqdm(datasets):
|
97 |
|
@@ -118,14 +114,11 @@ for data in tqdm(datasets):
|
|
118 |
### 5. 出力の保存
|
119 |
|
120 |
```python
|
121 |
-
|
122 |
import re
|
123 |
model_name = re.sub(".*/", "", model_name)
|
124 |
with open(f"./{model_name}-outputs.jsonl", 'w', encoding='utf-8') as f:
|
125 |
for result in results:
|
126 |
json.dump(result, f, ensure_ascii=False) # ensure_ascii=False for handling non-ASCII characters
|
127 |
f.write('\n')
|
128 |
-
|
129 |
-
|
130 |
```
|
131 |
## ・以上の手順で、{model_name}-outputs.jsonlというファイルに推論結果が書き出されます。
|
|
|
37 |
### 2. モデル・トークナイザの読み込み
|
38 |
|
39 |
```python
|
|
|
40 |
from transformers import (
|
41 |
AutoModelForCausalLM,
|
42 |
AutoTokenizer,
|
|
|
73 |
### 3. 入力データの準備
|
74 |
|
75 |
```python
|
|
|
76 |
# Load dataset
|
77 |
datasets = []
|
78 |
with open("./elyza-tasks-100-TV_0.jsonl", "r") as f:
|
|
|
83 |
if item.endswith("}"):
|
84 |
datasets.append(json.loads(item))
|
85 |
item = ""
|
|
|
86 |
```
|
87 |
|
88 |
### 4. 推論実行
|
89 |
|
90 |
```python
|
|
|
91 |
results = []
|
92 |
for data in tqdm(datasets):
|
93 |
|
|
|
114 |
### 5. 出力の保存
|
115 |
|
116 |
```python
|
|
|
117 |
import re
|
118 |
model_name = re.sub(".*/", "", model_name)
|
119 |
with open(f"./{model_name}-outputs.jsonl", 'w', encoding='utf-8') as f:
|
120 |
for result in results:
|
121 |
json.dump(result, f, ensure_ascii=False) # ensure_ascii=False for handling non-ASCII characters
|
122 |
f.write('\n')
|
|
|
|
|
123 |
```
|
124 |
## ・以上の手順で、{model_name}-outputs.jsonlというファイルに推論結果が書き出されます。
|