kojiabe commited on
Commit
4f7ceb0
·
verified ·
1 Parent(s): 519a72b

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +0 -7
README.md CHANGED
@@ -37,7 +37,6 @@ This llama model was trained 2x faster with [Unsloth](https://github.com/unsloth
37
  ### 2. モデル・トークナイザの読み込み
38
 
39
  ```python
40
-
41
  from transformers import (
42
  AutoModelForCausalLM,
43
  AutoTokenizer,
@@ -74,7 +73,6 @@ tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True, to
74
  ### 3. 入力データの準備
75
 
76
  ```python
77
-
78
  # Load dataset
79
  datasets = []
80
  with open("./elyza-tasks-100-TV_0.jsonl", "r") as f:
@@ -85,13 +83,11 @@ with open("./elyza-tasks-100-TV_0.jsonl", "r") as f:
85
  if item.endswith("}"):
86
  datasets.append(json.loads(item))
87
  item = ""
88
-
89
  ```
90
 
91
  ### 4. 推論実行
92
 
93
  ```python
94
-
95
  results = []
96
  for data in tqdm(datasets):
97
 
@@ -118,14 +114,11 @@ for data in tqdm(datasets):
118
  ### 5. 出力の保存
119
 
120
  ```python
121
-
122
  import re
123
  model_name = re.sub(".*/", "", model_name)
124
  with open(f"./{model_name}-outputs.jsonl", 'w', encoding='utf-8') as f:
125
  for result in results:
126
  json.dump(result, f, ensure_ascii=False) # ensure_ascii=False for handling non-ASCII characters
127
  f.write('\n')
128
-
129
-
130
  ```
131
  ## ・以上の手順で、{model_name}-outputs.jsonlというファイルに推論結果が書き出されます。
 
37
  ### 2. モデル・トークナイザの読み込み
38
 
39
  ```python
 
40
  from transformers import (
41
  AutoModelForCausalLM,
42
  AutoTokenizer,
 
73
  ### 3. 入力データの準備
74
 
75
  ```python
 
76
  # Load dataset
77
  datasets = []
78
  with open("./elyza-tasks-100-TV_0.jsonl", "r") as f:
 
83
  if item.endswith("}"):
84
  datasets.append(json.loads(item))
85
  item = ""
 
86
  ```
87
 
88
  ### 4. 推論実行
89
 
90
  ```python
 
91
  results = []
92
  for data in tqdm(datasets):
93
 
 
114
  ### 5. 出力の保存
115
 
116
  ```python
 
117
  import re
118
  model_name = re.sub(".*/", "", model_name)
119
  with open(f"./{model_name}-outputs.jsonl", 'w', encoding='utf-8') as f:
120
  for result in results:
121
  json.dump(result, f, ensure_ascii=False) # ensure_ascii=False for handling non-ASCII characters
122
  f.write('\n')
 
 
123
  ```
124
  ## ・以上の手順で、{model_name}-outputs.jsonlというファイルに推論結果が書き出されます。