Update README.md

README.md

@@ -78,6 +78,8 @@ vLLM aslo supports OpenAI-compatible serving. See the [documentation](https://do
 
 This model was created by applying [LLM Compressor with calibration samples from UltraChat](https://github.com/vllm-project/llm-compressor/blob/main/examples/quantization_w4a4_fp4/llama3_example.py), as presented in the code snipet below.
 
+<details>
+  
 ```python
 from datasets import load_dataset
 from transformers import AutoModelForCausalLM, AutoTokenizer
@@ -158,10 +160,11 @@ model.save_pretrained(SAVE_DIR, save_compressed=True)
 tokenizer.save_pretrained(SAVE_DIR)
 
 ```
+</details>
 
 ## Evaluation
 
-This model was evaluated on the well-known OpenLLM v1, OpenLLM v2, HumanEval, and HumanEval_64 benchmarks. All evaluations were conducted using [lm-evaluation-harness](https://github.com/neuralmagic/lm-evaluation-harness).
+This model was evaluated on the well-known OpenLLM v1, OpenLLM v2 and HumanEval_64 benchmarks using [lm-evaluation-harness](https://github.com/neuralmagic/lm-evaluation-harness). The Reasoning evals were done using [ligheval](https://github.com/neuralmagic/lighteval).
 
 ### Accuracy
 
@@ -176,114 +179,126 @@ This model was evaluated on the well-known OpenLLM v1, OpenLLM v2, HumanEval, an
     </tr>
   </thead>
   <tbody>
+    <!-- OpenLLM V1 (Core) -->
     <tr>
       <td rowspan="7"><b>OpenLLM V1</b></td>
+      <td>arc_challenge</td>
+      <td>70.65</td>
+      <td>70.22</td>
+      <td>99.39</td>
+    </tr>
+    <tr>
+      <td>gsm8k</td>
+      <td>74.15</td>
+      <td>74.68</td>
+      <td>100.71</td>
+    </tr>
+    <tr>
+      <td>hellaswag</td>
+      <td>84.00</td>
+      <td>83.33</td>
+      <td>99.20</td>
+    </tr>
+    <tr>
       <td>mmlu</td>
-      <td></td>
-      <td></td>
-      <td></td>
+      <td>81.84</td>
+      <td>81.23</td>
+      <td>99.25</td>
+    </tr>
+    <tr>
+      <td>truthfulqa_mc2</td>
+      <td>59.36</td>
+      <td>58.92</td>
+      <td>99.26</td>
+    </tr>
+    <tr>
+      <td>winogrande</td>
+      <td>75.93</td>
+      <td>76.80</td>
+      <td>101.15</td>
+    </tr>
+    <tr>
+      <td><b>Average</b></td>
+      <td><b>74.32</b></td>
+      <td><b>74.20</b></td>
+      <td><b>99.83</b></td>
+    </tr>
+    <tr>
+      <td rowspan="7"><b>OpenLLM V2</b></td>
+      <td>BBH (3-shot)</td>
+      <td>62.35</td>
+      <td>60.72</td>
+      <td>97.39</td>
+    </tr>
+    <tr>
+      <td>MMLU-Pro (5-shot)</td>
+      <td>54.39</td>
+      <td>51.13</td>
+      <td>94.01</td>
+    </tr>
+    <tr>
+      <td>MuSR (0-shot)</td>
+      <td>39.29</td>
+      <td>41.01</td>
+      <td>104.38</td>
+    </tr>
+    <tr>
+      <td>IFEval (0-shot)</td>
+      <td>88.97</td>
+      <td>87.29</td>
+      <td>98.11</td>
+    </tr>
+    <tr>
+      <td>GPQA (0-shot)</td>
+      <td>30.12</td>
+      <td>30.29</td>
+      <td>100.56</td>
+    </tr>
+    <tr>
+      <td>Math-|v|-5 (4-shot)</td>
+      <td>58.99</td>
+      <td>56.27</td>
+      <td>95.39</td>
+    </tr>
+    <tr>
+      <td><b>Average</b></td>
+      <td><b>55.69</b></td>
+      <td><b>54.45</b></td>
+      <td><b>97.79</b></td>
+    </tr>
+    <tr>
+      <td><b>Coding</b></td>
+      <td>HumanEval_64 pass@2</td>
+      <td>90.14</td>
+      <td>90.40</td>
+      <td>100.29</td>
+    </tr>
+    <tr>
+      <td rowspan="4"><b>Reasoning</b></td>
+      <td>AIME24 (0-shot)</td>
+      <td>75.86</td>
+      <td>68.97</td>
+      <td>90.93</td>
+    </tr>
+    <tr>
+      <td>AIME25 (0-shot)</td>
+      <td>72.41</td>
+      <td>65.52</td>
+      <td>90.52</td>
     </tr>
-  <tr>
-    <td>MMLU</td>
-    <td></td>
-    <td></td>
-    <td></td>
-  </tr>
-  <tr>
-    <td>ARC Challenge (0-shot)</td>
-    <td></td>
-    <td></td>
-    <td></td>
-  </tr>
-  <tr>
-    <td>GSM8K (8-shot, strict-match)</td>
-    <td></td>
-    <td></td>
-    <td></td>
-  </tr>
-  <tr>
-    <td>Hellaswag (10-shot)</td>
-    <td></td>
-    <td></td>
-    <td></td>
-  </tr>
-  <tr>
-    <td>Winogrande (5-shot)</td>
-    <td></td>
-    <td></td>
-    <td></td>
-  </tr>
-  <tr>
-    <td>TruthfulQA (0-shot, mc2)</td>
-    <td></td>
-    <td></td>
-    <td></td>
-  </tr>
-  <tr>
-    <td><b>Average</b></td>
-    <td><b></b></td>
-    <td><b></b></td>
-    <td><b>%</b></td>
-  </tr>
-  <tr>
-    <td rowspan="7"><b>OpenLLM V2</b></td>
-    <td>MMLU-Pro (5-shot)</td>
-    <td></td>
-    <td></td>
-    <td></td>
-  </tr>
-  <tr>
-    <td>IFEval (0-shot)</td>
-    <td></td>
-    <td></td>
-    <td></td>
-  </tr>
-  <tr>
-    <td>BBH (3-shot)</td>
-    <td></td>
-    <td></td>
-    <td></td>
-  </tr>
-  <tr>
-    <td>Math-|v|-5 (4-shot)</td>
-    <td></td>
-    <td></td>
-    <td></td>
-  </tr>
-  <tr>
-    <td>GPQA (0-shot)</td>
-    <td></td>
-    <td></td>
-    <td></td>
-  </tr>
-  <tr>
-    <td>MuSR (0-shot)</td>
-    <td></td>
-    <td></td>
-    <td></td>
-  </tr>
-  <tr>
-    <td><b>Average</b></td>
-    <td><b></b></td>
-    <td><b></b></td>
-    <td><b>%</b></td>
-  </tr>
-
-  <tr>
-    <td><b>Coding</b></td>
-    <td>HumanEval pass@1</td>
-    <td></td>
-    <td></td>
-    <td></td>
-  </tr>
-  <tr>
-    <td></td>
-    <td>HumanEval_64 pass@2</td>
-    <td></td>
-    <td></td>
-    <td></td>
-  </tr>
- </tbody>
+    <tr>
+      <td>GPQA (Diamond, 0-shot)</td>
+      <td>62.94</td>
+      <td>64.47</td>
+      <td>102.43</td>
+    </tr>
+    <tr>
+      <td><b>Average</b></td>
+      <td><b>70.40</b></td>
+      <td><b>66.32</b></td>
+      <td><b>94.21</b></td>
+    </tr>
+  </tbody>
 </table>
 
 
@@ -291,6 +306,8 @@ This model was evaluated on the well-known OpenLLM v1, OpenLLM v2, HumanEval, an
 
 The results were obtained using the following commands:
 
+<details>
+
 #### OpenLLM v1
 ```
 lm_eval \
@@ -314,22 +331,42 @@ lm_eval \
   --batch_size auto
 ```
 
-#### HumanEval and HumanEval_64
+#### HumanEval_64
 ```
 lm_eval \
   --model vllm \
   --model_args pretrained="RedHatAI/Qwen3-32B-NVFP4",dtype=auto,max_model_len=4096,tensor_parallel_size=2,enable_chunked_prefill=True,enforce_eager=True\
   --apply_chat_template \
   --fewshot_as_multiturn \
-  --tasks humaneval_instruct \
+  --tasks humaneval_64_instruct \
   --batch_size auto
+```
 
+#### LightEval 
 
-lm_eval \
-  --model vllm \
-  --model_args pretrained="RedHatAI/Qwen3-32B-NVFP4",dtype=auto,max_model_len=4096,tensor_parallel_size=2,enable_chunked_prefill=True,enforce_eager=True\
-  --apply_chat_template \
-  --fewshot_as_multiturn \
-  --tasks humaneval_64_instruct \
-  --batch_size auto
-```
+```
+# --- model_args.yaml ---
+cat > model_args.yaml <<'YAML'
+model_parameters:
+  model_name: "RedHatAI/Qwen3-8B-NVFP4"
+  dtype: auto
+  gpu_memory_utilization: 0.9
+  tensor_parallel_size: 2
+  max_model_length: 40960
+  generation_parameters:
+    seed: 42
+    temperature: 0.6
+    top_k: 20
+    top_p: 0.95
+    min_p: 0.0
+    max_new_tokens: 32768
+YAML
+
+lighteval vllm model_args.yaml \
+  "lighteval|aime24|0,lighteval|aime25|0,lighteval|gpqa:diamond|0" \
+  --max-samples -1 \
+  --output-dir out_dir
+
+```
+
+</details>