only 0-shot

src/leaderboard/read_evals.py

@@ -38,6 +38,7 @@ class EvalResult:
             data = json.load(fp)
 
         config = data.get("config")
+        n_shot = data.get("n-shot")
 
         # Precision
         precision = Precision.from_str(config.get("model_dtype"))
@@ -72,7 +73,7 @@ class EvalResult:
             task = task.value
 
             # We average all scores of a given metric (not all metrics are present in all files)
-            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k])
+            accs = np.array([v.get(task.metric, None) for k, v in data["results"].items() if task.benchmark == k and n_shot.get(k, -1) == 0])
             if accs.size == 0 or any([acc is None for acc in accs]):
                 continue