Fix missing PLU Results

results/zero-shot/aya-23-8b.json

@@ -83,30 +83,6 @@
             "exact_match": 0.2062780269058296,
             "f1": 0.4653972244152745
         },
-        {
-            "name": "turkish_plu_goal_inference",
-            "task": "multiple_choice",
-            "acc": 0.3918757467144564,
-            "acc_norm": 0.3859020310633214
-        },
-        {
-            "name": "turkish_plu_next_event_prediction",
-            "task": "multiple_choice",
-            "acc": 0.4687022900763359,
-            "acc_norm": 0.5374045801526718
-        },
-        {
-            "name": "turkish_plu_step_inference",
-            "task": "multiple_choice",
-            "acc": 0.33986928104575165,
-            "acc_norm": 0.45098039215686275
-        },
-        {
-            "name": "turkish_plu_step_ordering",
-            "task": "multiple_choice",
-            "acc": 0.6180215475024485,
-            "acc_norm": 0.6180215475024485
-        },
         {
             "name": "xcopa_tr",
             "task": "multiple_choice",
@@ -187,6 +163,36 @@
             "task": "multiple_choice",
             "acc": 0.5857142857142857,
             "acc_norm": 0.5857142857142857
+        },
+        {
+            "name": "turkish_plu",
+            "task": "multiple_choice",
+            "acc": 0.4928,
+            "acc_norm": 0.40416
+        },
+        {
+            "name": "turkish_plu_goal_inference",
+            "task": "multiple_choice",
+            "acc": 0.42054958183990443,
+            "acc_norm": 0.2724014336917563
+        },
+        {
+            "name": "turkish_plu_next_event_prediction",
+            "task": "multiple_choice",
+            "acc": 0.48854961832061067,
+            "acc_norm": 0.2732824427480916
+        },
+        {
+            "name": "turkish_plu_step_inference",
+            "task": "multiple_choice",
+            "acc": 0.37254901960784315,
+            "acc_norm": 0.35294117647058826
+        },
+        {
+            "name": "turkish_plu_step_ordering",
+            "task": "multiple_choice",
+            "acc": 0.6268364348677767,
+            "acc_norm": 0.6268364348677767
         }
     ]
 }

results/zero-shot/aya-expanse-8b.json

@@ -93,30 +93,6 @@
             "exact_match": 0.13452914798206278,
             "f1": 0.435087842533856
         },
-        {
-            "name": "turkish_plu_goal_inference",
-            "task": "multiple_choice",
-            "acc": 0.4062126642771804,
-            "acc_norm": 0.3930704898446834
-        },
-        {
-            "name": "turkish_plu_next_event_prediction",
-            "task": "multiple_choice",
-            "acc": 0.4900763358778626,
-            "acc_norm": 0.5465648854961832
-        },
-        {
-            "name": "turkish_plu_step_inference",
-            "task": "multiple_choice",
-            "acc": 0.3464052287581699,
-            "acc_norm": 0.4395424836601307
-        },
-        {
-            "name": "turkish_plu_step_ordering",
-            "task": "multiple_choice",
-            "acc": 0.5935357492654261,
-            "acc_norm": 0.5935357492654261
-        },
         {
             "name": "wiki_lingua_tr",
             "task": "summarization",
@@ -185,6 +161,36 @@
             "task": "multiple_choice",
             "acc": 0.5428571428571428,
             "acc_norm": 0.5428571428571428
+        },
+        {
+            "name": "turkish_plu",
+            "task": "multiple_choice",
+            "acc": 0.50208,
+            "acc_norm": 0.40704
+        },
+        {
+            "name": "turkish_plu_goal_inference",
+            "task": "multiple_choice",
+            "acc": 0.43010752688172044,
+            "acc_norm": 0.27956989247311825
+        },
+        {
+            "name": "turkish_plu_next_event_prediction",
+            "task": "multiple_choice",
+            "acc": 0.5114503816793893,
+            "acc_norm": 0.27938931297709924
+        },
+        {
+            "name": "turkish_plu_step_inference",
+            "task": "multiple_choice",
+            "acc": 0.4035947712418301,
+            "acc_norm": 0.37254901960784315
+        },
+        {
+            "name": "turkish_plu_step_ordering",
+            "task": "multiple_choice",
+            "acc": 0.614103819784525,
+            "acc_norm": 0.614103819784525
         }
     ]
 }

results/zero-shot/llama-3-8b-instruct.json

@@ -82,30 +82,6 @@
             "exact_match": 0.1289237668161435,
             "f1": 0.4134057883004977
         },
-        {
-            "name": "turkish_plu_goal_inference",
-            "task": "multiple_choice",
-            "acc": 0.38829151732377537,
-            "acc_norm": 0.43130227001194743
-        },
-        {
-            "name": "turkish_plu_next_event_prediction",
-            "task": "multiple_choice",
-            "acc": 0.4549618320610687,
-            "acc_norm": 0.517557251908397
-        },
-        {
-            "name": "turkish_plu_step_inference",
-            "task": "multiple_choice",
-            "acc": 0.3137254901960784,
-            "acc_norm": 0.44281045751633985
-        },
-        {
-            "name": "turkish_plu_step_ordering",
-            "task": "multiple_choice",
-            "acc": 0.6160626836434868,
-            "acc_norm": 0.6160626836434868
-        },
         {
             "name": "xcopa_tr",
             "task": "multiple_choice",
@@ -186,6 +162,36 @@
             "task": "multiple_choice",
             "acc": 0.6142857142857143,
             "acc_norm": 0.6142857142857143
+        },
+        {
+            "name": "turkish_plu",
+            "task": "multiple_choice",
+            "acc": 0.47136,
+            "acc_norm": 0.4032
+        },
+        {
+            "name": "turkish_plu_goal_inference",
+            "task": "multiple_choice",
+            "acc": 0.3763440860215054,
+            "acc_norm": 0.26642771804062126
+        },
+        {
+            "name": "turkish_plu_next_event_prediction",
+            "task": "multiple_choice",
+            "acc": 0.46564885496183206,
+            "acc_norm": 0.2595419847328244
+        },
+        {
+            "name": "turkish_plu_step_inference",
+            "task": "multiple_choice",
+            "acc": 0.3349673202614379,
+            "acc_norm": 0.35784313725490197
+        },
+        {
+            "name": "turkish_plu_step_ordering",
+            "task": "multiple_choice",
+            "acc": 0.6346718903036239,
+            "acc_norm": 0.6346718903036239
         }
     ]
 }

results/zero-shot/llama-3-8b.json

@@ -81,30 +81,6 @@
             "exact_match": 0.28475336322869954,
             "f1": 0.5013148868557868
         },
-        {
-            "name": "turkish_plu_goal_inference",
-            "task": "multiple_choice",
-            "acc": 0.38948626045400236,
-            "acc_norm": 0.4169653524492234
-        },
-        {
-            "name": "turkish_plu_next_event_prediction",
-            "task": "multiple_choice",
-            "acc": 0.4488549618320611,
-            "acc_norm": 0.5328244274809161
-        },
-        {
-            "name": "turkish_plu_step_inference",
-            "task": "multiple_choice",
-            "acc": 0.32189542483660133,
-            "acc_norm": 0.47058823529411764
-        },
-        {
-            "name": "turkish_plu_step_ordering",
-            "task": "multiple_choice",
-            "acc": 0.6278158667972575,
-            "acc_norm": 0.6278158667972575
-        },
         {
             "name": "xcopa_tr",
             "task": "multiple_choice",
@@ -185,6 +161,36 @@
             "task": "multiple_choice",
             "acc": 0.5428571428571428,
             "acc_norm": 0.5428571428571428
+        },
+        {
+            "name": "turkish_plu",
+            "task": "multiple_choice",
+            "acc": 0.46496,
+            "acc_norm": 0.39616
+        },
+        {
+            "name": "turkish_plu_goal_inference",
+            "task": "multiple_choice",
+            "acc": 0.36917562724014336,
+            "acc_norm": 0.25925925925925924
+        },
+        {
+            "name": "turkish_plu_next_event_prediction",
+            "task": "multiple_choice",
+            "acc": 0.46106870229007635,
+            "acc_norm": 0.26259541984732826
+        },
+        {
+            "name": "turkish_plu_step_inference",
+            "task": "multiple_choice",
+            "acc": 0.3284313725490196,
+            "acc_norm": 0.33986928104575165
+        },
+        {
+            "name": "turkish_plu_step_ordering",
+            "task": "multiple_choice",
+            "acc": 0.6278158667972575,
+            "acc_norm": 0.6278158667972575
         }
     ]
 }

results/zero-shot/llama-3.1-8b-instruct.json

@@ -81,30 +81,6 @@
             "exact_match": 0.23318385650224216,
             "f1": 0.5062272078338648
         },
-        {
-            "name": "turkish_plu_goal_inference",
-            "task": "multiple_choice",
-            "acc": 0.40860215053763443,
-            "acc_norm": 0.45997610513739545
-        },
-        {
-            "name": "turkish_plu_next_event_prediction",
-            "task": "multiple_choice",
-            "acc": 0.4442748091603053,
-            "acc_norm": 0.5419847328244275
-        },
-        {
-            "name": "turkish_plu_step_inference",
-            "task": "multiple_choice",
-            "acc": 0.33169934640522875,
-            "acc_norm": 0.4624183006535948
-        },
-        {
-            "name": "turkish_plu_step_ordering",
-            "task": "multiple_choice",
-            "acc": 0.633692458374143,
-            "acc_norm": 0.633692458374143
-        },
         {
             "name": "xcopa_tr",
             "task": "multiple_choice",
@@ -185,6 +161,36 @@
             "task": "multiple_choice",
             "acc": 0.6428571428571429,
             "acc_norm": 0.6428571428571429
+        },
+        {
+            "name": "turkish_plu",
+            "task": "multiple_choice",
+            "acc": 0.4848,
+            "acc_norm": 0.40896
+        },
+        {
+            "name": "turkish_plu_goal_inference",
+            "task": "multiple_choice",
+            "acc": 0.40860215053763443,
+            "acc_norm": 0.27718040621266427
+        },
+        {
+            "name": "turkish_plu_next_event_prediction",
+            "task": "multiple_choice",
+            "acc": 0.44580152671755724,
+            "acc_norm": 0.2549618320610687
+        },
+        {
+            "name": "turkish_plu_step_inference",
+            "task": "multiple_choice",
+            "acc": 0.3431372549019608,
+            "acc_norm": 0.33986928104575165
+        },
+        {
+            "name": "turkish_plu_step_ordering",
+            "task": "multiple_choice",
+            "acc": 0.6571988246816847,
+            "acc_norm": 0.6571988246816847
         }
     ]
 }

results/zero-shot/llama-3.1-8b.json

@@ -81,30 +81,6 @@
             "exact_match": 0.2757847533632287,
             "f1": 0.5178366277473359
         },
-        {
-            "name": "turkish_plu_goal_inference",
-            "task": "multiple_choice",
-            "acc": 0.4145758661887694,
-            "acc_norm": 0.4324970131421744
-        },
-        {
-            "name": "turkish_plu_next_event_prediction",
-            "task": "multiple_choice",
-            "acc": 0.4488549618320611,
-            "acc_norm": 0.5358778625954198
-        },
-        {
-            "name": "turkish_plu_step_inference",
-            "task": "multiple_choice",
-            "acc": 0.3382352941176471,
-            "acc_norm": 0.4738562091503268
-        },
-        {
-            "name": "turkish_plu_step_ordering",
-            "task": "multiple_choice",
-            "acc": 0.6425073457394711,
-            "acc_norm": 0.6425073457394711
-        },
         {
             "name": "xcopa_tr",
             "task": "multiple_choice",
@@ -185,6 +161,36 @@
             "task": "multiple_choice",
             "acc": 0.5857142857142857,
             "acc_norm": 0.5857142857142857
+        },
+        {
+            "name": "turkish_plu",
+            "task": "multiple_choice",
+            "acc": 0.47552,
+            "acc_norm": 0.39872
+        },
+        {
+            "name": "turkish_plu_goal_inference",
+            "task": "multiple_choice",
+            "acc": 0.38829151732377537,
+            "acc_norm": 0.2628434886499403
+        },
+        {
+            "name": "turkish_plu_next_event_prediction",
+            "task": "multiple_choice",
+            "acc": 0.4595419847328244,
+            "acc_norm": 0.26106870229007634
+        },
+        {
+            "name": "turkish_plu_step_inference",
+            "task": "multiple_choice",
+            "acc": 0.35130718954248363,
+            "acc_norm": 0.3431372549019608
+        },
+        {
+            "name": "turkish_plu_step_ordering",
+            "task": "multiple_choice",
+            "acc": 0.6317335945151812,
+            "acc_norm": 0.6317335945151812
         }
     ]
 }

results/zero-shot/llama-3.2-1b.json

@@ -93,30 +93,6 @@
             "exact_match": 0.06278026905829596,
             "f1": 0.21486130318406463
         },
-        {
-            "name": "turkish_plu_goal_inference",
-            "task": "multiple_choice",
-            "acc": 0.35842293906810035,
-            "acc_norm": 0.4026284348864994
-        },
-        {
-            "name": "turkish_plu_next_event_prediction",
-            "task": "multiple_choice",
-            "acc": 0.3709923664122137,
-            "acc_norm": 0.467175572519084
-        },
-        {
-            "name": "turkish_plu_step_inference",
-            "task": "multiple_choice",
-            "acc": 0.27941176470588236,
-            "acc_norm": 0.41830065359477125
-        },
-        {
-            "name": "turkish_plu_step_ordering",
-            "task": "multiple_choice",
-            "acc": 0.5759059745347699,
-            "acc_norm": 0.5759059745347699
-        },
         {
             "name": "wiki_lingua_tr",
             "task": "summarization",
@@ -217,6 +193,36 @@
             "task": "multiple_choice",
             "acc": 0.5285714285714286,
             "acc_norm": 0.5285714285714286
+        },
+        {
+            "name": "turkish_plu",
+            "task": "multiple_choice",
+            "acc": 0.4208,
+            "acc_norm": 0.368
+        },
+        {
+            "name": "turkish_plu_goal_inference",
+            "task": "multiple_choice",
+            "acc": 0.36200716845878134,
+            "acc_norm": 0.23894862604540024
+        },
+        {
+            "name": "turkish_plu_next_event_prediction",
+            "task": "multiple_choice",
+            "acc": 0.37251908396946565,
+            "acc_norm": 0.2366412213740458
+        },
+        {
+            "name": "turkish_plu_step_inference",
+            "task": "multiple_choice",
+            "acc": 0.29248366013071897,
+            "acc_norm": 0.3366013071895425
+        },
+        {
+            "name": "turkish_plu_step_ordering",
+            "task": "multiple_choice",
+            "acc": 0.5768854064642507,
+            "acc_norm": 0.5768854064642507
         }
     ]
 }

results/zero-shot/llama-3.2-3b-instruct.json

@@ -93,30 +93,6 @@
             "exact_match": 0.18721973094170405,
             "f1": 0.5109898180473623
         },
-        {
-            "name": "turkish_plu_goal_inference",
-            "task": "multiple_choice",
-            "acc": 0.3321385902031063,
-            "acc_norm": 0.3548387096774194
-        },
-        {
-            "name": "turkish_plu_next_event_prediction",
-            "task": "multiple_choice",
-            "acc": 0.3648854961832061,
-            "acc_norm": 0.4488549618320611
-        },
-        {
-            "name": "turkish_plu_step_inference",
-            "task": "multiple_choice",
-            "acc": 0.24183006535947713,
-            "acc_norm": 0.3758169934640523
-        },
-        {
-            "name": "turkish_plu_step_ordering",
-            "task": "multiple_choice",
-            "acc": 0.5710088148873653,
-            "acc_norm": 0.5710088148873653
-        },
         {
             "name": "wiki_lingua_tr",
             "task": "summarization",
@@ -217,6 +193,36 @@
             "task": "multiple_choice",
             "acc": 0.5428571428571428,
             "acc_norm": 0.5428571428571428
+        },
+        {
+            "name": "turkish_plu",
+            "task": "multiple_choice",
+            "acc": 0.44,
+            "acc_norm": 0.3952
+        },
+        {
+            "name": "turkish_plu_goal_inference",
+            "task": "multiple_choice",
+            "acc": 0.3548387096774194,
+            "acc_norm": 0.26045400238948624
+        },
+        {
+            "name": "turkish_plu_next_event_prediction",
+            "task": "multiple_choice",
+            "acc": 0.3938931297709924,
+            "acc_norm": 0.24427480916030533
+        },
+        {
+            "name": "turkish_plu_step_inference",
+            "task": "multiple_choice",
+            "acc": 0.2777777777777778,
+            "acc_norm": 0.3382352941176471
+        },
+        {
+            "name": "turkish_plu_step_ordering",
+            "task": "multiple_choice",
+            "acc": 0.6366307541625857,
+            "acc_norm": 0.6366307541625857
         }
     ]
 }

results/zero-shot/llama-3.2-3b.json

@@ -81,30 +81,6 @@
             "exact_match": 0.21188340807174888,
             "f1": 0.4583574684635767
         },
-        {
-            "name": "turkish_plu_goal_inference",
-            "task": "multiple_choice",
-            "acc": 0.3906810035842294,
-            "acc_norm": 0.3906810035842294
-        },
-        {
-            "name": "turkish_plu_next_event_prediction",
-            "task": "multiple_choice",
-            "acc": 0.4122137404580153,
-            "acc_norm": 0.5389312977099237
-        },
-        {
-            "name": "turkish_plu_step_inference",
-            "task": "multiple_choice",
-            "acc": 0.30718954248366015,
-            "acc_norm": 0.4493464052287582
-        },
-        {
-            "name": "turkish_plu_step_ordering",
-            "task": "multiple_choice",
-            "acc": 0.5974534769833496,
-            "acc_norm": 0.5974534769833496
-        },
         {
             "name": "xcopa_tr",
             "task": "multiple_choice",
@@ -185,6 +161,36 @@
             "task": "multiple_choice",
             "acc": 0.5714285714285714,
             "acc_norm": 0.5714285714285714
+        },
+        {
+            "name": "turkish_plu",
+            "task": "multiple_choice",
+            "acc": 0.45408,
+            "acc_norm": 0.38752
+        },
+        {
+            "name": "turkish_plu_goal_inference",
+            "task": "multiple_choice",
+            "acc": 0.4002389486260454,
+            "acc_norm": 0.25925925925925924
+        },
+        {
+            "name": "turkish_plu_next_event_prediction",
+            "task": "multiple_choice",
+            "acc": 0.43206106870229005,
+            "acc_norm": 0.26106870229007634
+        },
+        {
+            "name": "turkish_plu_step_inference",
+            "task": "multiple_choice",
+            "acc": 0.315359477124183,
+            "acc_norm": 0.35130718954248363
+        },
+        {
+            "name": "turkish_plu_step_ordering",
+            "task": "multiple_choice",
+            "acc": 0.5954946131243879,
+            "acc_norm": 0.5954946131243879
         }
     ]
 }