| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 869 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 869 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 100.00000000 | |
| 3 66.66666667 | |
| 4 50.00000000 | |
| 5 60.00000000 | |
| 6 66.66666667 | |
| 7 71.42857143 | |
| 8 75.00000000 | |
| 9 66.66666667 | |
| 10 70.00000000 | |
| 11 63.63636364 | |
| 12 66.66666667 | |
| 13 61.53846154 | |
| 14 64.28571429 | |
| 15 66.66666667 | |
| 16 68.75000000 | |
| 17 70.58823529 | |
| 18 72.22222222 | |
| 19 68.42105263 | |
| 20 65.00000000 | |
| 21 66.66666667 | |
| 22 68.18181818 | |
| 23 65.21739130 | |
| 24 66.66666667 | |
| 25 64.00000000 | |
| 26 65.38461538 | |
| 27 66.66666667 | |
| 28 64.28571429 | |
| 29 65.51724138 | |
| 30 66.66666667 | |
| 31 64.51612903 | |
| 32 62.50000000 | |
| 33 60.60606061 | |
| 34 58.82352941 | |
| 35 57.14285714 | |
| 36 55.55555556 | |
| 37 56.75675676 | |
| 38 55.26315789 | |
| 39 56.41025641 | |
| 40 57.50000000 | |
| 41 56.09756098 | |
| 42 54.76190476 | |
| 43 53.48837209 | |
| 44 52.27272727 | |
| 45 53.33333333 | |
| 46 54.34782609 | |
| 47 55.31914894 | |
| 48 54.16666667 | |
| 49 55.10204082 | |
| 50 56.00000000 | |
| 51 56.86274510 | |
| 52 57.69230769 | |
| 53 58.49056604 | |
| 54 57.40740741 | |
| 55 56.36363636 | |
| 56 55.35714286 | |
| 57 56.14035088 | |
| 58 56.89655172 | |
| 59 57.62711864 | |
| 60 58.33333333 | |
| 61 59.01639344 | |
| 62 59.67741935 | |
| 63 60.31746032 | |
| 64 60.93750000 | |
| 65 60.00000000 | |
| 66 60.60606061 | |
| 67 61.19402985 | |
| 68 61.76470588 | |
| 69 60.86956522 | |
| 70 60.00000000 | |
| 71 59.15492958 | |
| 72 58.33333333 | |
| 73 58.90410959 | |
| 74 59.45945946 | |
| 75 60.00000000 | |
| 76 59.21052632 | |
| 77 59.74025974 | |
| 78 60.25641026 | |
| 79 59.49367089 | |
| 80 60.00000000 | |
| 81 59.25925926 | |
| 82 59.75609756 | |
| 83 59.03614458 | |
| 84 58.33333333 | |
| 85 58.82352941 | |
| 86 58.13953488 | |
| 87 58.62068966 | |
| 88 59.09090909 | |
| 89 59.55056180 | |
| 90 60.00000000 | |
| 91 60.43956044 | |
| 92 60.86956522 | |
| 93 60.21505376 | |
| 94 59.57446809 | |
| 95 60.00000000 | |
| 96 60.41666667 | |
| 97 59.79381443 | |
| 98 59.18367347 | |
| 99 59.59595960 | |
| 100 59.00000000 | |
| 101 58.41584158 | |
| 102 58.82352941 | |
| 103 58.25242718 | |
| 104 58.65384615 | |
| 105 59.04761905 | |
| 106 58.49056604 | |
| 107 58.87850467 | |
| 108 58.33333333 | |
| 109 57.79816514 | |
| 110 58.18181818 | |
| 111 58.55855856 | |
| 112 58.92857143 | |
| 113 58.40707965 | |
| 114 57.89473684 | |
| 115 58.26086957 | |
| 116 58.62068966 | |
| 117 58.97435897 | |
| 118 59.32203390 | |
| 119 58.82352941 | |
| 120 59.16666667 | |
| 121 58.67768595 | |
| 122 59.01639344 | |
| 123 59.34959350 | |
| 124 58.87096774 | |
| 125 59.20000000 | |
| 126 59.52380952 | |
| 127 59.05511811 | |
| 128 59.37500000 | |
| 129 58.91472868 | |
| 130 58.46153846 | |
| 131 58.77862595 | |
| 132 58.33333333 | |
| 133 58.64661654 | |
| 134 58.20895522 | |
| 135 57.77777778 | |
| 136 57.35294118 | |
| 137 57.66423358 | |
| 138 57.97101449 | |
| 139 58.27338129 | |
| 140 57.85714286 | |
| 141 57.44680851 | |
| 142 57.74647887 | |
| 143 58.04195804 | |
| 144 57.63888889 | |
| 145 57.93103448 | |
| 146 57.53424658 | |
| 147 57.82312925 | |
| 148 57.43243243 | |
| 149 57.71812081 | |
| 150 58.00000000 | |
| 151 58.27814570 | |
| 152 58.55263158 | |
| 153 58.16993464 | |
| 154 57.79220779 | |
| 155 57.41935484 | |
| 156 57.69230769 | |
| 157 57.96178344 | |
| 158 57.59493671 | |
| 159 57.86163522 | |
| 160 57.50000000 | |
| 161 57.14285714 | |
| 162 56.79012346 | |
| 163 57.05521472 | |
| 164 56.70731707 | |
| 165 56.36363636 | |
| 166 56.62650602 | |
| 167 56.28742515 | |
| 168 56.54761905 | |
| 169 56.21301775 | |
| 170 55.88235294 | |
| 171 56.14035088 | |
| 172 55.81395349 | |
| 173 56.06936416 | |
| 174 56.32183908 | |
| 175 56.00000000 | |
| 176 55.68181818 | |
| 177 55.93220339 | |
| 178 55.61797753 | |
| 179 55.30726257 | |
| 180 55.00000000 | |
| 181 54.69613260 | |
| 182 54.94505495 | |
| 183 54.64480874 | |
| 184 54.34782609 | |
| 185 54.59459459 | |
| 186 54.83870968 | |
| 187 55.08021390 | |
| 188 54.78723404 | |
| 189 54.49735450 | |
| 190 54.21052632 | |
| 191 54.45026178 | |
| 192 54.16666667 | |
| 193 53.88601036 | |
| 194 54.12371134 | |
| 195 54.35897436 | |
| 196 54.08163265 | |
| 197 54.31472081 | |
| 198 54.04040404 | |
| 199 53.76884422 | |
| 200 54.00000000 | |
| 201 54.22885572 | |
| 202 54.45544554 | |
| 203 54.18719212 | |
| 204 54.41176471 | |
| 205 54.14634146 | |
| 206 53.88349515 | |
| 207 53.62318841 | |
| 208 53.84615385 | |
| 209 53.58851675 | |
| 210 53.80952381 | |
| 211 54.02843602 | |
| 212 53.77358491 | |
| 213 53.99061033 | |
| 214 54.20560748 | |
| 215 54.41860465 | |
| 216 54.16666667 | |
| 217 54.37788018 | |
| 218 54.12844037 | |
| 219 53.88127854 | |
| 220 54.09090909 | |
| 221 53.84615385 | |
| 222 53.60360360 | |
| 223 53.36322870 | |
| 224 53.57142857 | |
| 225 53.33333333 | |
| 226 53.53982301 | |
| 227 53.74449339 | |
| 228 53.94736842 | |
| 229 54.14847162 | |
| 230 54.34782609 | |
| 231 54.11255411 | |
| 232 53.87931034 | |
| 233 53.64806867 | |
| 234 53.41880342 | |
| 235 53.19148936 | |
| 236 52.96610169 | |
| 237 52.74261603 | |
| 238 52.94117647 | |
| 239 53.13807531 | |
| 240 53.33333333 | |
| 241 53.52697095 | |
| 242 53.30578512 | |
| 243 53.08641975 | |
| 244 52.86885246 | |
| 245 53.06122449 | |
| 246 53.25203252 | |
| 247 53.44129555 | |
| 248 53.62903226 | |
| 249 53.41365462 | |
| 250 53.60000000 | |
| 251 53.78486056 | |
| 252 53.96825397 | |
| 253 54.15019763 | |
| 254 53.93700787 | |
| 255 54.11764706 | |
| 256 54.29687500 | |
| 257 54.47470817 | |
| 258 54.26356589 | |
| 259 54.05405405 | |
| 260 53.84615385 | |
| 261 53.63984674 | |
| 262 53.43511450 | |
| 263 53.61216730 | |
| 264 53.78787879 | |
| 265 53.96226415 | |
| 266 53.75939850 | |
| 267 53.93258427 | |
| 268 53.73134328 | |
| 269 53.90334572 | |
| 270 54.07407407 | |
| 271 54.24354244 | |
| 272 54.04411765 | |
| 273 53.84615385 | |
| 274 53.64963504 | |
| 275 53.45454545 | |
| 276 53.62318841 | |
| 277 53.79061372 | |
| 278 53.59712230 | |
| 279 53.76344086 | |
| 280 53.92857143 | |
| 281 54.09252669 | |
| 282 54.25531915 | |
| 283 54.06360424 | |
| 284 54.22535211 | |
| 285 54.38596491 | |
| 286 54.54545455 | |
| 287 54.35540070 | |
| 288 54.51388889 | |
| 289 54.32525952 | |
| 290 54.48275862 | |
| 291 54.63917526 | |
| 292 54.79452055 | |
| 293 54.94880546 | |
| 294 54.76190476 | |
| 295 54.91525424 | |
| 296 55.06756757 | |
| 297 55.21885522 | |
| 298 55.36912752 | |
| 299 55.18394649 | |
| 300 55.33333333 | |
| 301 55.14950166 | |
| 302 55.29801325 | |
| 303 55.44554455 | |
| 304 55.59210526 | |
| 305 55.73770492 | |
| 306 55.55555556 | |
| 307 55.37459283 | |
| 308 55.51948052 | |
| 309 55.66343042 | |
| 310 55.80645161 | |
| 311 55.94855305 | |
| 312 55.76923077 | |
| 313 55.91054313 | |
| 314 55.73248408 | |
| 315 55.87301587 | |
| 316 56.01265823 | |
| 317 56.15141956 | |
| 318 55.97484277 | |
| 319 55.79937304 | |
| 320 55.62500000 | |
| 321 55.45171340 | |
| 322 55.59006211 | |
| 323 55.41795666 | |
| 324 55.24691358 | |
| 325 55.07692308 | |
| 326 55.21472393 | |
| 327 55.04587156 | |
| 328 55.18292683 | |
| 329 55.31914894 | |
| 330 55.15151515 | |
| 331 55.28700906 | |
| 332 55.42168675 | |
| 333 55.55555556 | |
| 334 55.68862275 | |
| 335 55.52238806 | |
| 336 55.35714286 | |
| 337 55.19287834 | |
| 338 55.02958580 | |
| 339 55.16224189 | |
| 340 55.29411765 | |
| 341 55.13196481 | |
| 342 54.97076023 | |
| 343 55.10204082 | |
| 344 55.23255814 | |
| 345 55.07246377 | |
| 346 55.20231214 | |
| 347 55.04322767 | |
| 348 54.88505747 | |
| 349 54.72779370 | |
| 350 54.85714286 | |
| 351 54.98575499 | |
| 352 54.82954545 | |
| 353 54.95750708 | |
| 354 54.80225989 | |
| 355 54.64788732 | |
| 356 54.77528090 | |
| 357 54.62184874 | |
| 358 54.74860335 | |
| 359 54.59610028 | |
| 360 54.72222222 | |
| 361 54.84764543 | |
| 362 54.69613260 | |
| 363 54.54545455 | |
| 364 54.39560440 | |
| 365 54.24657534 | |
| 366 54.09836066 | |
| 367 54.22343324 | |
| 368 54.07608696 | |
| 369 53.92953930 | |
| 370 53.78378378 | |
| 371 53.63881402 | |
| 372 53.49462366 | |
| 373 53.35120643 | |
| 374 53.47593583 | |
| 375 53.33333333 | |
| 376 53.19148936 | |
| 377 53.31564987 | |
| 378 53.43915344 | |
| 379 53.29815303 | |
| 380 53.42105263 | |
| 381 53.28083990 | |
| 382 53.14136126 | |
| 383 53.00261097 | |
| 384 53.12500000 | |
| 385 53.24675325 | |
| 386 53.36787565 | |
| 387 53.22997416 | |
| 388 53.09278351 | |
| 389 52.95629820 | |
| 390 52.82051282 | |
| 391 52.68542199 | |
| 392 52.80612245 | |
| 393 52.67175573 | |
| 394 52.79187817 | |
| 395 52.91139241 | |
| 396 52.77777778 | |
| 397 52.64483627 | |
| 398 52.51256281 | |
| 399 52.63157895 | |
| 400 52.50000000 | |
| 401 52.36907731 | |
| 402 52.48756219 | |
| 403 52.35732010 | |
| 404 52.47524752 | |
| 405 52.59259259 | |
| 406 52.70935961 | |
| 407 52.82555283 | |
| 408 52.94117647 | |
| 409 52.81173594 | |
| 410 52.68292683 | |
| 411 52.55474453 | |
| 412 52.66990291 | |
| 413 52.54237288 | |
| 414 52.65700483 | |
| 415 52.53012048 | |
| 416 52.40384615 | |
| 417 52.27817746 | |
| 418 52.15311005 | |
| 419 52.02863962 | |
| 420 51.90476190 | |
| 421 51.78147268 | |
| 422 51.89573460 | |
| 423 52.00945626 | |
| 424 51.88679245 | |
| 425 51.76470588 | |
| 426 51.64319249 | |
| 427 51.75644028 | |
| 428 51.63551402 | |
| 429 51.74825175 | |
| 430 51.62790698 | |
| 431 51.50812065 | |
| 432 51.62037037 | |
| 433 51.50115473 | |
| 434 51.61290323 | |
| 435 51.72413793 | |
| 436 51.60550459 | |
| 437 51.48741419 | |
| 438 51.36986301 | |
| 439 51.48063781 | |
| 440 51.36363636 | |
| 441 51.47392290 | |
| 442 51.35746606 | |
| 443 51.24153499 | |
| 444 51.12612613 | |
| 445 51.23595506 | |
| 446 51.34529148 | |
| 447 51.23042506 | |
| 448 51.11607143 | |
| 449 51.22494432 | |
| 450 51.11111111 | |
| 451 50.99778271 | |
| 452 50.88495575 | |
| 453 50.77262693 | |
| 454 50.66079295 | |
| 455 50.54945055 | |
| 456 50.43859649 | |
| 457 50.32822757 | |
| 458 50.43668122 | |
| 459 50.54466231 | |
| 460 50.43478261 | |
| 461 50.32537961 | |
| 462 50.43290043 | |
| 463 50.32397408 | |
| 464 50.21551724 | |
| 465 50.32258065 | |
| 466 50.21459227 | |
| 467 50.32119914 | |
| 468 50.42735043 | |
| 469 50.31982942 | |
| 470 50.21276596 | |
| 471 50.31847134 | |
| 472 50.42372881 | |
| 473 50.31712474 | |
| 474 50.42194093 | |
| 475 50.31578947 | |
| 476 50.21008403 | |
| 477 50.31446541 | |
| 478 50.41841004 | |
| 479 50.52192067 | |
| 480 50.62500000 | |
| 481 50.51975052 | |
| 482 50.62240664 | |
| 483 50.72463768 | |
| 484 50.82644628 | |
| 485 50.92783505 | |
| 486 51.02880658 | |
| 487 50.92402464 | |
| 488 50.81967213 | |
| 489 50.71574642 | |
| 490 50.61224490 | |
| 491 50.71283096 | |
| 492 50.60975610 | |
| 493 50.50709939 | |
| 494 50.60728745 | |
| 495 50.50505051 | |
| 496 50.40322581 | |
| 497 50.50301811 | |
| 498 50.60240964 | |
| 499 50.70140281 | |
| 500 50.80000000 | |
| 501 50.69860279 | |
| 502 50.79681275 | |
| 503 50.89463221 | |
| 504 50.79365079 | |
| 505 50.89108911 | |
| 506 50.79051383 | |
| 507 50.69033531 | |
| 508 50.78740157 | |
| 509 50.68762279 | |
| 510 50.58823529 | |
| 511 50.48923679 | |
| 512 50.39062500 | |
| 513 50.29239766 | |
| 514 50.38910506 | |
| 515 50.48543689 | |
| 516 50.58139535 | |
| 517 50.48355899 | |
| 518 50.38610039 | |
| 519 50.28901734 | |
| 520 50.19230769 | |
| 521 50.28790787 | |
| 522 50.38314176 | |
| 523 50.28680688 | |
| 524 50.19083969 | |
| 525 50.09523810 | |
| 526 50.00000000 | |
| 527 49.90512334 | |
| 528 49.81060606 | |
| 529 49.71644612 | |
| 530 49.81132075 | |
| 531 49.90583804 | |
| 532 50.00000000 | |
| 533 50.09380863 | |
| 534 50.00000000 | |
| 535 50.09345794 | |
| 536 50.18656716 | |
| 537 50.27932961 | |
| 538 50.18587361 | |
| 539 50.27829314 | |
| 540 50.18518519 | |
| 541 50.09242144 | |
| 542 50.18450185 | |
| 543 50.09208103 | |
| 544 50.00000000 | |
| 545 49.90825688 | |
| 546 50.00000000 | |
| 547 49.90859232 | |
| 548 50.00000000 | |
| 549 49.90892532 | |
| 550 50.00000000 | |
| 551 49.90925590 | |
| 552 50.00000000 | |
| 553 50.09041591 | |
| 554 50.18050542 | |
| 555 50.27027027 | |
| 556 50.17985612 | |
| 557 50.26929982 | |
| 558 50.17921147 | |
| 559 50.26833631 | |
| 560 50.35714286 | |
| 561 50.44563280 | |
| 562 50.53380783 | |
| 563 50.62166963 | |
| 564 50.53191489 | |
| 565 50.44247788 | |
| 566 50.53003534 | |
| 567 50.44091711 | |
| 568 50.52816901 | |
| 569 50.43936731 | |
| 570 50.35087719 | |
| 571 50.26269702 | |
| 572 50.17482517 | |
| 573 50.08726003 | |
| 574 50.17421603 | |
| 575 50.08695652 | |
| 576 50.17361111 | |
| 577 50.25996534 | |
| 578 50.34602076 | |
| 579 50.43177893 | |
| 580 50.51724138 | |
| 581 50.43029260 | |
| 582 50.34364261 | |
| 583 50.42881647 | |
| 584 50.51369863 | |
| 585 50.59829060 | |
| 586 50.51194539 | |
| 587 50.42589438 | |
| 588 50.51020408 | |
| 589 50.59422750 | |
| 590 50.67796610 | |
| 591 50.76142132 | |
| 592 50.67567568 | |
| 593 50.59021922 | |
| 594 50.67340067 | |
| 595 50.75630252 | |
| 596 50.67114094 | |
| 597 50.58626466 | |
| 598 50.50167224 | |
| 599 50.41736227 | |
| 600 50.50000000 | |
| 601 50.41597338 | |
| 602 50.33222591 | |
| 603 50.24875622 | |
| 604 50.16556291 | |
| 605 50.08264463 | |
| 606 50.16501650 | |
| 607 50.24711697 | |
| 608 50.32894737 | |
| 609 50.24630542 | |
| 610 50.32786885 | |
| 611 50.24549918 | |
| 612 50.16339869 | |
| 613 50.08156607 | |
| 614 50.00000000 | |
| 615 49.91869919 | |
| 616 50.00000000 | |
| 617 50.08103728 | |
| 618 50.16181230 | |
| 619 50.08077544 | |
| 620 50.16129032 | |
| 621 50.24154589 | |
| 622 50.32154341 | |
| 623 50.40128411 | |
| 624 50.48076923 | |
| 625 50.40000000 | |
| 626 50.31948882 | |
| 627 50.39872408 | |
| 628 50.31847134 | |
| 629 50.39745628 | |
| 630 50.47619048 | |
| 631 50.55467512 | |
| 632 50.47468354 | |
| 633 50.39494471 | |
| 634 50.31545741 | |
| 635 50.39370079 | |
| 636 50.47169811 | |
| 637 50.39246468 | |
| 638 50.47021944 | |
| 639 50.39123631 | |
| 640 50.46875000 | |
| 641 50.54602184 | |
| 642 50.46728972 | |
| 643 50.38880249 | |
| 644 50.46583851 | |
| 645 50.38759690 | |
| 646 50.46439628 | |
| 647 50.54095827 | |
| 648 50.46296296 | |
| 649 50.38520801 | |
| 650 50.46153846 | |
| 651 50.53763441 | |
| 652 50.46012270 | |
| 653 50.38284839 | |
| 654 50.30581040 | |
| 655 50.22900763 | |
| 656 50.15243902 | |
| 657 50.07610350 | |
| 658 50.15197568 | |
| 659 50.07587253 | |
| 660 50.00000000 | |
| 661 49.92435703 | |
| 662 50.00000000 | |
| 663 49.92458522 | |
| 664 50.00000000 | |
| 665 50.07518797 | |
| 666 50.00000000 | |
| 667 50.07496252 | |
| 668 50.00000000 | |
| 669 49.92526158 | |
| 670 50.00000000 | |
| 671 49.92548435 | |
| 672 49.85119048 | |
| 673 49.77711738 | |
| 674 49.70326409 | |
| 675 49.62962963 | |
| 676 49.55621302 | |
| 677 49.63072378 | |
| 678 49.55752212 | |
| 679 49.63181149 | |
| 680 49.70588235 | |
| 681 49.63289280 | |
| 682 49.70674487 | |
| 683 49.63396779 | |
| 684 49.70760234 | |
| 685 49.78102190 | |
| 686 49.70845481 | |
| 687 49.63609898 | |
| 688 49.70930233 | |
| 689 49.78229318 | |
| 690 49.85507246 | |
| 691 49.78292330 | |
| 692 49.85549133 | |
| 693 49.78354978 | |
| 694 49.71181556 | |
| 695 49.64028777 | |
| 696 49.56896552 | |
| 697 49.49784792 | |
| 698 49.42693410 | |
| 699 49.35622318 | |
| 700 49.42857143 | |
| 701 49.50071327 | |
| 702 49.57264957 | |
| 703 49.64438122 | |
| 704 49.71590909 | |
| 705 49.64539007 | |
| 706 49.71671388 | |
| 707 49.78783593 | |
| 708 49.85875706 | |
| 709 49.92947814 | |
| 710 50.00000000 | |
| 711 50.07032349 | |
| 712 50.14044944 | |
| 713 50.07012623 | |
| 714 50.00000000 | |
| 715 49.93006993 | |
| 716 49.86033520 | |
| 717 49.79079498 | |
| 718 49.72144847 | |
| 719 49.65229485 | |
| 720 49.72222222 | |
| 721 49.79195562 | |
| 722 49.86149584 | |
| 723 49.93084371 | |
| 724 50.00000000 | |
| 725 50.06896552 | |
| 726 50.00000000 | |
| 727 49.93122421 | |
| 728 50.00000000 | |
| 729 50.06858711 | |
| 730 50.13698630 | |
| 731 50.20519836 | |
| 732 50.27322404 | |
| 733 50.34106412 | |
| 734 50.40871935 | |
| 735 50.34013605 | |
| 736 50.40760870 | |
| 737 50.33921303 | |
| 738 50.27100271 | |
| 739 50.20297700 | |
| 740 50.13513514 | |
| 741 50.20242915 | |
| 742 50.13477089 | |
| 743 50.20188425 | |
| 744 50.13440860 | |
| 745 50.20134228 | |
| 746 50.26809651 | |
| 747 50.20080321 | |
| 748 50.13368984 | |
| 749 50.06675567 | |
| 750 50.13333333 | |
| Final result: 50.1333 +/- 1.8270 | |
| Random chance: 25.0083 +/- 1.5824 | |