| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 1548 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 1548 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 50.00000000 | |
| 5 40.00000000 | |
| 6 33.33333333 | |
| 7 42.85714286 | |
| 8 50.00000000 | |
| 9 44.44444444 | |
| 10 40.00000000 | |
| 11 36.36363636 | |
| 12 33.33333333 | |
| 13 38.46153846 | |
| 14 35.71428571 | |
| 15 40.00000000 | |
| 16 43.75000000 | |
| 17 41.17647059 | |
| 18 38.88888889 | |
| 19 42.10526316 | |
| 20 40.00000000 | |
| 21 38.09523810 | |
| 22 36.36363636 | |
| 23 34.78260870 | |
| 24 33.33333333 | |
| 25 36.00000000 | |
| 26 34.61538462 | |
| 27 33.33333333 | |
| 28 35.71428571 | |
| 29 37.93103448 | |
| 30 40.00000000 | |
| 31 41.93548387 | |
| 32 40.62500000 | |
| 33 39.39393939 | |
| 34 41.17647059 | |
| 35 42.85714286 | |
| 36 44.44444444 | |
| 37 43.24324324 | |
| 38 42.10526316 | |
| 39 41.02564103 | |
| 40 40.00000000 | |
| 41 39.02439024 | |
| 42 38.09523810 | |
| 43 39.53488372 | |
| 44 38.63636364 | |
| 45 37.77777778 | |
| 46 36.95652174 | |
| 47 36.17021277 | |
| 48 37.50000000 | |
| 49 38.77551020 | |
| 50 40.00000000 | |
| 51 39.21568627 | |
| 52 40.38461538 | |
| 53 39.62264151 | |
| 54 38.88888889 | |
| 55 40.00000000 | |
| 56 41.07142857 | |
| 57 42.10526316 | |
| 58 41.37931034 | |
| 59 40.67796610 | |
| 60 40.00000000 | |
| 61 40.98360656 | |
| 62 41.93548387 | |
| 63 41.26984127 | |
| 64 40.62500000 | |
| 65 40.00000000 | |
| 66 39.39393939 | |
| 67 40.29850746 | |
| 68 39.70588235 | |
| 69 39.13043478 | |
| 70 40.00000000 | |
| 71 40.84507042 | |
| 72 40.27777778 | |
| 73 39.72602740 | |
| 74 40.54054054 | |
| 75 41.33333333 | |
| 76 42.10526316 | |
| 77 42.85714286 | |
| 78 43.58974359 | |
| 79 43.03797468 | |
| 80 42.50000000 | |
| 81 43.20987654 | |
| 82 42.68292683 | |
| 83 43.37349398 | |
| 84 44.04761905 | |
| 85 43.52941176 | |
| 86 43.02325581 | |
| 87 43.67816092 | |
| 88 44.31818182 | |
| 89 43.82022472 | |
| 90 43.33333333 | |
| 91 42.85714286 | |
| 92 42.39130435 | |
| 93 41.93548387 | |
| 94 41.48936170 | |
| 95 42.10526316 | |
| 96 42.70833333 | |
| 97 42.26804124 | |
| 98 41.83673469 | |
| 99 41.41414141 | |
| 100 41.00000000 | |
| 101 41.58415842 | |
| 102 41.17647059 | |
| 103 40.77669903 | |
| 104 40.38461538 | |
| 105 40.95238095 | |
| 106 40.56603774 | |
| 107 40.18691589 | |
| 108 39.81481481 | |
| 109 39.44954128 | |
| 110 39.09090909 | |
| 111 38.73873874 | |
| 112 39.28571429 | |
| 113 38.93805310 | |
| 114 39.47368421 | |
| 115 40.00000000 | |
| 116 39.65517241 | |
| 117 40.17094017 | |
| 118 39.83050847 | |
| 119 39.49579832 | |
| 120 39.16666667 | |
| 121 39.66942149 | |
| 122 39.34426230 | |
| 123 39.02439024 | |
| 124 38.70967742 | |
| 125 39.20000000 | |
| 126 38.88888889 | |
| 127 38.58267717 | |
| 128 38.28125000 | |
| 129 37.98449612 | |
| 130 37.69230769 | |
| 131 37.40458015 | |
| 132 37.87878788 | |
| 133 38.34586466 | |
| 134 38.05970149 | |
| 135 37.77777778 | |
| 136 37.50000000 | |
| 137 37.22627737 | |
| 138 36.95652174 | |
| 139 36.69064748 | |
| 140 36.42857143 | |
| 141 36.87943262 | |
| 142 37.32394366 | |
| 143 37.06293706 | |
| 144 36.80555556 | |
| 145 37.24137931 | |
| 146 37.67123288 | |
| 147 37.41496599 | |
| 148 37.83783784 | |
| 149 37.58389262 | |
| 150 38.00000000 | |
| 151 37.74834437 | |
| 152 37.50000000 | |
| 153 37.25490196 | |
| 154 37.01298701 | |
| 155 36.77419355 | |
| 156 36.53846154 | |
| 157 36.94267516 | |
| 158 36.70886076 | |
| 159 36.47798742 | |
| 160 36.87500000 | |
| 161 36.64596273 | |
| 162 36.41975309 | |
| 163 36.19631902 | |
| 164 36.58536585 | |
| 165 36.96969697 | |
| 166 36.74698795 | |
| 167 37.12574850 | |
| 168 36.90476190 | |
| 169 37.27810651 | |
| 170 37.05882353 | |
| 171 37.42690058 | |
| 172 37.20930233 | |
| 173 36.99421965 | |
| 174 36.78160920 | |
| 175 36.57142857 | |
| 176 36.36363636 | |
| 177 36.15819209 | |
| 178 36.51685393 | |
| 179 36.87150838 | |
| 180 36.66666667 | |
| 181 36.46408840 | |
| 182 36.26373626 | |
| 183 36.61202186 | |
| 184 36.41304348 | |
| 185 36.75675676 | |
| 186 37.09677419 | |
| 187 36.89839572 | |
| 188 37.23404255 | |
| 189 37.03703704 | |
| 190 36.84210526 | |
| 191 37.17277487 | |
| 192 36.97916667 | |
| 193 36.78756477 | |
| 194 36.59793814 | |
| 195 36.92307692 | |
| 196 37.24489796 | |
| 197 37.05583756 | |
| 198 37.37373737 | |
| 199 37.18592965 | |
| 200 37.50000000 | |
| 201 37.31343284 | |
| 202 37.12871287 | |
| 203 36.94581281 | |
| 204 36.76470588 | |
| 205 37.07317073 | |
| 206 37.37864078 | |
| 207 37.68115942 | |
| 208 37.50000000 | |
| 209 37.32057416 | |
| 210 37.14285714 | |
| 211 37.44075829 | |
| 212 37.26415094 | |
| 213 37.08920188 | |
| 214 36.91588785 | |
| 215 37.20930233 | |
| 216 37.03703704 | |
| 217 36.86635945 | |
| 218 36.69724771 | |
| 219 36.98630137 | |
| 220 36.81818182 | |
| 221 36.65158371 | |
| 222 36.48648649 | |
| 223 36.77130045 | |
| 224 37.05357143 | |
| 225 37.33333333 | |
| 226 37.16814159 | |
| 227 37.44493392 | |
| 228 37.71929825 | |
| 229 37.55458515 | |
| 230 37.39130435 | |
| 231 37.22943723 | |
| 232 37.50000000 | |
| 233 37.76824034 | |
| 234 38.03418803 | |
| 235 37.87234043 | |
| 236 37.71186441 | |
| 237 37.55274262 | |
| 238 37.39495798 | |
| 239 37.23849372 | |
| 240 37.08333333 | |
| 241 36.92946058 | |
| 242 37.19008264 | |
| 243 37.44855967 | |
| 244 37.29508197 | |
| 245 37.55102041 | |
| 246 37.39837398 | |
| 247 37.65182186 | |
| 248 37.50000000 | |
| 249 37.34939759 | |
| 250 37.20000000 | |
| 251 37.05179283 | |
| 252 36.90476190 | |
| 253 36.75889328 | |
| 254 36.61417323 | |
| 255 36.86274510 | |
| 256 37.10937500 | |
| 257 37.35408560 | |
| 258 37.20930233 | |
| 259 37.45173745 | |
| 260 37.69230769 | |
| 261 37.54789272 | |
| 262 37.78625954 | |
| 263 38.02281369 | |
| 264 37.87878788 | |
| 265 37.73584906 | |
| 266 37.59398496 | |
| 267 37.45318352 | |
| 268 37.31343284 | |
| 269 37.54646840 | |
| 270 37.40740741 | |
| 271 37.26937269 | |
| 272 37.50000000 | |
| 273 37.36263736 | |
| 274 37.22627737 | |
| 275 37.09090909 | |
| 276 37.31884058 | |
| 277 37.54512635 | |
| 278 37.41007194 | |
| 279 37.27598566 | |
| 280 37.50000000 | |
| 281 37.36654804 | |
| 282 37.23404255 | |
| 283 37.10247350 | |
| 284 36.97183099 | |
| 285 37.19298246 | |
| 286 37.06293706 | |
| 287 36.93379791 | |
| 288 36.80555556 | |
| 289 36.67820069 | |
| 290 36.89655172 | |
| 291 37.11340206 | |
| 292 37.32876712 | |
| 293 37.54266212 | |
| 294 37.41496599 | |
| 295 37.28813559 | |
| 296 37.16216216 | |
| 297 37.03703704 | |
| 298 36.91275168 | |
| 299 36.78929766 | |
| 300 37.00000000 | |
| 301 36.87707641 | |
| 302 37.08609272 | |
| 303 36.96369637 | |
| 304 37.17105263 | |
| 305 37.37704918 | |
| 306 37.25490196 | |
| 307 37.45928339 | |
| 308 37.33766234 | |
| 309 37.21682848 | |
| 310 37.09677419 | |
| 311 37.29903537 | |
| 312 37.50000000 | |
| 313 37.69968051 | |
| 314 37.57961783 | |
| 315 37.46031746 | |
| 316 37.34177215 | |
| 317 37.53943218 | |
| 318 37.42138365 | |
| 319 37.61755486 | |
| 320 37.50000000 | |
| 321 37.38317757 | |
| 322 37.26708075 | |
| 323 37.15170279 | |
| 324 37.03703704 | |
| 325 37.23076923 | |
| 326 37.11656442 | |
| 327 37.30886850 | |
| 328 37.19512195 | |
| 329 37.38601824 | |
| 330 37.27272727 | |
| 331 37.16012085 | |
| 332 37.34939759 | |
| 333 37.53753754 | |
| 334 37.42514970 | |
| 335 37.61194030 | |
| 336 37.50000000 | |
| 337 37.38872404 | |
| 338 37.27810651 | |
| 339 37.16814159 | |
| 340 37.35294118 | |
| 341 37.53665689 | |
| 342 37.71929825 | |
| 343 37.60932945 | |
| 344 37.50000000 | |
| 345 37.39130435 | |
| 346 37.57225434 | |
| 347 37.75216138 | |
| 348 37.64367816 | |
| 349 37.53581662 | |
| 350 37.42857143 | |
| 351 37.32193732 | |
| 352 37.21590909 | |
| 353 37.39376771 | |
| 354 37.28813559 | |
| 355 37.18309859 | |
| 356 37.35955056 | |
| 357 37.53501401 | |
| 358 37.43016760 | |
| 359 37.32590529 | |
| 360 37.50000000 | |
| 361 37.67313019 | |
| 362 37.84530387 | |
| 363 38.01652893 | |
| 364 37.91208791 | |
| 365 38.08219178 | |
| 366 38.25136612 | |
| 367 38.14713896 | |
| 368 38.04347826 | |
| 369 37.94037940 | |
| 370 38.10810811 | |
| 371 38.00539084 | |
| 372 37.90322581 | |
| 373 37.80160858 | |
| 374 37.96791444 | |
| 375 38.13333333 | |
| 376 38.03191489 | |
| 377 37.93103448 | |
| 378 38.09523810 | |
| 379 37.99472296 | |
| 380 37.89473684 | |
| 381 38.05774278 | |
| 382 37.95811518 | |
| 383 38.12010444 | |
| 384 38.02083333 | |
| 385 37.92207792 | |
| 386 37.82383420 | |
| 387 37.72609819 | |
| 388 37.62886598 | |
| 389 37.78920308 | |
| 390 37.94871795 | |
| 391 38.10741688 | |
| 392 38.26530612 | |
| 393 38.16793893 | |
| 394 38.07106599 | |
| 395 38.22784810 | |
| 396 38.38383838 | |
| 397 38.53904282 | |
| 398 38.44221106 | |
| 399 38.59649123 | |
| 400 38.50000000 | |
| 401 38.65336658 | |
| 402 38.80597015 | |
| 403 38.70967742 | |
| 404 38.61386139 | |
| 405 38.76543210 | |
| 406 38.66995074 | |
| 407 38.57493857 | |
| 408 38.72549020 | |
| 409 38.87530562 | |
| 410 38.78048780 | |
| 411 38.68613139 | |
| 412 38.59223301 | |
| 413 38.49878935 | |
| 414 38.64734300 | |
| 415 38.79518072 | |
| 416 38.94230769 | |
| 417 38.84892086 | |
| 418 38.75598086 | |
| 419 38.90214797 | |
| 420 38.80952381 | |
| 421 38.71733967 | |
| 422 38.62559242 | |
| 423 38.53427896 | |
| 424 38.67924528 | |
| 425 38.58823529 | |
| 426 38.49765258 | |
| 427 38.40749415 | |
| 428 38.31775701 | |
| 429 38.46153846 | |
| 430 38.37209302 | |
| 431 38.51508121 | |
| 432 38.65740741 | |
| 433 38.56812933 | |
| 434 38.47926267 | |
| 435 38.39080460 | |
| 436 38.30275229 | |
| 437 38.44393593 | |
| 438 38.58447489 | |
| 439 38.72437358 | |
| 440 38.63636364 | |
| 441 38.77551020 | |
| 442 38.68778281 | |
| 443 38.60045147 | |
| 444 38.73873874 | |
| 445 38.65168539 | |
| 446 38.56502242 | |
| 447 38.47874720 | |
| 448 38.39285714 | |
| 449 38.30734967 | |
| 450 38.22222222 | |
| 451 38.13747228 | |
| 452 38.05309735 | |
| 453 38.18984547 | |
| 454 38.10572687 | |
| 455 38.02197802 | |
| 456 38.15789474 | |
| 457 38.07439825 | |
| 458 37.99126638 | |
| 459 37.90849673 | |
| 460 38.04347826 | |
| 461 38.17787419 | |
| 462 38.09523810 | |
| 463 38.22894168 | |
| 464 38.36206897 | |
| 465 38.27956989 | |
| 466 38.19742489 | |
| 467 38.32976445 | |
| 468 38.24786325 | |
| 469 38.16631130 | |
| 470 38.08510638 | |
| 471 38.00424628 | |
| 472 37.92372881 | |
| 473 38.05496829 | |
| 474 37.97468354 | |
| 475 37.89473684 | |
| 476 38.02521008 | |
| 477 37.94549266 | |
| 478 38.07531381 | |
| 479 37.99582463 | |
| 480 38.12500000 | |
| 481 38.25363825 | |
| 482 38.17427386 | |
| 483 38.09523810 | |
| 484 38.22314050 | |
| 485 38.14432990 | |
| 486 38.27160494 | |
| 487 38.39835729 | |
| 488 38.31967213 | |
| 489 38.24130879 | |
| 490 38.36734694 | |
| 491 38.28920570 | |
| 492 38.21138211 | |
| 493 38.13387424 | |
| 494 38.25910931 | |
| 495 38.38383838 | |
| 496 38.50806452 | |
| 497 38.43058350 | |
| 498 38.35341365 | |
| 499 38.47695391 | |
| 500 38.60000000 | |
| 501 38.52295409 | |
| 502 38.44621514 | |
| 503 38.36978131 | |
| 504 38.49206349 | |
| 505 38.41584158 | |
| 506 38.53754941 | |
| 507 38.46153846 | |
| 508 38.58267717 | |
| 509 38.50687623 | |
| 510 38.62745098 | |
| 511 38.55185910 | |
| 512 38.47656250 | |
| 513 38.40155945 | |
| 514 38.32684825 | |
| 515 38.44660194 | |
| 516 38.56589147 | |
| 517 38.49129594 | |
| 518 38.61003861 | |
| 519 38.53564547 | |
| 520 38.46153846 | |
| 521 38.38771593 | |
| 522 38.50574713 | |
| 523 38.43212237 | |
| 524 38.54961832 | |
| 525 38.47619048 | |
| 526 38.40304183 | |
| 527 38.51992410 | |
| 528 38.44696970 | |
| 529 38.37429112 | |
| 530 38.49056604 | |
| 531 38.41807910 | |
| 532 38.34586466 | |
| 533 38.46153846 | |
| 534 38.57677903 | |
| 535 38.69158879 | |
| 536 38.80597015 | |
| 537 38.91992551 | |
| 538 39.03345725 | |
| 539 38.96103896 | |
| 540 38.88888889 | |
| 541 38.81700555 | |
| 542 38.92988930 | |
| 543 38.85819521 | |
| 544 38.78676471 | |
| 545 38.71559633 | |
| 546 38.64468864 | |
| 547 38.57404022 | |
| 548 38.68613139 | |
| 549 38.61566485 | |
| 550 38.54545455 | |
| 551 38.47549909 | |
| 552 38.40579710 | |
| 553 38.33634720 | |
| 554 38.26714801 | |
| 555 38.19819820 | |
| 556 38.30935252 | |
| 557 38.42010772 | |
| 558 38.53046595 | |
| 559 38.64042934 | |
| 560 38.57142857 | |
| 561 38.50267380 | |
| 562 38.61209964 | |
| 563 38.72113677 | |
| 564 38.65248227 | |
| 565 38.76106195 | |
| 566 38.86925795 | |
| 567 38.80070547 | |
| 568 38.73239437 | |
| 569 38.66432337 | |
| 570 38.59649123 | |
| 571 38.70402802 | |
| 572 38.63636364 | |
| 573 38.56893543 | |
| 574 38.50174216 | |
| 575 38.43478261 | |
| 576 38.54166667 | |
| 577 38.64818024 | |
| 578 38.58131488 | |
| 579 38.51468048 | |
| 580 38.62068966 | |
| 581 38.55421687 | |
| 582 38.48797251 | |
| 583 38.59348199 | |
| 584 38.69863014 | |
| 585 38.80341880 | |
| 586 38.73720137 | |
| 587 38.67120954 | |
| 588 38.60544218 | |
| 589 38.53989813 | |
| 590 38.47457627 | |
| 591 38.40947547 | |
| 592 38.34459459 | |
| 593 38.27993255 | |
| 594 38.38383838 | |
| 595 38.31932773 | |
| 596 38.42281879 | |
| 597 38.35845896 | |
| 598 38.29431438 | |
| 599 38.39732888 | |
| 600 38.33333333 | |
| 601 38.26955075 | |
| 602 38.20598007 | |
| 603 38.30845771 | |
| 604 38.24503311 | |
| 605 38.34710744 | |
| 606 38.28382838 | |
| 607 38.38550247 | |
| 608 38.32236842 | |
| 609 38.25944171 | |
| 610 38.19672131 | |
| 611 38.13420622 | |
| 612 38.07189542 | |
| 613 38.17292007 | |
| 614 38.27361564 | |
| 615 38.21138211 | |
| 616 38.31168831 | |
| 617 38.41166937 | |
| 618 38.34951456 | |
| 619 38.28756058 | |
| 620 38.22580645 | |
| 621 38.32528180 | |
| 622 38.26366559 | |
| 623 38.20224719 | |
| 624 38.30128205 | |
| 625 38.24000000 | |
| 626 38.33865815 | |
| 627 38.27751196 | |
| 628 38.21656051 | |
| 629 38.31478537 | |
| 630 38.41269841 | |
| 631 38.35182250 | |
| 632 38.44936709 | |
| 633 38.38862559 | |
| 634 38.32807571 | |
| 635 38.26771654 | |
| 636 38.20754717 | |
| 637 38.14756672 | |
| 638 38.08777429 | |
| 639 38.18466354 | |
| 640 38.28125000 | |
| 641 38.22152886 | |
| 642 38.16199377 | |
| 643 38.10264386 | |
| 644 38.19875776 | |
| 645 38.13953488 | |
| 646 38.08049536 | |
| 647 38.02163833 | |
| 648 37.96296296 | |
| 649 37.90446841 | |
| 650 37.84615385 | |
| 651 37.78801843 | |
| 652 37.73006135 | |
| 653 37.67228178 | |
| 654 37.76758410 | |
| 655 37.70992366 | |
| 656 37.80487805 | |
| 657 37.74733638 | |
| 658 37.84194529 | |
| 659 37.93626707 | |
| 660 37.87878788 | |
| 661 37.82148260 | |
| 662 37.76435045 | |
| 663 37.70739065 | |
| 664 37.65060241 | |
| 665 37.59398496 | |
| 666 37.53753754 | |
| 667 37.63118441 | |
| 668 37.72455090 | |
| 669 37.81763827 | |
| 670 37.76119403 | |
| 671 37.70491803 | |
| 672 37.64880952 | |
| 673 37.59286776 | |
| 674 37.53709199 | |
| 675 37.48148148 | |
| 676 37.42603550 | |
| 677 37.37075332 | |
| 678 37.31563422 | |
| 679 37.26067747 | |
| 680 37.20588235 | |
| 681 37.29809104 | |
| 682 37.24340176 | |
| 683 37.18887262 | |
| 684 37.28070175 | |
| 685 37.37226277 | |
| 686 37.31778426 | |
| 687 37.26346434 | |
| 688 37.20930233 | |
| 689 37.30043541 | |
| 690 37.24637681 | |
| 691 37.19247467 | |
| 692 37.13872832 | |
| 693 37.08513709 | |
| 694 37.03170029 | |
| 695 36.97841727 | |
| 696 37.06896552 | |
| 697 37.01578192 | |
| 698 36.96275072 | |
| 699 37.05293276 | |
| 700 37.00000000 | |
| 701 37.08987161 | |
| 702 37.17948718 | |
| 703 37.12660028 | |
| 704 37.07386364 | |
| 705 37.02127660 | |
| 706 36.96883853 | |
| 707 36.91654880 | |
| 708 36.86440678 | |
| 709 36.81241185 | |
| 710 36.76056338 | |
| 711 36.84950774 | |
| 712 36.93820225 | |
| 713 37.02664797 | |
| 714 37.11484594 | |
| 715 37.06293706 | |
| 716 37.01117318 | |
| 717 37.09902371 | |
| 718 37.04735376 | |
| 719 37.13490960 | |
| 720 37.08333333 | |
| 721 37.03190014 | |
| 722 36.98060942 | |
| 723 37.06777317 | |
| 724 37.15469613 | |
| 725 37.24137931 | |
| 726 37.19008264 | |
| 727 37.13892710 | |
| 728 37.22527473 | |
| 729 37.17421125 | |
| 730 37.12328767 | |
| 731 37.07250342 | |
| 732 37.02185792 | |
| 733 37.10777626 | |
| 734 37.19346049 | |
| 735 37.14285714 | |
| 736 37.09239130 | |
| 737 37.17774763 | |
| 738 37.12737127 | |
| 739 37.21244926 | |
| 740 37.16216216 | |
| 741 37.11201080 | |
| 742 37.06199461 | |
| 743 37.14670256 | |
| 744 37.23118280 | |
| 745 37.18120805 | |
| 746 37.13136729 | |
| 747 37.21552878 | |
| 748 37.16577540 | |
| 749 37.11615487 | |
| 750 37.20000000 | |
| Final result: 37.2000 +/- 1.7661 | |
| Random chance: 25.0000 +/- 1.5822 | |