| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 869 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 869 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 25.00000000 | |
| 5 40.00000000 | |
| 6 50.00000000 | |
| 7 57.14285714 | |
| 8 62.50000000 | |
| 9 55.55555556 | |
| 10 60.00000000 | |
| 11 54.54545455 | |
| 12 58.33333333 | |
| 13 53.84615385 | |
| 14 57.14285714 | |
| 15 60.00000000 | |
| 16 62.50000000 | |
| 17 64.70588235 | |
| 18 66.66666667 | |
| 19 63.15789474 | |
| 20 60.00000000 | |
| 21 61.90476190 | |
| 22 63.63636364 | |
| 23 60.86956522 | |
| 24 62.50000000 | |
| 25 60.00000000 | |
| 26 61.53846154 | |
| 27 62.96296296 | |
| 28 60.71428571 | |
| 29 62.06896552 | |
| 30 63.33333333 | |
| 31 61.29032258 | |
| 32 62.50000000 | |
| 33 60.60606061 | |
| 34 58.82352941 | |
| 35 57.14285714 | |
| 36 55.55555556 | |
| 37 56.75675676 | |
| 38 55.26315789 | |
| 39 56.41025641 | |
| 40 57.50000000 | |
| 41 56.09756098 | |
| 42 54.76190476 | |
| 43 53.48837209 | |
| 44 52.27272727 | |
| 45 53.33333333 | |
| 46 54.34782609 | |
| 47 55.31914894 | |
| 48 54.16666667 | |
| 49 53.06122449 | |
| 50 54.00000000 | |
| 51 54.90196078 | |
| 52 55.76923077 | |
| 53 56.60377358 | |
| 54 55.55555556 | |
| 55 54.54545455 | |
| 56 55.35714286 | |
| 57 56.14035088 | |
| 58 56.89655172 | |
| 59 57.62711864 | |
| 60 58.33333333 | |
| 61 59.01639344 | |
| 62 59.67741935 | |
| 63 60.31746032 | |
| 64 60.93750000 | |
| 65 60.00000000 | |
| 66 60.60606061 | |
| 67 61.19402985 | |
| 68 60.29411765 | |
| 69 59.42028986 | |
| 70 58.57142857 | |
| 71 57.74647887 | |
| 72 56.94444444 | |
| 73 57.53424658 | |
| 74 58.10810811 | |
| 75 58.66666667 | |
| 76 57.89473684 | |
| 77 58.44155844 | |
| 78 58.97435897 | |
| 79 58.22784810 | |
| 80 58.75000000 | |
| 81 58.02469136 | |
| 82 58.53658537 | |
| 83 57.83132530 | |
| 84 58.33333333 | |
| 85 58.82352941 | |
| 86 58.13953488 | |
| 87 57.47126437 | |
| 88 57.95454545 | |
| 89 58.42696629 | |
| 90 58.88888889 | |
| 91 59.34065934 | |
| 92 59.78260870 | |
| 93 59.13978495 | |
| 94 58.51063830 | |
| 95 58.94736842 | |
| 96 59.37500000 | |
| 97 58.76288660 | |
| 98 58.16326531 | |
| 99 58.58585859 | |
| 100 58.00000000 | |
| 101 57.42574257 | |
| 102 57.84313725 | |
| 103 57.28155340 | |
| 104 57.69230769 | |
| 105 58.09523810 | |
| 106 57.54716981 | |
| 107 57.94392523 | |
| 108 57.40740741 | |
| 109 56.88073394 | |
| 110 57.27272727 | |
| 111 57.65765766 | |
| 112 58.03571429 | |
| 113 57.52212389 | |
| 114 57.89473684 | |
| 115 58.26086957 | |
| 116 58.62068966 | |
| 117 58.97435897 | |
| 118 59.32203390 | |
| 119 58.82352941 | |
| 120 58.33333333 | |
| 121 57.85123967 | |
| 122 58.19672131 | |
| 123 58.53658537 | |
| 124 58.06451613 | |
| 125 58.40000000 | |
| 126 58.73015873 | |
| 127 58.26771654 | |
| 128 58.59375000 | |
| 129 58.13953488 | |
| 130 57.69230769 | |
| 131 58.01526718 | |
| 132 58.33333333 | |
| 133 58.64661654 | |
| 134 58.20895522 | |
| 135 57.77777778 | |
| 136 57.35294118 | |
| 137 57.66423358 | |
| 138 57.97101449 | |
| 139 57.55395683 | |
| 140 57.14285714 | |
| 141 56.73758865 | |
| 142 57.04225352 | |
| 143 57.34265734 | |
| 144 56.94444444 | |
| 145 57.24137931 | |
| 146 56.84931507 | |
| 147 57.14285714 | |
| 148 56.75675676 | |
| 149 57.04697987 | |
| 150 57.33333333 | |
| 151 57.61589404 | |
| 152 57.89473684 | |
| 153 57.51633987 | |
| 154 57.14285714 | |
| 155 56.77419355 | |
| 156 57.05128205 | |
| 157 57.32484076 | |
| 158 56.96202532 | |
| 159 57.23270440 | |
| 160 56.87500000 | |
| 161 56.52173913 | |
| 162 56.17283951 | |
| 163 55.82822086 | |
| 164 55.48780488 | |
| 165 55.15151515 | |
| 166 55.42168675 | |
| 167 55.08982036 | |
| 168 55.35714286 | |
| 169 55.02958580 | |
| 170 54.70588235 | |
| 171 54.97076023 | |
| 172 55.23255814 | |
| 173 55.49132948 | |
| 174 55.74712644 | |
| 175 55.42857143 | |
| 176 55.11363636 | |
| 177 55.36723164 | |
| 178 55.61797753 | |
| 179 55.30726257 | |
| 180 55.00000000 | |
| 181 54.69613260 | |
| 182 54.94505495 | |
| 183 54.64480874 | |
| 184 54.34782609 | |
| 185 54.59459459 | |
| 186 54.83870968 | |
| 187 54.54545455 | |
| 188 54.25531915 | |
| 189 53.96825397 | |
| 190 53.68421053 | |
| 191 53.92670157 | |
| 192 53.64583333 | |
| 193 53.36787565 | |
| 194 53.60824742 | |
| 195 53.84615385 | |
| 196 54.08163265 | |
| 197 54.31472081 | |
| 198 54.04040404 | |
| 199 54.27135678 | |
| 200 54.50000000 | |
| 201 54.72636816 | |
| 202 54.45544554 | |
| 203 54.18719212 | |
| 204 54.41176471 | |
| 205 54.14634146 | |
| 206 53.88349515 | |
| 207 53.62318841 | |
| 208 53.84615385 | |
| 209 53.58851675 | |
| 210 53.80952381 | |
| 211 54.02843602 | |
| 212 53.77358491 | |
| 213 53.99061033 | |
| 214 54.20560748 | |
| 215 54.41860465 | |
| 216 54.16666667 | |
| 217 54.37788018 | |
| 218 54.12844037 | |
| 219 53.88127854 | |
| 220 53.63636364 | |
| 221 53.39366516 | |
| 222 53.15315315 | |
| 223 52.91479821 | |
| 224 53.12500000 | |
| 225 52.88888889 | |
| 226 53.09734513 | |
| 227 53.30396476 | |
| 228 53.50877193 | |
| 229 53.71179039 | |
| 230 53.47826087 | |
| 231 53.24675325 | |
| 232 53.01724138 | |
| 233 52.78969957 | |
| 234 52.56410256 | |
| 235 52.34042553 | |
| 236 52.11864407 | |
| 237 51.89873418 | |
| 238 51.68067227 | |
| 239 51.88284519 | |
| 240 52.08333333 | |
| 241 52.28215768 | |
| 242 52.06611570 | |
| 243 52.26337449 | |
| 244 52.04918033 | |
| 245 52.24489796 | |
| 246 52.43902439 | |
| 247 52.63157895 | |
| 248 52.82258065 | |
| 249 52.61044177 | |
| 250 52.80000000 | |
| 251 52.98804781 | |
| 252 53.17460317 | |
| 253 53.35968379 | |
| 254 53.14960630 | |
| 255 53.33333333 | |
| 256 53.51562500 | |
| 257 53.69649805 | |
| 258 53.48837209 | |
| 259 53.28185328 | |
| 260 53.07692308 | |
| 261 52.87356322 | |
| 262 52.67175573 | |
| 263 52.85171103 | |
| 264 53.03030303 | |
| 265 53.20754717 | |
| 266 53.00751880 | |
| 267 53.18352060 | |
| 268 52.98507463 | |
| 269 53.15985130 | |
| 270 53.33333333 | |
| 271 53.50553506 | |
| 272 53.30882353 | |
| 273 53.11355311 | |
| 274 52.91970803 | |
| 275 52.72727273 | |
| 276 52.53623188 | |
| 277 52.70758123 | |
| 278 52.51798561 | |
| 279 52.68817204 | |
| 280 52.85714286 | |
| 281 53.02491103 | |
| 282 53.19148936 | |
| 283 53.00353357 | |
| 284 53.16901408 | |
| 285 53.33333333 | |
| 286 53.49650350 | |
| 287 53.31010453 | |
| 288 53.47222222 | |
| 289 53.28719723 | |
| 290 53.44827586 | |
| 291 53.26460481 | |
| 292 53.42465753 | |
| 293 53.58361775 | |
| 294 53.40136054 | |
| 295 53.55932203 | |
| 296 53.71621622 | |
| 297 53.87205387 | |
| 298 54.02684564 | |
| 299 53.84615385 | |
| 300 54.00000000 | |
| 301 53.82059801 | |
| 302 53.97350993 | |
| 303 54.12541254 | |
| 304 54.27631579 | |
| 305 54.42622951 | |
| 306 54.24836601 | |
| 307 54.39739414 | |
| 308 54.54545455 | |
| 309 54.69255663 | |
| 310 54.83870968 | |
| 311 54.98392283 | |
| 312 54.80769231 | |
| 313 54.95207668 | |
| 314 54.77707006 | |
| 315 54.60317460 | |
| 316 54.74683544 | |
| 317 54.88958991 | |
| 318 54.71698113 | |
| 319 54.54545455 | |
| 320 54.37500000 | |
| 321 54.51713396 | |
| 322 54.65838509 | |
| 323 54.48916409 | |
| 324 54.32098765 | |
| 325 54.46153846 | |
| 326 54.60122699 | |
| 327 54.43425076 | |
| 328 54.57317073 | |
| 329 54.40729483 | |
| 330 54.24242424 | |
| 331 54.38066465 | |
| 332 54.51807229 | |
| 333 54.65465465 | |
| 334 54.79041916 | |
| 335 54.62686567 | |
| 336 54.46428571 | |
| 337 54.30267062 | |
| 338 54.14201183 | |
| 339 54.27728614 | |
| 340 54.41176471 | |
| 341 54.25219941 | |
| 342 54.09356725 | |
| 343 54.22740525 | |
| 344 54.36046512 | |
| 345 54.20289855 | |
| 346 54.33526012 | |
| 347 54.17867435 | |
| 348 54.02298851 | |
| 349 53.86819484 | |
| 350 54.00000000 | |
| 351 54.13105413 | |
| 352 54.26136364 | |
| 353 54.10764873 | |
| 354 53.95480226 | |
| 355 53.80281690 | |
| 356 53.93258427 | |
| 357 53.78151261 | |
| 358 53.91061453 | |
| 359 53.76044568 | |
| 360 53.88888889 | |
| 361 54.01662050 | |
| 362 54.14364641 | |
| 363 53.99449036 | |
| 364 53.84615385 | |
| 365 53.69863014 | |
| 366 53.55191257 | |
| 367 53.40599455 | |
| 368 53.53260870 | |
| 369 53.38753388 | |
| 370 53.24324324 | |
| 371 53.36927224 | |
| 372 53.22580645 | |
| 373 53.08310992 | |
| 374 53.20855615 | |
| 375 53.33333333 | |
| 376 53.19148936 | |
| 377 53.31564987 | |
| 378 53.43915344 | |
| 379 53.29815303 | |
| 380 53.42105263 | |
| 381 53.28083990 | |
| 382 53.14136126 | |
| 383 53.00261097 | |
| 384 52.86458333 | |
| 385 52.72727273 | |
| 386 52.84974093 | |
| 387 52.71317829 | |
| 388 52.57731959 | |
| 389 52.69922879 | |
| 390 52.56410256 | |
| 391 52.42966752 | |
| 392 52.55102041 | |
| 393 52.41730280 | |
| 394 52.53807107 | |
| 395 52.65822785 | |
| 396 52.52525253 | |
| 397 52.39294710 | |
| 398 52.26130653 | |
| 399 52.38095238 | |
| 400 52.50000000 | |
| 401 52.36907731 | |
| 402 52.48756219 | |
| 403 52.60545906 | |
| 404 52.72277228 | |
| 405 52.83950617 | |
| 406 52.95566502 | |
| 407 53.07125307 | |
| 408 53.18627451 | |
| 409 53.05623472 | |
| 410 53.17073171 | |
| 411 53.28467153 | |
| 412 53.39805825 | |
| 413 53.26876513 | |
| 414 53.38164251 | |
| 415 53.25301205 | |
| 416 53.12500000 | |
| 417 52.99760192 | |
| 418 52.87081340 | |
| 419 52.74463007 | |
| 420 52.61904762 | |
| 421 52.49406176 | |
| 422 52.60663507 | |
| 423 52.71867612 | |
| 424 52.59433962 | |
| 425 52.47058824 | |
| 426 52.34741784 | |
| 427 52.45901639 | |
| 428 52.33644860 | |
| 429 52.44755245 | |
| 430 52.32558140 | |
| 431 52.20417633 | |
| 432 52.31481481 | |
| 433 52.19399538 | |
| 434 52.30414747 | |
| 435 52.41379310 | |
| 436 52.29357798 | |
| 437 52.17391304 | |
| 438 52.05479452 | |
| 439 52.16400911 | |
| 440 52.04545455 | |
| 441 52.15419501 | |
| 442 52.03619910 | |
| 443 51.91873589 | |
| 444 51.80180180 | |
| 445 51.68539326 | |
| 446 51.79372197 | |
| 447 51.67785235 | |
| 448 51.78571429 | |
| 449 51.89309577 | |
| 450 51.77777778 | |
| 451 51.66297118 | |
| 452 51.54867257 | |
| 453 51.43487859 | |
| 454 51.32158590 | |
| 455 51.42857143 | |
| 456 51.31578947 | |
| 457 51.20350109 | |
| 458 51.31004367 | |
| 459 51.41612200 | |
| 460 51.30434783 | |
| 461 51.19305857 | |
| 462 51.29870130 | |
| 463 51.18790497 | |
| 464 51.07758621 | |
| 465 50.96774194 | |
| 466 50.85836910 | |
| 467 50.96359743 | |
| 468 51.06837607 | |
| 469 50.95948827 | |
| 470 50.85106383 | |
| 471 50.95541401 | |
| 472 51.05932203 | |
| 473 50.95137421 | |
| 474 50.84388186 | |
| 475 50.73684211 | |
| 476 50.63025210 | |
| 477 50.73375262 | |
| 478 50.83682008 | |
| 479 50.93945720 | |
| 480 51.04166667 | |
| 481 50.93555094 | |
| 482 51.03734440 | |
| 483 51.13871636 | |
| 484 51.23966942 | |
| 485 51.34020619 | |
| 486 51.44032922 | |
| 487 51.33470226 | |
| 488 51.43442623 | |
| 489 51.32924335 | |
| 490 51.22448980 | |
| 491 51.12016293 | |
| 492 51.01626016 | |
| 493 50.91277890 | |
| 494 51.01214575 | |
| 495 50.90909091 | |
| 496 50.80645161 | |
| 497 50.90543260 | |
| 498 51.00401606 | |
| 499 51.10220441 | |
| 500 51.20000000 | |
| 501 51.09780439 | |
| 502 51.19521912 | |
| 503 51.29224652 | |
| 504 51.19047619 | |
| 505 51.28712871 | |
| 506 51.18577075 | |
| 507 51.08481262 | |
| 508 50.98425197 | |
| 509 50.88408644 | |
| 510 50.78431373 | |
| 511 50.88062622 | |
| 512 50.97656250 | |
| 513 50.87719298 | |
| 514 50.97276265 | |
| 515 51.06796117 | |
| 516 50.96899225 | |
| 517 50.87040619 | |
| 518 50.77220077 | |
| 519 50.67437380 | |
| 520 50.57692308 | |
| 521 50.67178503 | |
| 522 50.76628352 | |
| 523 50.66921606 | |
| 524 50.57251908 | |
| 525 50.47619048 | |
| 526 50.38022814 | |
| 527 50.28462998 | |
| 528 50.18939394 | |
| 529 50.09451796 | |
| 530 50.18867925 | |
| 531 50.09416196 | |
| 532 50.18796992 | |
| 533 50.28142589 | |
| 534 50.18726592 | |
| 535 50.09345794 | |
| 536 50.18656716 | |
| 537 50.27932961 | |
| 538 50.18587361 | |
| 539 50.27829314 | |
| 540 50.18518519 | |
| 541 50.09242144 | |
| 542 50.18450185 | |
| 543 50.27624309 | |
| 544 50.18382353 | |
| 545 50.27522936 | |
| 546 50.36630037 | |
| 547 50.27422303 | |
| 548 50.36496350 | |
| 549 50.27322404 | |
| 550 50.36363636 | |
| 551 50.27223230 | |
| 552 50.36231884 | |
| 553 50.45207957 | |
| 554 50.54151625 | |
| 555 50.63063063 | |
| 556 50.53956835 | |
| 557 50.62836625 | |
| 558 50.53763441 | |
| 559 50.62611807 | |
| 560 50.71428571 | |
| 561 50.80213904 | |
| 562 50.88967972 | |
| 563 50.97690941 | |
| 564 50.88652482 | |
| 565 50.79646018 | |
| 566 50.88339223 | |
| 567 50.79365079 | |
| 568 50.88028169 | |
| 569 50.79086116 | |
| 570 50.70175439 | |
| 571 50.78809107 | |
| 572 50.69930070 | |
| 573 50.61082024 | |
| 574 50.69686411 | |
| 575 50.60869565 | |
| 576 50.69444444 | |
| 577 50.77989601 | |
| 578 50.86505190 | |
| 579 50.94991364 | |
| 580 51.03448276 | |
| 581 50.94664372 | |
| 582 50.85910653 | |
| 583 50.77186964 | |
| 584 50.85616438 | |
| 585 50.94017094 | |
| 586 50.85324232 | |
| 587 50.76660988 | |
| 588 50.85034014 | |
| 589 50.93378608 | |
| 590 51.01694915 | |
| 591 51.09983080 | |
| 592 51.01351351 | |
| 593 50.92748735 | |
| 594 51.01010101 | |
| 595 51.09243697 | |
| 596 51.00671141 | |
| 597 50.92127303 | |
| 598 50.83612040 | |
| 599 50.75125209 | |
| 600 50.83333333 | |
| 601 50.74875208 | |
| 602 50.66445183 | |
| 603 50.58043118 | |
| 604 50.49668874 | |
| 605 50.57851240 | |
| 606 50.66006601 | |
| 607 50.74135091 | |
| 608 50.82236842 | |
| 609 50.73891626 | |
| 610 50.81967213 | |
| 611 50.73649755 | |
| 612 50.81699346 | |
| 613 50.73409462 | |
| 614 50.65146580 | |
| 615 50.73170732 | |
| 616 50.64935065 | |
| 617 50.72933549 | |
| 618 50.64724919 | |
| 619 50.56542811 | |
| 620 50.64516129 | |
| 621 50.72463768 | |
| 622 50.64308682 | |
| 623 50.72231140 | |
| 624 50.80128205 | |
| 625 50.72000000 | |
| 626 50.63897764 | |
| 627 50.71770335 | |
| 628 50.79617834 | |
| 629 50.71542130 | |
| 630 50.79365079 | |
| 631 50.87163233 | |
| 632 50.79113924 | |
| 633 50.71090047 | |
| 634 50.78864353 | |
| 635 50.86614173 | |
| 636 50.94339623 | |
| 637 50.86342229 | |
| 638 50.94043887 | |
| 639 50.86071987 | |
| 640 50.78125000 | |
| 641 50.85803432 | |
| 642 50.77881620 | |
| 643 50.85536547 | |
| 644 50.93167702 | |
| 645 50.85271318 | |
| 646 50.92879257 | |
| 647 51.00463679 | |
| 648 50.92592593 | |
| 649 50.84745763 | |
| 650 50.92307692 | |
| 651 50.99846390 | |
| 652 50.92024540 | |
| 653 50.99540582 | |
| 654 51.07033639 | |
| 655 50.99236641 | |
| 656 50.91463415 | |
| 657 50.83713851 | |
| 658 50.91185410 | |
| 659 50.83459788 | |
| 660 50.90909091 | |
| 661 50.83207262 | |
| 662 50.90634441 | |
| 663 50.82956259 | |
| 664 50.90361446 | |
| 665 50.97744361 | |
| 666 50.90090090 | |
| 667 50.97451274 | |
| 668 50.89820359 | |
| 669 50.82212257 | |
| 670 50.89552239 | |
| 671 50.96870343 | |
| 672 50.89285714 | |
| 673 50.81723626 | |
| 674 50.74183976 | |
| 675 50.66666667 | |
| 676 50.59171598 | |
| 677 50.66469719 | |
| 678 50.58997050 | |
| 679 50.51546392 | |
| 680 50.58823529 | |
| 681 50.51395007 | |
| 682 50.43988270 | |
| 683 50.36603221 | |
| 684 50.29239766 | |
| 685 50.36496350 | |
| 686 50.29154519 | |
| 687 50.21834061 | |
| 688 50.29069767 | |
| 689 50.36284470 | |
| 690 50.28985507 | |
| 691 50.21707670 | |
| 692 50.28901734 | |
| 693 50.21645022 | |
| 694 50.14409222 | |
| 695 50.07194245 | |
| 696 50.00000000 | |
| 697 49.92826399 | |
| 698 49.85673352 | |
| 699 49.78540773 | |
| 700 49.85714286 | |
| 701 49.92867332 | |
| 702 50.00000000 | |
| 703 49.92887624 | |
| 704 50.00000000 | |
| 705 50.07092199 | |
| 706 50.14164306 | |
| 707 50.21216407 | |
| 708 50.28248588 | |
| 709 50.35260931 | |
| 710 50.28169014 | |
| 711 50.35161744 | |
| 712 50.42134831 | |
| 713 50.35063114 | |
| 714 50.28011204 | |
| 715 50.20979021 | |
| 716 50.13966480 | |
| 717 50.06973501 | |
| 718 50.00000000 | |
| 719 49.93045897 | |
| 720 50.00000000 | |
| 721 50.06934813 | |
| 722 50.13850416 | |
| 723 50.20746888 | |
| 724 50.27624309 | |
| 725 50.34482759 | |
| 726 50.27548209 | |
| 727 50.20632737 | |
| 728 50.13736264 | |
| 729 50.20576132 | |
| 730 50.27397260 | |
| 731 50.34199726 | |
| 732 50.40983607 | |
| 733 50.47748977 | |
| 734 50.54495913 | |
| 735 50.47619048 | |
| 736 50.54347826 | |
| 737 50.47489824 | |
| 738 50.40650407 | |
| 739 50.33829499 | |
| 740 50.27027027 | |
| 741 50.33738192 | |
| 742 50.26954178 | |
| 743 50.33647376 | |
| 744 50.26881720 | |
| 745 50.20134228 | |
| 746 50.13404826 | |
| 747 50.06693440 | |
| 748 50.00000000 | |
| 749 49.93324433 | |
| 750 50.00000000 | |
| Final result: 50.0000 +/- 1.8270 | |
| Random chance: 25.0083 +/- 1.5824 | |