| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 869 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 869 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 25.00000000 | |
| 5 40.00000000 | |
| 6 50.00000000 | |
| 7 57.14285714 | |
| 8 62.50000000 | |
| 9 55.55555556 | |
| 10 60.00000000 | |
| 11 54.54545455 | |
| 12 58.33333333 | |
| 13 53.84615385 | |
| 14 57.14285714 | |
| 15 60.00000000 | |
| 16 62.50000000 | |
| 17 64.70588235 | |
| 18 66.66666667 | |
| 19 63.15789474 | |
| 20 60.00000000 | |
| 21 61.90476190 | |
| 22 63.63636364 | |
| 23 60.86956522 | |
| 24 62.50000000 | |
| 25 60.00000000 | |
| 26 61.53846154 | |
| 27 62.96296296 | |
| 28 60.71428571 | |
| 29 62.06896552 | |
| 30 63.33333333 | |
| 31 61.29032258 | |
| 32 62.50000000 | |
| 33 60.60606061 | |
| 34 58.82352941 | |
| 35 57.14285714 | |
| 36 55.55555556 | |
| 37 56.75675676 | |
| 38 55.26315789 | |
| 39 56.41025641 | |
| 40 57.50000000 | |
| 41 56.09756098 | |
| 42 54.76190476 | |
| 43 53.48837209 | |
| 44 52.27272727 | |
| 45 53.33333333 | |
| 46 54.34782609 | |
| 47 55.31914894 | |
| 48 54.16666667 | |
| 49 53.06122449 | |
| 50 54.00000000 | |
| 51 54.90196078 | |
| 52 55.76923077 | |
| 53 56.60377358 | |
| 54 55.55555556 | |
| 55 54.54545455 | |
| 56 55.35714286 | |
| 57 56.14035088 | |
| 58 56.89655172 | |
| 59 57.62711864 | |
| 60 58.33333333 | |
| 61 59.01639344 | |
| 62 59.67741935 | |
| 63 60.31746032 | |
| 64 60.93750000 | |
| 65 60.00000000 | |
| 66 60.60606061 | |
| 67 61.19402985 | |
| 68 60.29411765 | |
| 69 59.42028986 | |
| 70 58.57142857 | |
| 71 57.74647887 | |
| 72 56.94444444 | |
| 73 57.53424658 | |
| 74 58.10810811 | |
| 75 58.66666667 | |
| 76 57.89473684 | |
| 77 58.44155844 | |
| 78 58.97435897 | |
| 79 58.22784810 | |
| 80 58.75000000 | |
| 81 58.02469136 | |
| 82 58.53658537 | |
| 83 57.83132530 | |
| 84 58.33333333 | |
| 85 58.82352941 | |
| 86 58.13953488 | |
| 87 57.47126437 | |
| 88 57.95454545 | |
| 89 58.42696629 | |
| 90 58.88888889 | |
| 91 59.34065934 | |
| 92 59.78260870 | |
| 93 59.13978495 | |
| 94 58.51063830 | |
| 95 58.94736842 | |
| 96 59.37500000 | |
| 97 58.76288660 | |
| 98 58.16326531 | |
| 99 58.58585859 | |
| 100 58.00000000 | |
| 101 57.42574257 | |
| 102 57.84313725 | |
| 103 57.28155340 | |
| 104 57.69230769 | |
| 105 58.09523810 | |
| 106 57.54716981 | |
| 107 57.94392523 | |
| 108 57.40740741 | |
| 109 56.88073394 | |
| 110 57.27272727 | |
| 111 57.65765766 | |
| 112 58.03571429 | |
| 113 57.52212389 | |
| 114 57.89473684 | |
| 115 58.26086957 | |
| 116 58.62068966 | |
| 117 58.97435897 | |
| 118 59.32203390 | |
| 119 58.82352941 | |
| 120 58.33333333 | |
| 121 57.85123967 | |
| 122 58.19672131 | |
| 123 58.53658537 | |
| 124 58.06451613 | |
| 125 58.40000000 | |
| 126 58.73015873 | |
| 127 58.26771654 | |
| 128 58.59375000 | |
| 129 58.13953488 | |
| 130 57.69230769 | |
| 131 58.01526718 | |
| 132 57.57575758 | |
| 133 57.89473684 | |
| 134 57.46268657 | |
| 135 57.03703704 | |
| 136 56.61764706 | |
| 137 56.93430657 | |
| 138 57.24637681 | |
| 139 56.83453237 | |
| 140 56.42857143 | |
| 141 56.02836879 | |
| 142 56.33802817 | |
| 143 56.64335664 | |
| 144 56.25000000 | |
| 145 56.55172414 | |
| 146 56.16438356 | |
| 147 56.46258503 | |
| 148 56.08108108 | |
| 149 56.37583893 | |
| 150 56.66666667 | |
| 151 56.95364238 | |
| 152 57.23684211 | |
| 153 56.86274510 | |
| 154 56.49350649 | |
| 155 56.12903226 | |
| 156 56.41025641 | |
| 157 56.68789809 | |
| 158 56.32911392 | |
| 159 56.60377358 | |
| 160 56.25000000 | |
| 161 55.90062112 | |
| 162 55.55555556 | |
| 163 55.21472393 | |
| 164 54.87804878 | |
| 165 54.54545455 | |
| 166 54.81927711 | |
| 167 54.49101796 | |
| 168 54.76190476 | |
| 169 54.43786982 | |
| 170 54.11764706 | |
| 171 54.38596491 | |
| 172 54.65116279 | |
| 173 54.91329480 | |
| 174 55.17241379 | |
| 175 54.85714286 | |
| 176 54.54545455 | |
| 177 54.80225989 | |
| 178 54.49438202 | |
| 179 54.18994413 | |
| 180 53.88888889 | |
| 181 53.59116022 | |
| 182 53.84615385 | |
| 183 53.55191257 | |
| 184 53.26086957 | |
| 185 53.51351351 | |
| 186 53.76344086 | |
| 187 53.47593583 | |
| 188 53.19148936 | |
| 189 52.91005291 | |
| 190 52.63157895 | |
| 191 52.87958115 | |
| 192 52.60416667 | |
| 193 52.33160622 | |
| 194 52.57731959 | |
| 195 52.82051282 | |
| 196 53.06122449 | |
| 197 53.29949239 | |
| 198 53.03030303 | |
| 199 53.26633166 | |
| 200 53.50000000 | |
| 201 53.73134328 | |
| 202 53.46534653 | |
| 203 53.20197044 | |
| 204 53.43137255 | |
| 205 53.17073171 | |
| 206 52.91262136 | |
| 207 52.65700483 | |
| 208 52.88461538 | |
| 209 52.63157895 | |
| 210 52.85714286 | |
| 211 53.08056872 | |
| 212 52.83018868 | |
| 213 53.05164319 | |
| 214 53.27102804 | |
| 215 53.48837209 | |
| 216 53.24074074 | |
| 217 53.45622120 | |
| 218 53.21100917 | |
| 219 52.96803653 | |
| 220 52.72727273 | |
| 221 52.48868778 | |
| 222 52.25225225 | |
| 223 52.01793722 | |
| 224 52.23214286 | |
| 225 52.00000000 | |
| 226 52.21238938 | |
| 227 52.42290749 | |
| 228 52.63157895 | |
| 229 52.83842795 | |
| 230 52.60869565 | |
| 231 52.38095238 | |
| 232 52.15517241 | |
| 233 51.93133047 | |
| 234 51.70940171 | |
| 235 51.48936170 | |
| 236 51.27118644 | |
| 237 51.05485232 | |
| 238 51.26050420 | |
| 239 51.46443515 | |
| 240 51.66666667 | |
| 241 51.86721992 | |
| 242 52.06611570 | |
| 243 51.85185185 | |
| 244 51.63934426 | |
| 245 51.83673469 | |
| 246 52.03252033 | |
| 247 52.22672065 | |
| 248 52.41935484 | |
| 249 52.20883534 | |
| 250 52.40000000 | |
| 251 52.58964143 | |
| 252 52.77777778 | |
| 253 52.96442688 | |
| 254 52.75590551 | |
| 255 52.94117647 | |
| 256 53.12500000 | |
| 257 53.30739300 | |
| 258 53.10077519 | |
| 259 52.89575290 | |
| 260 52.69230769 | |
| 261 52.49042146 | |
| 262 52.29007634 | |
| 263 52.47148289 | |
| 264 52.65151515 | |
| 265 52.83018868 | |
| 266 52.63157895 | |
| 267 52.80898876 | |
| 268 52.61194030 | |
| 269 52.41635688 | |
| 270 52.59259259 | |
| 271 52.76752768 | |
| 272 52.57352941 | |
| 273 52.38095238 | |
| 274 52.18978102 | |
| 275 52.00000000 | |
| 276 51.81159420 | |
| 277 51.98555957 | |
| 278 51.79856115 | |
| 279 51.97132616 | |
| 280 52.14285714 | |
| 281 52.31316726 | |
| 282 52.48226950 | |
| 283 52.29681979 | |
| 284 52.46478873 | |
| 285 52.63157895 | |
| 286 52.79720280 | |
| 287 52.61324042 | |
| 288 52.77777778 | |
| 289 52.59515571 | |
| 290 52.75862069 | |
| 291 52.57731959 | |
| 292 52.73972603 | |
| 293 52.90102389 | |
| 294 52.72108844 | |
| 295 52.88135593 | |
| 296 53.04054054 | |
| 297 53.19865320 | |
| 298 53.35570470 | |
| 299 53.17725753 | |
| 300 53.33333333 | |
| 301 53.15614618 | |
| 302 53.31125828 | |
| 303 53.46534653 | |
| 304 53.61842105 | |
| 305 53.77049180 | |
| 306 53.59477124 | |
| 307 53.74592834 | |
| 308 53.89610390 | |
| 309 54.04530744 | |
| 310 54.19354839 | |
| 311 54.34083601 | |
| 312 54.16666667 | |
| 313 54.31309904 | |
| 314 54.14012739 | |
| 315 53.96825397 | |
| 316 54.11392405 | |
| 317 54.25867508 | |
| 318 54.08805031 | |
| 319 53.91849530 | |
| 320 53.75000000 | |
| 321 53.58255452 | |
| 322 53.72670807 | |
| 323 53.56037152 | |
| 324 53.39506173 | |
| 325 53.23076923 | |
| 326 53.37423313 | |
| 327 53.21100917 | |
| 328 53.35365854 | |
| 329 53.49544073 | |
| 330 53.33333333 | |
| 331 53.47432024 | |
| 332 53.61445783 | |
| 333 53.75375375 | |
| 334 53.89221557 | |
| 335 53.73134328 | |
| 336 53.57142857 | |
| 337 53.41246291 | |
| 338 53.25443787 | |
| 339 53.39233038 | |
| 340 53.52941176 | |
| 341 53.37243402 | |
| 342 53.21637427 | |
| 343 53.35276968 | |
| 344 53.48837209 | |
| 345 53.33333333 | |
| 346 53.46820809 | |
| 347 53.31412104 | |
| 348 53.16091954 | |
| 349 53.00859599 | |
| 350 53.14285714 | |
| 351 53.27635328 | |
| 352 53.40909091 | |
| 353 53.25779037 | |
| 354 53.10734463 | |
| 355 52.95774648 | |
| 356 53.08988764 | |
| 357 52.94117647 | |
| 358 53.07262570 | |
| 359 52.92479109 | |
| 360 53.05555556 | |
| 361 53.18559557 | |
| 362 53.31491713 | |
| 363 53.16804408 | |
| 364 53.02197802 | |
| 365 52.87671233 | |
| 366 52.73224044 | |
| 367 52.58855586 | |
| 368 52.71739130 | |
| 369 52.57452575 | |
| 370 52.43243243 | |
| 371 52.29110512 | |
| 372 52.15053763 | |
| 373 52.01072386 | |
| 374 52.13903743 | |
| 375 52.00000000 | |
| 376 51.86170213 | |
| 377 51.98938992 | |
| 378 52.11640212 | |
| 379 51.97889182 | |
| 380 52.10526316 | |
| 381 51.96850394 | |
| 382 51.83246073 | |
| 383 51.69712794 | |
| 384 51.56250000 | |
| 385 51.42857143 | |
| 386 51.29533679 | |
| 387 51.16279070 | |
| 388 51.03092784 | |
| 389 50.89974293 | |
| 390 50.76923077 | |
| 391 50.63938619 | |
| 392 50.76530612 | |
| 393 50.63613232 | |
| 394 50.76142132 | |
| 395 50.88607595 | |
| 396 50.75757576 | |
| 397 50.62972292 | |
| 398 50.50251256 | |
| 399 50.62656642 | |
| 400 50.75000000 | |
| 401 50.62344140 | |
| 402 50.74626866 | |
| 403 50.86848635 | |
| 404 50.99009901 | |
| 405 51.11111111 | |
| 406 51.23152709 | |
| 407 51.35135135 | |
| 408 51.47058824 | |
| 409 51.34474328 | |
| 410 51.46341463 | |
| 411 51.58150852 | |
| 412 51.69902913 | |
| 413 51.57384988 | |
| 414 51.69082126 | |
| 415 51.56626506 | |
| 416 51.44230769 | |
| 417 51.31894484 | |
| 418 51.19617225 | |
| 419 51.07398568 | |
| 420 50.95238095 | |
| 421 51.06888361 | |
| 422 51.18483412 | |
| 423 51.30023641 | |
| 424 51.17924528 | |
| 425 51.05882353 | |
| 426 50.93896714 | |
| 427 51.05386417 | |
| 428 50.93457944 | |
| 429 51.04895105 | |
| 430 50.93023256 | |
| 431 50.81206497 | |
| 432 50.92592593 | |
| 433 50.80831409 | |
| 434 50.69124424 | |
| 435 50.80459770 | |
| 436 50.91743119 | |
| 437 50.80091533 | |
| 438 50.68493151 | |
| 439 50.79726651 | |
| 440 50.68181818 | |
| 441 50.79365079 | |
| 442 50.67873303 | |
| 443 50.56433409 | |
| 444 50.45045045 | |
| 445 50.33707865 | |
| 446 50.44843049 | |
| 447 50.33557047 | |
| 448 50.44642857 | |
| 449 50.55679287 | |
| 450 50.44444444 | |
| 451 50.33259424 | |
| 452 50.22123894 | |
| 453 50.11037528 | |
| 454 50.00000000 | |
| 455 49.89010989 | |
| 456 49.78070175 | |
| 457 49.67177243 | |
| 458 49.78165939 | |
| 459 49.89106754 | |
| 460 49.78260870 | |
| 461 49.67462039 | |
| 462 49.78354978 | |
| 463 49.67602592 | |
| 464 49.56896552 | |
| 465 49.46236559 | |
| 466 49.35622318 | |
| 467 49.46466809 | |
| 468 49.57264957 | |
| 469 49.46695096 | |
| 470 49.36170213 | |
| 471 49.46921444 | |
| 472 49.57627119 | |
| 473 49.47145877 | |
| 474 49.57805907 | |
| 475 49.47368421 | |
| 476 49.36974790 | |
| 477 49.47589099 | |
| 478 49.58158996 | |
| 479 49.68684760 | |
| 480 49.79166667 | |
| 481 49.89604990 | |
| 482 50.00000000 | |
| 483 50.10351967 | |
| 484 50.20661157 | |
| 485 50.30927835 | |
| 486 50.41152263 | |
| 487 50.30800821 | |
| 488 50.40983607 | |
| 489 50.30674847 | |
| 490 50.20408163 | |
| 491 50.10183299 | |
| 492 50.00000000 | |
| 493 49.89858012 | |
| 494 50.00000000 | |
| 495 49.89898990 | |
| 496 49.79838710 | |
| 497 49.89939638 | |
| 498 50.00000000 | |
| 499 50.10020040 | |
| 500 50.00000000 | |
| 501 49.90019960 | |
| 502 50.00000000 | |
| 503 50.09940358 | |
| 504 50.00000000 | |
| 505 50.09900990 | |
| 506 50.00000000 | |
| 507 49.90138067 | |
| 508 50.00000000 | |
| 509 49.90176817 | |
| 510 49.80392157 | |
| 511 49.90215264 | |
| 512 50.00000000 | |
| 513 49.90253411 | |
| 514 50.00000000 | |
| 515 50.09708738 | |
| 516 50.00000000 | |
| 517 49.90328820 | |
| 518 49.80694981 | |
| 519 49.71098266 | |
| 520 49.61538462 | |
| 521 49.71209213 | |
| 522 49.61685824 | |
| 523 49.52198853 | |
| 524 49.42748092 | |
| 525 49.33333333 | |
| 526 49.23954373 | |
| 527 49.14611006 | |
| 528 49.05303030 | |
| 529 48.96030246 | |
| 530 49.05660377 | |
| 531 48.96421846 | |
| 532 49.06015038 | |
| 533 49.15572233 | |
| 534 49.06367041 | |
| 535 48.97196262 | |
| 536 49.06716418 | |
| 537 49.16201117 | |
| 538 49.07063197 | |
| 539 49.16512059 | |
| 540 49.07407407 | |
| 541 48.98336414 | |
| 542 49.07749077 | |
| 543 49.17127072 | |
| 544 49.08088235 | |
| 545 49.17431193 | |
| 546 49.26739927 | |
| 547 49.17733090 | |
| 548 49.27007299 | |
| 549 49.18032787 | |
| 550 49.27272727 | |
| 551 49.18330309 | |
| 552 49.27536232 | |
| 553 49.36708861 | |
| 554 49.45848375 | |
| 555 49.54954955 | |
| 556 49.46043165 | |
| 557 49.55116697 | |
| 558 49.46236559 | |
| 559 49.55277281 | |
| 560 49.64285714 | |
| 561 49.73262032 | |
| 562 49.82206406 | |
| 563 49.91119005 | |
| 564 49.82269504 | |
| 565 49.73451327 | |
| 566 49.82332155 | |
| 567 49.73544974 | |
| 568 49.82394366 | |
| 569 49.73637961 | |
| 570 49.64912281 | |
| 571 49.73730298 | |
| 572 49.65034965 | |
| 573 49.56369983 | |
| 574 49.65156794 | |
| 575 49.56521739 | |
| 576 49.65277778 | |
| 577 49.74003466 | |
| 578 49.82698962 | |
| 579 49.91364421 | |
| 580 50.00000000 | |
| 581 49.91394148 | |
| 582 49.82817869 | |
| 583 49.74271012 | |
| 584 49.82876712 | |
| 585 49.91452991 | |
| 586 49.82935154 | |
| 587 49.74446337 | |
| 588 49.82993197 | |
| 589 49.91511036 | |
| 590 50.00000000 | |
| 591 50.08460237 | |
| 592 50.00000000 | |
| 593 49.91568297 | |
| 594 50.00000000 | |
| 595 49.91596639 | |
| 596 49.83221477 | |
| 597 49.74874372 | |
| 598 49.66555184 | |
| 599 49.58263773 | |
| 600 49.66666667 | |
| 601 49.58402662 | |
| 602 49.50166113 | |
| 603 49.41956882 | |
| 604 49.33774834 | |
| 605 49.42148760 | |
| 606 49.50495050 | |
| 607 49.58813839 | |
| 608 49.67105263 | |
| 609 49.58949097 | |
| 610 49.67213115 | |
| 611 49.59083470 | |
| 612 49.67320261 | |
| 613 49.59216966 | |
| 614 49.51140065 | |
| 615 49.59349593 | |
| 616 49.51298701 | |
| 617 49.59481361 | |
| 618 49.51456311 | |
| 619 49.43457189 | |
| 620 49.51612903 | |
| 621 49.59742351 | |
| 622 49.51768489 | |
| 623 49.59871589 | |
| 624 49.67948718 | |
| 625 49.60000000 | |
| 626 49.52076677 | |
| 627 49.60127592 | |
| 628 49.68152866 | |
| 629 49.60254372 | |
| 630 49.68253968 | |
| 631 49.76228209 | |
| 632 49.68354430 | |
| 633 49.60505529 | |
| 634 49.68454259 | |
| 635 49.60629921 | |
| 636 49.68553459 | |
| 637 49.60753532 | |
| 638 49.68652038 | |
| 639 49.76525822 | |
| 640 49.68750000 | |
| 641 49.76599064 | |
| 642 49.68847352 | |
| 643 49.76671851 | |
| 644 49.84472050 | |
| 645 49.76744186 | |
| 646 49.84520124 | |
| 647 49.92272025 | |
| 648 49.84567901 | |
| 649 49.76887519 | |
| 650 49.84615385 | |
| 651 49.92319508 | |
| 652 49.84662577 | |
| 653 49.92343032 | |
| 654 50.00000000 | |
| 655 49.92366412 | |
| 656 49.84756098 | |
| 657 49.77168950 | |
| 658 49.84802432 | |
| 659 49.77238240 | |
| 660 49.84848485 | |
| 661 49.77307110 | |
| 662 49.84894260 | |
| 663 49.77375566 | |
| 664 49.84939759 | |
| 665 49.92481203 | |
| 666 49.84984985 | |
| 667 49.92503748 | |
| 668 49.85029940 | |
| 669 49.77578475 | |
| 670 49.85074627 | |
| 671 49.92548435 | |
| 672 49.85119048 | |
| 673 49.77711738 | |
| 674 49.70326409 | |
| 675 49.62962963 | |
| 676 49.55621302 | |
| 677 49.63072378 | |
| 678 49.55752212 | |
| 679 49.48453608 | |
| 680 49.55882353 | |
| 681 49.48604993 | |
| 682 49.41348974 | |
| 683 49.34114202 | |
| 684 49.41520468 | |
| 685 49.48905109 | |
| 686 49.41690962 | |
| 687 49.34497817 | |
| 688 49.41860465 | |
| 689 49.49201742 | |
| 690 49.56521739 | |
| 691 49.49348770 | |
| 692 49.56647399 | |
| 693 49.49494949 | |
| 694 49.42363112 | |
| 695 49.35251799 | |
| 696 49.28160920 | |
| 697 49.21090387 | |
| 698 49.14040115 | |
| 699 49.07010014 | |
| 700 49.14285714 | |
| 701 49.07275321 | |
| 702 49.14529915 | |
| 703 49.07539118 | |
| 704 49.14772727 | |
| 705 49.21985816 | |
| 706 49.29178470 | |
| 707 49.36350778 | |
| 708 49.43502825 | |
| 709 49.50634697 | |
| 710 49.43661972 | |
| 711 49.50773558 | |
| 712 49.57865169 | |
| 713 49.50911641 | |
| 714 49.43977591 | |
| 715 49.37062937 | |
| 716 49.30167598 | |
| 717 49.23291492 | |
| 718 49.16434540 | |
| 719 49.09596662 | |
| 720 49.16666667 | |
| 721 49.09847434 | |
| 722 49.16897507 | |
| 723 49.23928077 | |
| 724 49.30939227 | |
| 725 49.37931034 | |
| 726 49.31129477 | |
| 727 49.24346630 | |
| 728 49.17582418 | |
| 729 49.10836763 | |
| 730 49.17808219 | |
| 731 49.24760602 | |
| 732 49.31693989 | |
| 733 49.38608458 | |
| 734 49.45504087 | |
| 735 49.38775510 | |
| 736 49.45652174 | |
| 737 49.38941655 | |
| 738 49.45799458 | |
| 739 49.39106901 | |
| 740 49.32432432 | |
| 741 49.39271255 | |
| 742 49.32614555 | |
| 743 49.39434724 | |
| 744 49.32795699 | |
| 745 49.26174497 | |
| 746 49.32975871 | |
| 747 49.26372155 | |
| 748 49.19786096 | |
| 749 49.13217623 | |
| 750 49.20000000 | |
| Final result: 49.2000 +/- 1.8267 | |
| Random chance: 25.0083 +/- 1.5824 | |