| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 1548 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 1548 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 50.00000000 | |
| 5 40.00000000 | |
| 6 33.33333333 | |
| 7 42.85714286 | |
| 8 50.00000000 | |
| 9 44.44444444 | |
| 10 50.00000000 | |
| 11 45.45454545 | |
| 12 41.66666667 | |
| 13 38.46153846 | |
| 14 35.71428571 | |
| 15 33.33333333 | |
| 16 37.50000000 | |
| 17 35.29411765 | |
| 18 33.33333333 | |
| 19 36.84210526 | |
| 20 40.00000000 | |
| 21 38.09523810 | |
| 22 36.36363636 | |
| 23 34.78260870 | |
| 24 33.33333333 | |
| 25 36.00000000 | |
| 26 34.61538462 | |
| 27 37.03703704 | |
| 28 39.28571429 | |
| 29 41.37931034 | |
| 30 43.33333333 | |
| 31 45.16129032 | |
| 32 46.87500000 | |
| 33 45.45454545 | |
| 34 47.05882353 | |
| 35 48.57142857 | |
| 36 47.22222222 | |
| 37 45.94594595 | |
| 38 44.73684211 | |
| 39 43.58974359 | |
| 40 45.00000000 | |
| 41 46.34146341 | |
| 42 45.23809524 | |
| 43 46.51162791 | |
| 44 45.45454545 | |
| 45 46.66666667 | |
| 46 45.65217391 | |
| 47 44.68085106 | |
| 48 45.83333333 | |
| 49 44.89795918 | |
| 50 46.00000000 | |
| 51 45.09803922 | |
| 52 46.15384615 | |
| 53 45.28301887 | |
| 54 44.44444444 | |
| 55 45.45454545 | |
| 56 46.42857143 | |
| 57 47.36842105 | |
| 58 46.55172414 | |
| 59 45.76271186 | |
| 60 45.00000000 | |
| 61 45.90163934 | |
| 62 46.77419355 | |
| 63 46.03174603 | |
| 64 45.31250000 | |
| 65 44.61538462 | |
| 66 43.93939394 | |
| 67 44.77611940 | |
| 68 44.11764706 | |
| 69 43.47826087 | |
| 70 44.28571429 | |
| 71 45.07042254 | |
| 72 45.83333333 | |
| 73 45.20547945 | |
| 74 45.94594595 | |
| 75 45.33333333 | |
| 76 46.05263158 | |
| 77 46.75324675 | |
| 78 47.43589744 | |
| 79 46.83544304 | |
| 80 46.25000000 | |
| 81 46.91358025 | |
| 82 46.34146341 | |
| 83 46.98795181 | |
| 84 46.42857143 | |
| 85 45.88235294 | |
| 86 45.34883721 | |
| 87 45.97701149 | |
| 88 46.59090909 | |
| 89 46.06741573 | |
| 90 45.55555556 | |
| 91 45.05494505 | |
| 92 44.56521739 | |
| 93 44.08602151 | |
| 94 43.61702128 | |
| 95 44.21052632 | |
| 96 44.79166667 | |
| 97 44.32989691 | |
| 98 43.87755102 | |
| 99 43.43434343 | |
| 100 43.00000000 | |
| 101 42.57425743 | |
| 102 42.15686275 | |
| 103 41.74757282 | |
| 104 42.30769231 | |
| 105 42.85714286 | |
| 106 42.45283019 | |
| 107 42.05607477 | |
| 108 41.66666667 | |
| 109 41.28440367 | |
| 110 40.90909091 | |
| 111 40.54054054 | |
| 112 40.17857143 | |
| 113 39.82300885 | |
| 114 40.35087719 | |
| 115 40.00000000 | |
| 116 39.65517241 | |
| 117 40.17094017 | |
| 118 39.83050847 | |
| 119 39.49579832 | |
| 120 39.16666667 | |
| 121 39.66942149 | |
| 122 39.34426230 | |
| 123 39.02439024 | |
| 124 38.70967742 | |
| 125 39.20000000 | |
| 126 38.88888889 | |
| 127 38.58267717 | |
| 128 38.28125000 | |
| 129 37.98449612 | |
| 130 37.69230769 | |
| 131 38.16793893 | |
| 132 38.63636364 | |
| 133 38.34586466 | |
| 134 38.05970149 | |
| 135 37.77777778 | |
| 136 37.50000000 | |
| 137 37.22627737 | |
| 138 36.95652174 | |
| 139 37.41007194 | |
| 140 37.85714286 | |
| 141 38.29787234 | |
| 142 38.73239437 | |
| 143 38.46153846 | |
| 144 38.19444444 | |
| 145 38.62068966 | |
| 146 38.35616438 | |
| 147 38.09523810 | |
| 148 38.51351351 | |
| 149 38.25503356 | |
| 150 38.00000000 | |
| 151 38.41059603 | |
| 152 38.15789474 | |
| 153 37.90849673 | |
| 154 37.66233766 | |
| 155 37.41935484 | |
| 156 37.17948718 | |
| 157 37.57961783 | |
| 158 37.34177215 | |
| 159 37.10691824 | |
| 160 37.50000000 | |
| 161 37.26708075 | |
| 162 37.03703704 | |
| 163 37.42331288 | |
| 164 37.80487805 | |
| 165 38.18181818 | |
| 166 37.95180723 | |
| 167 38.32335329 | |
| 168 38.09523810 | |
| 169 38.46153846 | |
| 170 38.23529412 | |
| 171 38.59649123 | |
| 172 38.37209302 | |
| 173 38.15028902 | |
| 174 37.93103448 | |
| 175 37.71428571 | |
| 176 37.50000000 | |
| 177 37.28813559 | |
| 178 37.07865169 | |
| 179 36.87150838 | |
| 180 36.66666667 | |
| 181 37.01657459 | |
| 182 36.81318681 | |
| 183 37.15846995 | |
| 184 36.95652174 | |
| 185 37.29729730 | |
| 186 37.63440860 | |
| 187 37.96791444 | |
| 188 38.29787234 | |
| 189 38.09523810 | |
| 190 37.89473684 | |
| 191 38.21989529 | |
| 192 38.02083333 | |
| 193 37.82383420 | |
| 194 38.14432990 | |
| 195 38.46153846 | |
| 196 38.26530612 | |
| 197 38.07106599 | |
| 198 37.87878788 | |
| 199 37.68844221 | |
| 200 37.50000000 | |
| 201 37.31343284 | |
| 202 37.12871287 | |
| 203 36.94581281 | |
| 204 36.76470588 | |
| 205 37.07317073 | |
| 206 36.89320388 | |
| 207 37.19806763 | |
| 208 37.01923077 | |
| 209 36.84210526 | |
| 210 36.66666667 | |
| 211 36.96682464 | |
| 212 36.79245283 | |
| 213 36.61971831 | |
| 214 36.44859813 | |
| 215 36.74418605 | |
| 216 36.57407407 | |
| 217 36.86635945 | |
| 218 36.69724771 | |
| 219 36.52968037 | |
| 220 36.36363636 | |
| 221 36.19909502 | |
| 222 36.03603604 | |
| 223 36.32286996 | |
| 224 36.60714286 | |
| 225 36.88888889 | |
| 226 36.72566372 | |
| 227 36.56387665 | |
| 228 36.84210526 | |
| 229 36.68122271 | |
| 230 36.52173913 | |
| 231 36.36363636 | |
| 232 36.63793103 | |
| 233 36.90987124 | |
| 234 37.17948718 | |
| 235 37.02127660 | |
| 236 36.86440678 | |
| 237 36.70886076 | |
| 238 36.55462185 | |
| 239 36.82008368 | |
| 240 36.66666667 | |
| 241 36.51452282 | |
| 242 36.77685950 | |
| 243 36.62551440 | |
| 244 36.47540984 | |
| 245 36.32653061 | |
| 246 36.17886179 | |
| 247 36.03238866 | |
| 248 35.88709677 | |
| 249 35.74297189 | |
| 250 35.60000000 | |
| 251 35.85657371 | |
| 252 35.71428571 | |
| 253 35.57312253 | |
| 254 35.43307087 | |
| 255 35.68627451 | |
| 256 35.93750000 | |
| 257 35.79766537 | |
| 258 35.65891473 | |
| 259 35.52123552 | |
| 260 35.38461538 | |
| 261 35.24904215 | |
| 262 35.11450382 | |
| 263 34.98098859 | |
| 264 34.84848485 | |
| 265 34.71698113 | |
| 266 34.58646617 | |
| 267 34.45692884 | |
| 268 34.32835821 | |
| 269 34.57249071 | |
| 270 34.44444444 | |
| 271 34.31734317 | |
| 272 34.55882353 | |
| 273 34.43223443 | |
| 274 34.30656934 | |
| 275 34.18181818 | |
| 276 34.42028986 | |
| 277 34.65703971 | |
| 278 34.53237410 | |
| 279 34.40860215 | |
| 280 34.64285714 | |
| 281 34.51957295 | |
| 282 34.39716312 | |
| 283 34.27561837 | |
| 284 34.15492958 | |
| 285 34.03508772 | |
| 286 33.91608392 | |
| 287 33.79790941 | |
| 288 33.68055556 | |
| 289 33.56401384 | |
| 290 33.79310345 | |
| 291 34.02061856 | |
| 292 34.24657534 | |
| 293 34.12969283 | |
| 294 34.01360544 | |
| 295 33.89830508 | |
| 296 33.78378378 | |
| 297 33.67003367 | |
| 298 33.89261745 | |
| 299 33.77926421 | |
| 300 34.00000000 | |
| 301 34.21926910 | |
| 302 34.43708609 | |
| 303 34.32343234 | |
| 304 34.53947368 | |
| 305 34.75409836 | |
| 306 34.64052288 | |
| 307 34.85342020 | |
| 308 34.74025974 | |
| 309 34.62783172 | |
| 310 34.51612903 | |
| 311 34.72668810 | |
| 312 34.93589744 | |
| 313 35.14376997 | |
| 314 35.03184713 | |
| 315 34.92063492 | |
| 316 34.81012658 | |
| 317 35.01577287 | |
| 318 34.90566038 | |
| 319 35.10971787 | |
| 320 35.00000000 | |
| 321 34.89096573 | |
| 322 34.78260870 | |
| 323 34.67492260 | |
| 324 34.56790123 | |
| 325 34.76923077 | |
| 326 34.66257669 | |
| 327 34.86238532 | |
| 328 34.75609756 | |
| 329 34.65045593 | |
| 330 34.54545455 | |
| 331 34.44108761 | |
| 332 34.63855422 | |
| 333 34.53453453 | |
| 334 34.43113772 | |
| 335 34.32835821 | |
| 336 34.22619048 | |
| 337 34.12462908 | |
| 338 34.02366864 | |
| 339 33.92330383 | |
| 340 34.11764706 | |
| 341 34.01759531 | |
| 342 34.21052632 | |
| 343 34.11078717 | |
| 344 34.01162791 | |
| 345 33.91304348 | |
| 346 34.10404624 | |
| 347 34.29394813 | |
| 348 34.19540230 | |
| 349 34.38395415 | |
| 350 34.28571429 | |
| 351 34.18803419 | |
| 352 34.09090909 | |
| 353 34.27762040 | |
| 354 34.18079096 | |
| 355 34.08450704 | |
| 356 34.26966292 | |
| 357 34.45378151 | |
| 358 34.35754190 | |
| 359 34.26183844 | |
| 360 34.44444444 | |
| 361 34.62603878 | |
| 362 34.80662983 | |
| 363 34.98622590 | |
| 364 34.89010989 | |
| 365 34.79452055 | |
| 366 34.97267760 | |
| 367 34.87738420 | |
| 368 34.78260870 | |
| 369 34.68834688 | |
| 370 34.86486486 | |
| 371 34.77088949 | |
| 372 34.67741935 | |
| 373 34.58445040 | |
| 374 34.49197861 | |
| 375 34.66666667 | |
| 376 34.57446809 | |
| 377 34.74801061 | |
| 378 34.92063492 | |
| 379 34.82849604 | |
| 380 34.73684211 | |
| 381 34.64566929 | |
| 382 34.55497382 | |
| 383 34.72584856 | |
| 384 34.63541667 | |
| 385 34.54545455 | |
| 386 34.45595855 | |
| 387 34.36692506 | |
| 388 34.27835052 | |
| 389 34.44730077 | |
| 390 34.61538462 | |
| 391 34.78260870 | |
| 392 34.94897959 | |
| 393 34.86005089 | |
| 394 34.77157360 | |
| 395 34.93670886 | |
| 396 35.10101010 | |
| 397 35.26448363 | |
| 398 35.17587940 | |
| 399 35.33834586 | |
| 400 35.25000000 | |
| 401 35.41147132 | |
| 402 35.57213930 | |
| 403 35.48387097 | |
| 404 35.39603960 | |
| 405 35.30864198 | |
| 406 35.22167488 | |
| 407 35.13513514 | |
| 408 35.04901961 | |
| 409 35.20782396 | |
| 410 35.12195122 | |
| 411 35.03649635 | |
| 412 34.95145631 | |
| 413 34.86682809 | |
| 414 35.02415459 | |
| 415 35.18072289 | |
| 416 35.33653846 | |
| 417 35.25179856 | |
| 418 35.40669856 | |
| 419 35.56085919 | |
| 420 35.47619048 | |
| 421 35.39192399 | |
| 422 35.30805687 | |
| 423 35.22458629 | |
| 424 35.14150943 | |
| 425 35.05882353 | |
| 426 34.97652582 | |
| 427 34.89461358 | |
| 428 35.04672897 | |
| 429 35.19813520 | |
| 430 35.11627907 | |
| 431 35.26682135 | |
| 432 35.41666667 | |
| 433 35.33487298 | |
| 434 35.25345622 | |
| 435 35.17241379 | |
| 436 35.09174312 | |
| 437 35.24027460 | |
| 438 35.15981735 | |
| 439 35.07972665 | |
| 440 35.00000000 | |
| 441 35.14739229 | |
| 442 35.06787330 | |
| 443 34.98871332 | |
| 444 34.90990991 | |
| 445 34.83146067 | |
| 446 34.75336323 | |
| 447 34.67561521 | |
| 448 34.59821429 | |
| 449 34.52115813 | |
| 450 34.44444444 | |
| 451 34.36807095 | |
| 452 34.29203540 | |
| 453 34.43708609 | |
| 454 34.36123348 | |
| 455 34.28571429 | |
| 456 34.42982456 | |
| 457 34.35448578 | |
| 458 34.27947598 | |
| 459 34.20479303 | |
| 460 34.34782609 | |
| 461 34.27331887 | |
| 462 34.41558442 | |
| 463 34.34125270 | |
| 464 34.48275862 | |
| 465 34.40860215 | |
| 466 34.33476395 | |
| 467 34.47537473 | |
| 468 34.40170940 | |
| 469 34.54157783 | |
| 470 34.46808511 | |
| 471 34.39490446 | |
| 472 34.32203390 | |
| 473 34.46088795 | |
| 474 34.38818565 | |
| 475 34.31578947 | |
| 476 34.45378151 | |
| 477 34.38155136 | |
| 478 34.51882845 | |
| 479 34.44676409 | |
| 480 34.37500000 | |
| 481 34.30353430 | |
| 482 34.23236515 | |
| 483 34.16149068 | |
| 484 34.09090909 | |
| 485 34.02061856 | |
| 486 33.95061728 | |
| 487 34.08624230 | |
| 488 34.01639344 | |
| 489 33.94683027 | |
| 490 34.08163265 | |
| 491 34.01221996 | |
| 492 33.94308943 | |
| 493 33.87423935 | |
| 494 34.00809717 | |
| 495 34.14141414 | |
| 496 34.27419355 | |
| 497 34.20523139 | |
| 498 34.13654618 | |
| 499 34.06813627 | |
| 500 34.00000000 | |
| 501 33.93213573 | |
| 502 34.06374502 | |
| 503 33.99602386 | |
| 504 33.92857143 | |
| 505 33.86138614 | |
| 506 33.99209486 | |
| 507 33.92504931 | |
| 508 34.05511811 | |
| 509 33.98821218 | |
| 510 33.92156863 | |
| 511 33.85518591 | |
| 512 33.78906250 | |
| 513 33.72319688 | |
| 514 33.65758755 | |
| 515 33.78640777 | |
| 516 33.91472868 | |
| 517 33.84912959 | |
| 518 33.97683398 | |
| 519 33.91136802 | |
| 520 33.84615385 | |
| 521 33.78119002 | |
| 522 33.90804598 | |
| 523 33.84321224 | |
| 524 33.96946565 | |
| 525 33.90476190 | |
| 526 33.84030418 | |
| 527 33.96584440 | |
| 528 34.09090909 | |
| 529 34.21550095 | |
| 530 34.33962264 | |
| 531 34.27495292 | |
| 532 34.21052632 | |
| 533 34.33395872 | |
| 534 34.45692884 | |
| 535 34.57943925 | |
| 536 34.70149254 | |
| 537 34.82309125 | |
| 538 34.94423792 | |
| 539 34.87940631 | |
| 540 34.81481481 | |
| 541 34.75046211 | |
| 542 34.87084871 | |
| 543 34.80662983 | |
| 544 34.74264706 | |
| 545 34.67889908 | |
| 546 34.61538462 | |
| 547 34.55210238 | |
| 548 34.67153285 | |
| 549 34.60837887 | |
| 550 34.54545455 | |
| 551 34.66424682 | |
| 552 34.60144928 | |
| 553 34.53887884 | |
| 554 34.47653430 | |
| 555 34.41441441 | |
| 556 34.53237410 | |
| 557 34.64991023 | |
| 558 34.76702509 | |
| 559 34.70483005 | |
| 560 34.64285714 | |
| 561 34.58110517 | |
| 562 34.69750890 | |
| 563 34.81349911 | |
| 564 34.75177305 | |
| 565 34.86725664 | |
| 566 34.80565371 | |
| 567 34.74426808 | |
| 568 34.68309859 | |
| 569 34.62214411 | |
| 570 34.73684211 | |
| 571 34.85113835 | |
| 572 34.79020979 | |
| 573 34.90401396 | |
| 574 34.84320557 | |
| 575 34.78260870 | |
| 576 34.89583333 | |
| 577 34.83535529 | |
| 578 34.77508651 | |
| 579 34.71502591 | |
| 580 34.82758621 | |
| 581 34.76764200 | |
| 582 34.70790378 | |
| 583 34.81989708 | |
| 584 34.93150685 | |
| 585 35.04273504 | |
| 586 34.98293515 | |
| 587 34.92333901 | |
| 588 34.86394558 | |
| 589 34.80475382 | |
| 590 34.74576271 | |
| 591 34.68697124 | |
| 592 34.62837838 | |
| 593 34.56998314 | |
| 594 34.68013468 | |
| 595 34.62184874 | |
| 596 34.73154362 | |
| 597 34.67336683 | |
| 598 34.61538462 | |
| 599 34.72454090 | |
| 600 34.66666667 | |
| 601 34.77537438 | |
| 602 34.71760797 | |
| 603 34.82587065 | |
| 604 34.76821192 | |
| 605 34.87603306 | |
| 606 34.81848185 | |
| 607 34.92586491 | |
| 608 34.86842105 | |
| 609 34.81116585 | |
| 610 34.75409836 | |
| 611 34.69721768 | |
| 612 34.64052288 | |
| 613 34.58401305 | |
| 614 34.52768730 | |
| 615 34.47154472 | |
| 616 34.41558442 | |
| 617 34.35980551 | |
| 618 34.30420712 | |
| 619 34.24878837 | |
| 620 34.19354839 | |
| 621 34.29951691 | |
| 622 34.24437299 | |
| 623 34.18940610 | |
| 624 34.29487179 | |
| 625 34.24000000 | |
| 626 34.34504792 | |
| 627 34.29027113 | |
| 628 34.23566879 | |
| 629 34.34022258 | |
| 630 34.44444444 | |
| 631 34.38985737 | |
| 632 34.49367089 | |
| 633 34.43917852 | |
| 634 34.38485804 | |
| 635 34.33070866 | |
| 636 34.27672956 | |
| 637 34.22291994 | |
| 638 34.16927900 | |
| 639 34.27230047 | |
| 640 34.37500000 | |
| 641 34.32137285 | |
| 642 34.26791277 | |
| 643 34.21461897 | |
| 644 34.16149068 | |
| 645 34.10852713 | |
| 646 34.05572755 | |
| 647 34.00309119 | |
| 648 33.95061728 | |
| 649 33.89830508 | |
| 650 33.84615385 | |
| 651 33.79416283 | |
| 652 33.74233129 | |
| 653 33.69065850 | |
| 654 33.79204893 | |
| 655 33.74045802 | |
| 656 33.84146341 | |
| 657 33.78995434 | |
| 658 33.89057751 | |
| 659 33.99089530 | |
| 660 33.93939394 | |
| 661 33.88804841 | |
| 662 33.98791541 | |
| 663 33.93665158 | |
| 664 33.88554217 | |
| 665 33.83458647 | |
| 666 33.78378378 | |
| 667 33.73313343 | |
| 668 33.83233533 | |
| 669 33.93124066 | |
| 670 33.88059701 | |
| 671 33.83010432 | |
| 672 33.77976190 | |
| 673 33.72956909 | |
| 674 33.67952522 | |
| 675 33.62962963 | |
| 676 33.57988166 | |
| 677 33.53028065 | |
| 678 33.48082596 | |
| 679 33.43151694 | |
| 680 33.52941176 | |
| 681 33.62701909 | |
| 682 33.57771261 | |
| 683 33.52855051 | |
| 684 33.62573099 | |
| 685 33.72262774 | |
| 686 33.67346939 | |
| 687 33.62445415 | |
| 688 33.57558140 | |
| 689 33.67198839 | |
| 690 33.62318841 | |
| 691 33.57452967 | |
| 692 33.52601156 | |
| 693 33.47763348 | |
| 694 33.42939481 | |
| 695 33.38129496 | |
| 696 33.47701149 | |
| 697 33.42898135 | |
| 698 33.38108883 | |
| 699 33.47639485 | |
| 700 33.42857143 | |
| 701 33.52353780 | |
| 702 33.61823362 | |
| 703 33.57041252 | |
| 704 33.66477273 | |
| 705 33.61702128 | |
| 706 33.56940510 | |
| 707 33.52192362 | |
| 708 33.47457627 | |
| 709 33.42736248 | |
| 710 33.52112676 | |
| 711 33.61462729 | |
| 712 33.56741573 | |
| 713 33.66058906 | |
| 714 33.75350140 | |
| 715 33.70629371 | |
| 716 33.65921788 | |
| 717 33.61227336 | |
| 718 33.56545961 | |
| 719 33.51877608 | |
| 720 33.47222222 | |
| 721 33.42579750 | |
| 722 33.37950139 | |
| 723 33.47164592 | |
| 724 33.42541436 | |
| 725 33.51724138 | |
| 726 33.47107438 | |
| 727 33.42503439 | |
| 728 33.37912088 | |
| 729 33.33333333 | |
| 730 33.28767123 | |
| 731 33.24213406 | |
| 732 33.19672131 | |
| 733 33.28785812 | |
| 734 33.37874659 | |
| 735 33.33333333 | |
| 736 33.28804348 | |
| 737 33.37856174 | |
| 738 33.33333333 | |
| 739 33.42354533 | |
| 740 33.37837838 | |
| 741 33.33333333 | |
| 742 33.28840970 | |
| 743 33.24360700 | |
| 744 33.33333333 | |
| 745 33.28859060 | |
| 746 33.24396783 | |
| 747 33.19946452 | |
| 748 33.15508021 | |
| 749 33.11081442 | |
| 750 33.20000000 | |
| Final result: 33.2000 +/- 1.7207 | |
| Random chance: 25.0000 +/- 1.5822 | |