common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 817 tasks in prompt multiple_choice_score: selecting 750 random tasks from 817 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 25.00000000 5 20.00000000 6 16.66666667 7 14.28571429 8 12.50000000 9 11.11111111 10 10.00000000 11 9.09090909 12 16.66666667 13 23.07692308 14 21.42857143 15 26.66666667 16 31.25000000 17 29.41176471 18 27.77777778 19 31.57894737 20 30.00000000 21 28.57142857 22 27.27272727 23 26.08695652 24 29.16666667 25 28.00000000 26 26.92307692 27 25.92592593 28 25.00000000 29 27.58620690 30 26.66666667 31 25.80645161 32 28.12500000 33 30.30303030 34 29.41176471 35 28.57142857 36 27.77777778 37 29.72972973 38 28.94736842 39 30.76923077 40 30.00000000 41 29.26829268 42 28.57142857 43 30.23255814 44 31.81818182 45 33.33333333 46 32.60869565 47 34.04255319 48 35.41666667 49 34.69387755 50 36.00000000 51 35.29411765 52 34.61538462 53 33.96226415 54 33.33333333 55 34.54545455 56 33.92857143 57 33.33333333 58 32.75862069 59 32.20338983 60 33.33333333 61 32.78688525 62 33.87096774 63 33.33333333 64 34.37500000 65 33.84615385 66 34.84848485 67 34.32835821 68 33.82352941 69 34.78260870 70 35.71428571 71 35.21126761 72 36.11111111 73 36.98630137 74 36.48648649 75 36.00000000 76 35.52631579 77 35.06493506 78 34.61538462 79 34.17721519 80 35.00000000 81 34.56790123 82 35.36585366 83 34.93975904 84 34.52380952 85 34.11764706 86 33.72093023 87 33.33333333 88 32.95454545 89 33.70786517 90 33.33333333 91 32.96703297 92 32.60869565 93 32.25806452 94 31.91489362 95 31.57894737 96 32.29166667 97 32.98969072 98 32.65306122 99 33.33333333 100 34.00000000 101 34.65346535 102 34.31372549 103 33.98058252 104 33.65384615 105 33.33333333 106 33.01886792 107 33.64485981 108 34.25925926 109 33.94495413 110 34.54545455 111 34.23423423 112 34.82142857 113 34.51327434 114 35.08771930 115 34.78260870 116 34.48275862 117 34.18803419 118 33.89830508 119 33.61344538 120 33.33333333 121 33.88429752 122 33.60655738 123 33.33333333 124 33.87096774 125 34.40000000 126 34.12698413 127 33.85826772 128 33.59375000 129 33.33333333 130 33.84615385 131 33.58778626 132 33.33333333 133 33.08270677 134 32.83582090 135 33.33333333 136 33.08823529 137 33.57664234 138 34.05797101 139 34.53237410 140 34.28571429 141 34.04255319 142 33.80281690 143 33.56643357 144 34.02777778 145 33.79310345 146 33.56164384 147 33.33333333 148 33.10810811 149 32.88590604 150 32.66666667 151 33.11258278 152 32.89473684 153 32.67973856 154 32.46753247 155 32.25806452 156 32.69230769 157 32.48407643 158 32.91139241 159 32.70440252 160 32.50000000 161 32.91925466 162 33.33333333 163 33.12883436 164 32.92682927 165 33.33333333 166 33.73493976 167 33.53293413 168 33.33333333 169 33.13609467 170 32.94117647 171 32.74853801 172 32.55813953 173 32.36994220 174 32.75862069 175 33.14285714 176 33.52272727 177 33.33333333 178 33.14606742 179 32.96089385 180 33.33333333 181 33.14917127 182 32.96703297 183 32.78688525 184 32.60869565 185 32.43243243 186 32.79569892 187 32.62032086 188 32.44680851 189 32.80423280 190 32.63157895 191 32.46073298 192 32.29166667 193 32.12435233 194 32.47422680 195 32.30769231 196 32.14285714 197 32.48730964 198 32.82828283 199 32.66331658 200 32.50000000 201 32.33830846 202 32.17821782 203 32.01970443 204 31.86274510 205 32.19512195 206 32.03883495 207 32.36714976 208 32.21153846 209 32.05741627 210 31.90476190 211 31.75355450 212 31.60377358 213 31.45539906 214 31.30841121 215 31.16279070 216 31.48148148 217 31.79723502 218 32.11009174 219 31.96347032 220 31.81818182 221 31.67420814 222 31.53153153 223 31.39013453 224 31.69642857 225 31.55555556 226 31.85840708 227 31.71806167 228 31.57894737 229 31.44104803 230 31.30434783 231 31.16883117 232 31.03448276 233 31.33047210 234 31.62393162 235 31.48936170 236 31.77966102 237 32.06751055 238 32.35294118 239 32.21757322 240 32.08333333 241 31.95020747 242 31.81818182 243 31.68724280 244 31.55737705 245 31.42857143 246 31.30081301 247 31.57894737 248 31.45161290 249 31.72690763 250 31.60000000 251 31.87250996 252 31.74603175 253 31.62055336 254 31.49606299 255 31.37254902 256 31.25000000 257 31.12840467 258 31.00775194 259 30.88803089 260 30.76923077 261 31.03448276 262 30.91603053 263 30.79847909 264 31.06060606 265 31.32075472 266 31.20300752 267 31.08614232 268 30.97014925 269 30.85501859 270 30.74074074 271 30.62730627 272 30.51470588 273 30.76923077 274 30.65693431 275 30.54545455 276 30.43478261 277 30.32490975 278 30.21582734 279 30.10752688 280 30.35714286 281 30.60498221 282 30.49645390 283 30.74204947 284 30.98591549 285 30.87719298 286 31.11888112 287 31.35888502 288 31.59722222 289 31.48788927 290 31.37931034 291 31.61512027 292 31.50684932 293 31.74061433 294 31.63265306 295 31.86440678 296 32.09459459 297 31.98653199 298 31.87919463 299 31.77257525 300 31.66666667 301 31.56146179 302 31.45695364 303 31.35313531 304 31.25000000 305 31.14754098 306 31.04575163 307 30.94462541 308 31.16883117 309 31.06796117 310 30.96774194 311 30.86816720 312 31.08974359 313 30.99041534 314 30.89171975 315 30.79365079 316 30.69620253 317 30.59936909 318 30.50314465 319 30.40752351 320 30.31250000 321 30.21806854 322 30.43478261 323 30.34055728 324 30.24691358 325 30.15384615 326 30.06134969 327 30.27522936 328 30.48780488 329 30.39513678 330 30.60606061 331 30.51359517 332 30.72289157 333 30.63063063 334 30.53892216 335 30.74626866 336 30.95238095 337 30.86053412 338 30.76923077 339 30.97345133 340 30.88235294 341 31.08504399 342 31.28654971 343 31.19533528 344 31.39534884 345 31.59420290 346 31.50289017 347 31.70028818 348 31.60919540 349 31.51862464 350 31.42857143 351 31.33903134 352 31.25000000 353 31.16147309 354 31.07344633 355 31.26760563 356 31.17977528 357 31.37254902 358 31.56424581 359 31.47632312 360 31.38888889 361 31.57894737 362 31.49171271 363 31.68044077 364 31.59340659 365 31.50684932 366 31.42076503 367 31.60762943 368 31.52173913 369 31.43631436 370 31.35135135 371 31.26684636 372 31.45161290 373 31.36729223 374 31.55080214 375 31.46666667 376 31.38297872 377 31.29973475 378 31.21693122 379 31.13456464 380 31.31578947 381 31.23359580 382 31.15183246 383 31.07049608 384 30.98958333 385 30.90909091 386 30.82901554 387 30.74935401 388 30.92783505 389 30.84832905 390 30.76923077 391 30.94629156 392 30.86734694 393 31.04325700 394 31.21827411 395 31.13924051 396 31.06060606 397 31.23425693 398 31.15577889 399 31.07769424 400 31.00000000 401 30.92269327 402 31.09452736 403 31.01736973 404 30.94059406 405 30.86419753 406 30.78817734 407 30.71253071 408 30.63725490 409 30.56234719 410 30.48780488 411 30.41362530 412 30.58252427 413 30.75060533 414 30.91787440 415 30.84337349 416 30.76923077 417 30.69544365 418 30.62200957 419 30.54892601 420 30.71428571 421 30.87885986 422 30.80568720 423 30.73286052 424 30.66037736 425 30.58823529 426 30.51643192 427 30.44496487 428 30.37383178 429 30.30303030 430 30.23255814 431 30.16241299 432 30.32407407 433 30.48498845 434 30.64516129 435 30.57471264 436 30.50458716 437 30.43478261 438 30.59360731 439 30.75170843 440 30.68181818 441 30.61224490 442 30.54298643 443 30.47404063 444 30.63063063 445 30.56179775 446 30.49327354 447 30.42505593 448 30.35714286 449 30.51224944 450 30.66666667 451 30.59866962 452 30.53097345 453 30.68432671 454 30.61674009 455 30.54945055 456 30.48245614 457 30.41575492 458 30.34934498 459 30.28322440 460 30.21739130 461 30.15184382 462 30.30303030 463 30.45356371 464 30.60344828 465 30.75268817 466 30.68669528 467 30.62098501 468 30.76923077 469 30.70362473 470 30.63829787 471 30.78556263 472 30.72033898 473 30.65539112 474 30.59071730 475 30.73684211 476 30.67226891 477 30.81761006 478 30.75313808 479 30.68893528 480 30.62500000 481 30.56133056 482 30.70539419 483 30.64182195 484 30.78512397 485 30.92783505 486 30.86419753 487 31.00616016 488 30.94262295 489 31.08384458 490 31.22448980 491 31.36456212 492 31.50406504 493 31.44016227 494 31.57894737 495 31.51515152 496 31.45161290 497 31.38832998 498 31.32530120 499 31.26252505 500 31.20000000 501 31.33732535 502 31.27490040 503 31.21272366 504 31.34920635 505 31.28712871 506 31.22529644 507 31.36094675 508 31.29921260 509 31.43418468 510 31.56862745 511 31.50684932 512 31.44531250 513 31.38401559 514 31.32295720 515 31.45631068 516 31.39534884 517 31.52804642 518 31.46718147 519 31.40655106 520 31.34615385 521 31.28598848 522 31.22605364 523 31.16634799 524 31.10687023 525 31.04761905 526 30.98859316 527 30.92979127 528 30.87121212 529 31.00189036 530 31.13207547 531 31.07344633 532 31.01503759 533 31.14446529 534 31.08614232 535 31.02803738 536 30.97014925 537 31.09869646 538 31.04089219 539 30.98330241 540 30.92592593 541 30.86876155 542 30.81180812 543 30.75506446 544 30.69852941 545 30.64220183 546 30.58608059 547 30.71297989 548 30.83941606 549 30.78324226 550 30.72727273 551 30.67150635 552 30.79710145 553 30.92224231 554 30.86642599 555 30.81081081 556 30.75539568 557 30.70017953 558 30.64516129 559 30.59033989 560 30.71428571 561 30.65953654 562 30.60498221 563 30.55062167 564 30.49645390 565 30.61946903 566 30.74204947 567 30.68783069 568 30.80985915 569 30.75571178 570 30.70175439 571 30.64798599 572 30.59440559 573 30.54101222 574 30.66202091 575 30.78260870 576 30.90277778 577 31.02253033 578 31.14186851 579 31.26079447 580 31.37931034 581 31.32530120 582 31.44329897 583 31.38936535 584 31.50684932 585 31.45299145 586 31.56996587 587 31.51618399 588 31.46258503 589 31.40916808 590 31.35593220 591 31.30287648 592 31.25000000 593 31.19730185 594 31.31313131 595 31.26050420 596 31.20805369 597 31.15577889 598 31.10367893 599 31.05175292 600 31.00000000 601 31.11480865 602 31.06312292 603 31.17744610 604 31.12582781 605 31.07438017 606 31.18811881 607 31.13673806 608 31.08552632 609 31.19868637 610 31.14754098 611 31.09656301 612 31.20915033 613 31.15823817 614 31.10749186 615 31.05691057 616 31.16883117 617 31.28038898 618 31.22977346 619 31.34087237 620 31.29032258 621 31.23993559 622 31.35048232 623 31.30016051 624 31.25000000 625 31.20000000 626 31.15015974 627 31.25996810 628 31.36942675 629 31.31955485 630 31.26984127 631 31.37876387 632 31.32911392 633 31.27962085 634 31.38801262 635 31.33858268 636 31.28930818 637 31.24018838 638 31.19122257 639 31.14241002 640 31.25000000 641 31.20124805 642 31.15264798 643 31.10419907 644 31.05590062 645 31.16279070 646 31.11455108 647 31.06646059 648 31.17283951 649 31.12480740 650 31.23076923 651 31.33640553 652 31.44171779 653 31.54670750 654 31.49847095 655 31.45038168 656 31.55487805 657 31.65905632 658 31.61094225 659 31.71471927 660 31.66666667 661 31.61875946 662 31.57099698 663 31.52337858 664 31.47590361 665 31.42857143 666 31.53153153 667 31.63418291 668 31.58682635 669 31.68908819 670 31.64179104 671 31.59463487 672 31.54761905 673 31.50074294 674 31.45400593 675 31.55555556 676 31.65680473 677 31.61004431 678 31.56342183 679 31.51693667 680 31.47058824 681 31.42437592 682 31.52492669 683 31.47877013 684 31.57894737 685 31.53284672 686 31.48688047 687 31.44104803 688 31.39534884 689 31.49492017 690 31.44927536 691 31.54848046 692 31.64739884 693 31.60173160 694 31.55619597 695 31.51079137 696 31.60919540 697 31.56384505 698 31.51862464 699 31.47353362 700 31.57142857 701 31.52639087 702 31.48148148 703 31.43669986 704 31.39204545 705 31.34751773 706 31.44475921 707 31.54172560 708 31.49717514 709 31.45275035 710 31.40845070 711 31.36427567 712 31.46067416 713 31.41654979 714 31.37254902 715 31.46853147 716 31.42458101 717 31.52022315 718 31.47632312 719 31.57162726 720 31.52777778 721 31.48404993 722 31.44044321 723 31.39695712 724 31.35359116 725 31.31034483 726 31.40495868 727 31.36176066 728 31.31868132 729 31.41289438 730 31.36986301 731 31.32694938 732 31.42076503 733 31.37789905 734 31.33514986 735 31.29251701 736 31.25000000 737 31.34328358 738 31.30081301 739 31.25845737 740 31.21621622 741 31.17408907 742 31.13207547 743 31.09017497 744 31.18279570 745 31.14093960 746 31.23324397 747 31.32530120 748 31.28342246 749 31.37516689 750 31.46666667 Final result: 31.4667 +/- 1.6968 Random chance: 19.8992 +/- 1.4588