common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 817 tasks in prompt multiple_choice_score: selecting 750 random tasks from 817 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 25.00000000 5 20.00000000 6 16.66666667 7 14.28571429 8 25.00000000 9 22.22222222 10 20.00000000 11 18.18181818 12 25.00000000 13 30.76923077 14 28.57142857 15 26.66666667 16 31.25000000 17 29.41176471 18 27.77777778 19 26.31578947 20 25.00000000 21 23.80952381 22 22.72727273 23 21.73913043 24 25.00000000 25 24.00000000 26 23.07692308 27 22.22222222 28 21.42857143 29 24.13793103 30 23.33333333 31 25.80645161 32 28.12500000 33 30.30303030 34 29.41176471 35 28.57142857 36 27.77777778 37 29.72972973 38 28.94736842 39 30.76923077 40 30.00000000 41 29.26829268 42 28.57142857 43 30.23255814 44 29.54545455 45 31.11111111 46 30.43478261 47 31.91489362 48 33.33333333 49 32.65306122 50 34.00000000 51 33.33333333 52 32.69230769 53 32.07547170 54 31.48148148 55 32.72727273 56 32.14285714 57 31.57894737 58 31.03448276 59 32.20338983 60 33.33333333 61 32.78688525 62 33.87096774 63 33.33333333 64 32.81250000 65 32.30769231 66 33.33333333 67 32.83582090 68 32.35294118 69 33.33333333 70 32.85714286 71 32.39436620 72 33.33333333 73 32.87671233 74 32.43243243 75 32.00000000 76 32.89473684 77 32.46753247 78 32.05128205 79 31.64556962 80 32.50000000 81 32.09876543 82 32.92682927 83 32.53012048 84 32.14285714 85 31.76470588 86 31.39534884 87 31.03448276 88 30.68181818 89 31.46067416 90 31.11111111 91 30.76923077 92 30.43478261 93 30.10752688 94 29.78723404 95 30.52631579 96 31.25000000 97 31.95876289 98 31.63265306 99 32.32323232 100 33.00000000 101 33.66336634 102 33.33333333 103 33.00970874 104 32.69230769 105 32.38095238 106 32.07547170 107 32.71028037 108 33.33333333 109 33.02752294 110 33.63636364 111 33.33333333 112 33.92857143 113 34.51327434 114 35.08771930 115 34.78260870 116 34.48275862 117 34.18803419 118 33.89830508 119 33.61344538 120 34.16666667 121 34.71074380 122 34.42622951 123 34.14634146 124 34.67741935 125 35.20000000 126 35.71428571 127 35.43307087 128 35.15625000 129 35.65891473 130 36.15384615 131 35.87786260 132 35.60606061 133 35.33834586 134 35.07462687 135 35.55555556 136 35.29411765 137 35.76642336 138 36.23188406 139 36.69064748 140 36.42857143 141 36.17021277 142 35.91549296 143 35.66433566 144 36.11111111 145 35.86206897 146 35.61643836 147 36.05442177 148 35.81081081 149 35.57046980 150 35.33333333 151 35.76158940 152 35.52631579 153 35.29411765 154 35.06493506 155 35.48387097 156 35.89743590 157 35.66878981 158 36.07594937 159 35.84905660 160 35.62500000 161 35.40372671 162 35.80246914 163 35.58282209 164 35.36585366 165 35.75757576 166 36.14457831 167 35.92814371 168 36.30952381 169 36.09467456 170 35.88235294 171 35.67251462 172 35.46511628 173 35.26011561 174 35.63218391 175 35.42857143 176 35.79545455 177 35.59322034 178 35.39325843 179 35.75418994 180 36.11111111 181 35.91160221 182 35.71428571 183 35.51912568 184 35.32608696 185 35.13513514 186 35.48387097 187 35.29411765 188 35.10638298 189 35.44973545 190 35.26315789 191 35.07853403 192 34.89583333 193 34.71502591 194 35.05154639 195 34.87179487 196 34.69387755 197 35.02538071 198 35.35353535 199 35.17587940 200 35.50000000 201 35.32338308 202 35.64356436 203 35.46798030 204 35.29411765 205 35.60975610 206 35.43689320 207 35.74879227 208 35.57692308 209 35.40669856 210 35.23809524 211 35.07109005 212 34.90566038 213 34.74178404 214 34.57943925 215 34.41860465 216 34.72222222 217 35.02304147 218 35.32110092 219 35.15981735 220 35.00000000 221 34.84162896 222 34.68468468 223 34.52914798 224 34.82142857 225 34.66666667 226 34.95575221 227 35.24229075 228 35.08771930 229 34.93449782 230 34.78260870 231 34.63203463 232 34.48275862 233 34.33476395 234 34.61538462 235 34.46808511 236 34.74576271 237 35.02109705 238 35.29411765 239 35.14644351 240 35.00000000 241 34.85477178 242 34.71074380 243 34.97942387 244 34.83606557 245 34.69387755 246 34.55284553 247 34.81781377 248 34.67741935 249 34.93975904 250 34.80000000 251 35.05976096 252 34.92063492 253 34.78260870 254 34.64566929 255 34.50980392 256 34.37500000 257 34.24124514 258 34.10852713 259 33.97683398 260 33.84615385 261 34.09961686 262 33.96946565 263 33.84030418 264 34.09090909 265 34.33962264 266 34.21052632 267 34.08239700 268 33.95522388 269 33.82899628 270 33.70370370 271 33.57933579 272 33.45588235 273 33.69963370 274 33.57664234 275 33.45454545 276 33.33333333 277 33.21299639 278 33.09352518 279 32.97491039 280 33.21428571 281 33.45195730 282 33.33333333 283 33.56890459 284 33.45070423 285 33.33333333 286 33.56643357 287 33.44947735 288 33.68055556 289 33.56401384 290 33.44827586 291 33.67697595 292 33.56164384 293 33.78839590 294 33.67346939 295 33.89830508 296 34.12162162 297 34.34343434 298 34.22818792 299 34.11371237 300 34.00000000 301 33.88704319 302 33.77483444 303 33.66336634 304 33.55263158 305 33.44262295 306 33.33333333 307 33.22475570 308 33.44155844 309 33.33333333 310 33.22580645 311 33.11897106 312 33.33333333 313 33.22683706 314 33.12101911 315 33.01587302 316 32.91139241 317 32.80757098 318 32.70440252 319 32.60188088 320 32.50000000 321 32.39875389 322 32.60869565 323 32.50773994 324 32.40740741 325 32.30769231 326 32.20858896 327 32.11009174 328 32.31707317 329 32.21884498 330 32.42424242 331 32.32628399 332 32.53012048 333 32.73273273 334 32.63473054 335 32.83582090 336 33.03571429 337 32.93768546 338 32.84023669 339 33.03834808 340 32.94117647 341 33.13782991 342 33.33333333 343 33.23615160 344 33.43023256 345 33.62318841 346 33.52601156 347 33.71757925 348 33.62068966 349 33.81088825 350 33.71428571 351 33.61823362 352 33.80681818 353 33.71104816 354 33.61581921 355 33.80281690 356 33.70786517 357 33.61344538 358 33.51955307 359 33.42618384 360 33.33333333 361 33.51800554 362 33.42541436 363 33.60881543 364 33.51648352 365 33.42465753 366 33.33333333 367 33.51498638 368 33.42391304 369 33.33333333 370 33.24324324 371 33.15363881 372 33.33333333 373 33.24396783 374 33.42245989 375 33.33333333 376 33.24468085 377 33.42175066 378 33.33333333 379 33.24538259 380 33.15789474 381 33.07086614 382 32.98429319 383 32.89817232 384 32.81250000 385 32.72727273 386 32.64248705 387 32.55813953 388 32.73195876 389 32.64781491 390 32.56410256 391 32.73657289 392 32.65306122 393 32.82442748 394 32.99492386 395 32.91139241 396 33.08080808 397 33.24937028 398 33.16582915 399 33.08270677 400 33.00000000 401 32.91770574 402 33.08457711 403 33.00248139 404 32.92079208 405 32.83950617 406 32.75862069 407 32.67813268 408 32.59803922 409 32.51833741 410 32.43902439 411 32.36009732 412 32.52427184 413 32.68765133 414 32.85024155 415 32.77108434 416 32.69230769 417 32.61390887 418 32.53588517 419 32.45823389 420 32.61904762 421 32.77909739 422 32.70142180 423 32.86052009 424 32.78301887 425 32.70588235 426 32.62910798 427 32.55269321 428 32.47663551 429 32.40093240 430 32.32558140 431 32.25058005 432 32.40740741 433 32.56351039 434 32.71889401 435 32.64367816 436 32.56880734 437 32.49427918 438 32.64840183 439 32.80182232 440 32.95454545 441 32.87981859 442 32.80542986 443 32.73137698 444 32.88288288 445 32.80898876 446 32.73542601 447 32.66219239 448 32.58928571 449 32.73942094 450 32.66666667 451 32.59423503 452 32.52212389 453 32.67108168 454 32.59911894 455 32.52747253 456 32.45614035 457 32.38512035 458 32.31441048 459 32.24400871 460 32.17391304 461 32.10412148 462 32.25108225 463 32.39740821 464 32.54310345 465 32.68817204 466 32.61802575 467 32.54817987 468 32.69230769 469 32.62260128 470 32.55319149 471 32.48407643 472 32.62711864 473 32.55813953 474 32.70042194 475 32.84210526 476 32.77310924 477 32.70440252 478 32.63598326 479 32.56784969 480 32.50000000 481 32.43243243 482 32.57261411 483 32.50517598 484 32.64462810 485 32.78350515 486 32.71604938 487 32.64887064 488 32.58196721 489 32.71983640 490 32.85714286 491 32.99389002 492 33.13008130 493 33.06288032 494 33.19838057 495 33.33333333 496 33.26612903 497 33.19919517 498 33.13253012 499 33.06613226 500 33.00000000 501 33.13373253 502 33.06772908 503 33.00198807 504 33.13492063 505 33.06930693 506 33.00395257 507 33.13609467 508 33.26771654 509 33.39882122 510 33.52941176 511 33.46379648 512 33.39843750 513 33.33333333 514 33.26848249 515 33.39805825 516 33.33333333 517 33.46228240 518 33.39768340 519 33.33333333 520 33.26923077 521 33.20537428 522 33.14176245 523 33.07839388 524 33.01526718 525 32.95238095 526 32.88973384 527 33.01707780 528 32.95454545 529 33.08128544 530 33.01886792 531 32.95668550 532 32.89473684 533 33.02063790 534 32.95880150 535 32.89719626 536 32.83582090 537 32.96089385 538 32.89962825 539 32.83858998 540 32.96296296 541 32.90203327 542 32.84132841 543 32.78084715 544 32.90441176 545 32.84403670 546 32.78388278 547 32.90676417 548 33.02919708 549 32.96903461 550 32.90909091 551 32.84936479 552 32.97101449 553 33.09222423 554 33.03249097 555 32.97297297 556 32.91366906 557 32.85457810 558 32.79569892 559 32.73703041 560 32.85714286 561 32.79857398 562 32.74021352 563 32.68206039 564 32.80141844 565 32.92035398 566 33.03886926 567 32.98059965 568 33.09859155 569 33.21616872 570 33.15789474 571 33.27495622 572 33.21678322 573 33.15881326 574 33.27526132 575 33.39130435 576 33.50694444 577 33.62218371 578 33.56401384 579 33.67875648 580 33.79310345 581 33.73493976 582 33.84879725 583 33.79073756 584 33.90410959 585 33.84615385 586 33.95904437 587 33.90119250 588 33.84353741 589 33.78607810 590 33.72881356 591 33.67174281 592 33.61486486 593 33.55817875 594 33.67003367 595 33.61344538 596 33.55704698 597 33.50083752 598 33.44481605 599 33.38898164 600 33.33333333 601 33.44425957 602 33.55481728 603 33.66500829 604 33.60927152 605 33.55371901 606 33.66336634 607 33.60790774 608 33.55263158 609 33.49753695 610 33.44262295 611 33.38788871 612 33.49673203 613 33.44208809 614 33.38762215 615 33.33333333 616 33.44155844 617 33.54943274 618 33.49514563 619 33.60258481 620 33.54838710 621 33.49436393 622 33.60128617 623 33.54735152 624 33.49358974 625 33.44000000 626 33.38658147 627 33.33333333 628 33.43949045 629 33.38632750 630 33.33333333 631 33.43898574 632 33.38607595 633 33.49131122 634 33.59621451 635 33.54330709 636 33.49056604 637 33.43799058 638 33.38557994 639 33.33333333 640 33.43750000 641 33.38533541 642 33.48909657 643 33.43701400 644 33.38509317 645 33.48837209 646 33.43653251 647 33.38485317 648 33.48765432 649 33.43605547 650 33.53846154 651 33.64055300 652 33.74233129 653 33.84379786 654 33.79204893 655 33.74045802 656 33.84146341 657 33.94216134 658 33.89057751 659 33.99089530 660 33.93939394 661 33.88804841 662 33.83685801 663 33.78582202 664 33.73493976 665 33.83458647 666 33.93393393 667 33.88305847 668 33.83233533 669 33.78176383 670 33.73134328 671 33.68107303 672 33.63095238 673 33.58098068 674 33.53115727 675 33.62962963 676 33.72781065 677 33.67799114 678 33.62831858 679 33.57879234 680 33.52941176 681 33.48017621 682 33.43108504 683 33.38213763 684 33.47953216 685 33.43065693 686 33.38192420 687 33.33333333 688 33.28488372 689 33.38171263 690 33.33333333 691 33.42981187 692 33.52601156 693 33.62193362 694 33.57348703 695 33.52517986 696 33.62068966 697 33.57245337 698 33.52435530 699 33.47639485 700 33.57142857 701 33.52353780 702 33.47578348 703 33.42816501 704 33.38068182 705 33.33333333 706 33.42776204 707 33.52192362 708 33.47457627 709 33.42736248 710 33.52112676 711 33.47398031 712 33.56741573 713 33.52033661 714 33.47338936 715 33.56643357 716 33.51955307 717 33.61227336 718 33.70473538 719 33.79694019 720 33.75000000 721 33.70319001 722 33.65650970 723 33.60995851 724 33.70165746 725 33.65517241 726 33.74655647 727 33.70013755 728 33.65384615 729 33.74485597 730 33.69863014 731 33.65253078 732 33.74316940 733 33.69713506 734 33.65122616 735 33.60544218 736 33.55978261 737 33.64993216 738 33.60433604 739 33.55886333 740 33.51351351 741 33.60323887 742 33.55795148 743 33.51278600 744 33.60215054 745 33.55704698 746 33.64611260 747 33.73493976 748 33.68983957 749 33.77837116 750 33.86666667 Final result: 33.8667 +/- 1.7292 Random chance: 19.8992 +/- 1.4588