common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 869 tasks in prompt multiple_choice_score: selecting 750 random tasks from 869 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 100.00000000 3 100.00000000 4 75.00000000 5 80.00000000 6 83.33333333 7 85.71428571 8 87.50000000 9 77.77777778 10 80.00000000 11 72.72727273 12 75.00000000 13 69.23076923 14 71.42857143 15 73.33333333 16 75.00000000 17 76.47058824 18 72.22222222 19 73.68421053 20 70.00000000 21 71.42857143 22 72.72727273 23 69.56521739 24 70.83333333 25 68.00000000 26 65.38461538 27 66.66666667 28 64.28571429 29 65.51724138 30 66.66666667 31 64.51612903 32 65.62500000 33 63.63636364 34 61.76470588 35 62.85714286 36 61.11111111 37 62.16216216 38 63.15789474 39 64.10256410 40 65.00000000 41 65.85365854 42 64.28571429 43 62.79069767 44 61.36363636 45 62.22222222 46 63.04347826 47 63.82978723 48 64.58333333 49 65.30612245 50 66.00000000 51 64.70588235 52 65.38461538 53 64.15094340 54 62.96296296 55 61.81818182 56 62.50000000 57 63.15789474 58 63.79310345 59 64.40677966 60 65.00000000 61 65.57377049 62 64.51612903 63 65.07936508 64 65.62500000 65 64.61538462 66 65.15151515 67 65.67164179 68 66.17647059 69 66.66666667 70 65.71428571 71 64.78873239 72 63.88888889 73 64.38356164 74 64.86486486 75 65.33333333 76 65.78947368 77 66.23376623 78 66.66666667 79 65.82278481 80 66.25000000 81 65.43209877 82 65.85365854 83 65.06024096 84 65.47619048 85 65.88235294 86 66.27906977 87 65.51724138 88 65.90909091 89 66.29213483 90 66.66666667 91 67.03296703 92 67.39130435 93 66.66666667 94 65.95744681 95 66.31578947 96 66.66666667 97 65.97938144 98 65.30612245 99 65.65656566 100 66.00000000 101 65.34653465 102 65.68627451 103 65.04854369 104 65.38461538 105 65.71428571 106 65.09433962 107 65.42056075 108 64.81481481 109 64.22018349 110 64.54545455 111 64.86486486 112 65.17857143 113 64.60176991 114 64.91228070 115 65.21739130 116 65.51724138 117 64.95726496 118 65.25423729 119 64.70588235 120 65.00000000 121 64.46280992 122 64.75409836 123 65.04065041 124 64.51612903 125 64.80000000 126 65.07936508 127 65.35433071 128 65.62500000 129 65.11627907 130 64.61538462 131 64.88549618 132 65.15151515 133 65.41353383 134 64.92537313 135 64.44444444 136 63.97058824 137 64.23357664 138 64.49275362 139 64.74820144 140 65.00000000 141 64.53900709 142 64.78873239 143 65.03496503 144 64.58333333 145 64.82758621 146 64.38356164 147 64.62585034 148 64.86486486 149 65.10067114 150 65.33333333 151 65.56291391 152 65.78947368 153 66.01307190 154 65.58441558 155 65.80645161 156 66.02564103 157 66.24203822 158 65.82278481 159 66.03773585 160 65.62500000 161 65.83850932 162 66.04938272 163 65.64417178 164 65.24390244 165 65.45454545 166 65.66265060 167 65.26946108 168 65.47619048 169 65.08875740 170 65.29411765 171 65.49707602 172 65.11627907 173 65.31791908 174 65.51724138 175 65.14285714 176 65.34090909 177 65.53672316 178 65.73033708 179 65.92178771 180 66.11111111 181 65.74585635 182 65.93406593 183 65.57377049 184 65.76086957 185 65.94594595 186 66.12903226 187 66.31016043 188 65.95744681 189 65.60846561 190 65.26315789 191 65.44502618 192 65.62500000 193 65.80310881 194 65.97938144 195 66.15384615 196 66.32653061 197 66.49746193 198 66.16161616 199 65.82914573 200 66.00000000 201 66.16915423 202 65.84158416 203 66.00985222 204 66.17647059 205 65.85365854 206 65.53398058 207 65.70048309 208 65.86538462 209 66.02870813 210 66.19047619 211 66.35071090 212 66.03773585 213 66.19718310 214 66.35514019 215 66.51162791 216 66.20370370 217 65.89861751 218 65.59633028 219 65.75342466 220 65.45454545 221 65.15837104 222 64.86486486 223 64.57399103 224 64.73214286 225 64.44444444 226 64.60176991 227 64.75770925 228 64.91228070 229 65.06550218 230 65.21739130 231 64.93506494 232 64.65517241 233 64.80686695 234 64.95726496 235 64.68085106 236 64.40677966 237 64.13502110 238 64.28571429 239 64.43514644 240 64.58333333 241 64.73029046 242 64.87603306 243 64.60905350 244 64.34426230 245 64.48979592 246 64.63414634 247 64.77732794 248 64.91935484 249 64.65863454 250 64.80000000 251 64.94023904 252 65.07936508 253 65.21739130 254 64.96062992 255 65.09803922 256 64.84375000 257 64.59143969 258 64.72868217 259 64.47876448 260 64.61538462 261 64.75095785 262 64.88549618 263 65.01901141 264 65.15151515 265 64.90566038 266 64.66165414 267 64.79400749 268 64.92537313 269 65.05576208 270 65.18518519 271 65.31365314 272 65.07352941 273 64.83516484 274 64.59854015 275 64.36363636 276 64.49275362 277 64.62093863 278 64.38848921 279 64.51612903 280 64.64285714 281 64.76868327 282 64.89361702 283 64.66431095 284 64.78873239 285 64.91228070 286 65.03496503 287 65.15679443 288 65.27777778 289 65.05190311 290 65.17241379 291 65.29209622 292 65.41095890 293 65.52901024 294 65.30612245 295 65.42372881 296 65.54054054 297 65.65656566 298 65.77181208 299 65.88628763 300 65.66666667 301 65.44850498 302 65.56291391 303 65.67656766 304 65.78947368 305 65.90163934 306 65.68627451 307 65.79804560 308 65.90909091 309 66.01941748 310 66.12903226 311 66.23794212 312 66.34615385 313 66.45367412 314 66.24203822 315 66.34920635 316 66.45569620 317 66.56151420 318 66.35220126 319 66.14420063 320 65.93750000 321 66.04361371 322 66.14906832 323 66.25386997 324 66.35802469 325 66.46153846 326 66.56441718 327 66.36085627 328 66.46341463 329 66.56534954 330 66.36363636 331 66.46525680 332 66.56626506 333 66.66666667 334 66.76646707 335 66.56716418 336 66.66666667 337 66.76557864 338 66.56804734 339 66.66666667 340 66.76470588 341 66.86217009 342 66.95906433 343 67.05539359 344 67.15116279 345 66.95652174 346 67.05202312 347 66.85878963 348 66.66666667 349 66.76217765 350 66.57142857 351 66.66666667 352 66.76136364 353 66.85552408 354 66.66666667 355 66.47887324 356 66.57303371 357 66.66666667 358 66.75977654 359 66.57381616 360 66.66666667 361 66.75900277 362 66.85082873 363 66.66666667 364 66.48351648 365 66.57534247 366 66.39344262 367 66.21253406 368 66.30434783 369 66.39566396 370 66.48648649 371 66.57681941 372 66.66666667 373 66.75603217 374 66.84491979 375 66.66666667 376 66.48936170 377 66.31299735 378 66.40211640 379 66.22691293 380 66.31578947 381 66.14173228 382 65.96858639 383 65.79634465 384 65.88541667 385 65.97402597 386 66.06217617 387 66.14987080 388 65.97938144 389 65.80976864 390 65.64102564 391 65.47314578 392 65.56122449 393 65.64885496 394 65.73604061 395 65.82278481 396 65.65656566 397 65.74307305 398 65.57788945 399 65.66416040 400 65.75000000 401 65.58603491 402 65.67164179 403 65.75682382 404 65.84158416 405 65.92592593 406 66.00985222 407 66.09336609 408 66.17647059 409 66.01466993 410 66.09756098 411 66.18004866 412 66.26213592 413 66.10169492 414 66.18357488 415 66.02409639 416 66.10576923 417 65.94724221 418 66.02870813 419 65.87112172 420 65.95238095 421 65.79572447 422 65.87677725 423 65.95744681 424 65.80188679 425 65.88235294 426 65.72769953 427 65.80796253 428 65.65420561 429 65.73426573 430 65.58139535 431 65.66125290 432 65.74074074 433 65.58891455 434 65.66820276 435 65.74712644 436 65.82568807 437 65.90389016 438 65.75342466 439 65.83143508 440 65.68181818 441 65.53287982 442 65.38461538 443 65.23702032 444 65.09009009 445 65.16853933 446 65.24663677 447 65.10067114 448 65.17857143 449 65.25612472 450 65.11111111 451 65.18847007 452 65.26548673 453 65.12141280 454 65.19823789 455 65.05494505 456 65.13157895 457 65.20787746 458 65.28384279 459 65.35947712 460 65.43478261 461 65.50976139 462 65.58441558 463 65.44276458 464 65.30172414 465 65.37634409 466 65.45064378 467 65.52462527 468 65.59829060 469 65.45842217 470 65.53191489 471 65.60509554 472 65.46610169 473 65.32769556 474 65.40084388 475 65.47368421 476 65.54621849 477 65.61844864 478 65.69037657 479 65.76200418 480 65.83333333 481 65.90436590 482 65.97510373 483 66.04554865 484 66.11570248 485 66.18556701 486 66.25514403 487 66.11909651 488 66.18852459 489 66.25766871 490 66.12244898 491 65.98778004 492 66.05691057 493 65.92292089 494 65.99190283 495 65.85858586 496 65.92741935 497 65.79476861 498 65.86345382 499 65.93186373 500 66.00000000 501 65.86826347 502 65.93625498 503 66.00397614 504 66.07142857 505 66.13861386 506 66.00790514 507 65.87771203 508 65.74803150 509 65.61886051 510 65.49019608 511 65.36203523 512 65.42968750 513 65.30214425 514 65.36964981 515 65.43689320 516 65.50387597 517 65.57059961 518 65.63706564 519 65.70327553 520 65.57692308 521 65.64299424 522 65.70881226 523 65.58317400 524 65.45801527 525 65.33333333 526 65.20912548 527 65.08538899 528 64.96212121 529 64.83931947 530 64.90566038 531 64.78342750 532 64.84962406 533 64.91557223 534 64.79400749 535 64.67289720 536 64.73880597 537 64.80446927 538 64.86988848 539 64.93506494 540 64.81481481 541 64.69500924 542 64.76014760 543 64.64088398 544 64.70588235 545 64.77064220 546 64.65201465 547 64.53382084 548 64.59854015 549 64.66302368 550 64.72727273 551 64.60980036 552 64.49275362 553 64.55696203 554 64.62093863 555 64.68468468 556 64.56834532 557 64.63195691 558 64.51612903 559 64.57960644 560 64.64285714 561 64.70588235 562 64.76868327 563 64.83126110 564 64.89361702 565 64.77876106 566 64.84098940 567 64.72663139 568 64.78873239 569 64.67486819 570 64.56140351 571 64.62346760 572 64.51048951 573 64.57242583 574 64.63414634 575 64.52173913 576 64.58333333 577 64.64471404 578 64.70588235 579 64.76683938 580 64.82758621 581 64.71600688 582 64.60481100 583 64.66552316 584 64.72602740 585 64.78632479 586 64.67576792 587 64.73594549 588 64.79591837 589 64.85568761 590 64.91525424 591 64.97461929 592 64.86486486 593 64.75548061 594 64.81481481 595 64.70588235 596 64.59731544 597 64.48911223 598 64.54849498 599 64.60767947 600 64.50000000 601 64.55906822 602 64.45182724 603 64.34494196 604 64.40397351 605 64.46280992 606 64.52145215 607 64.57990115 608 64.63815789 609 64.53201970 610 64.59016393 611 64.48445172 612 64.54248366 613 64.43719413 614 64.49511401 615 64.39024390 616 64.28571429 617 64.34359806 618 64.40129450 619 64.29725363 620 64.35483871 621 64.41223833 622 64.30868167 623 64.20545746 624 64.10256410 625 64.00000000 626 63.89776358 627 63.95534290 628 64.01273885 629 63.91096979 630 63.96825397 631 64.02535658 632 63.92405063 633 63.82306477 634 63.88012618 635 63.93700787 636 63.99371069 637 63.89324961 638 63.94984326 639 64.00625978 640 63.90625000 641 63.96255850 642 63.86292835 643 63.76360809 644 63.81987578 645 63.72093023 646 63.77708978 647 63.83307573 648 63.73456790 649 63.63636364 650 63.69230769 651 63.74807988 652 63.65030675 653 63.55283308 654 63.60856269 655 63.51145038 656 63.41463415 657 63.31811263 658 63.22188450 659 63.12594841 660 63.18181818 661 63.08623298 662 63.14199396 663 63.19758673 664 63.25301205 665 63.30827068 666 63.21321321 667 63.26836582 668 63.17365269 669 63.07922272 670 63.13432836 671 63.04023845 672 63.09523810 673 63.00148588 674 62.90801187 675 62.81481481 676 62.72189349 677 62.77695716 678 62.83185841 679 62.88659794 680 62.94117647 681 62.84875184 682 62.75659824 683 62.66471449 684 62.71929825 685 62.77372263 686 62.68221574 687 62.59097525 688 62.64534884 689 62.55442671 690 62.60869565 691 62.66280753 692 62.71676301 693 62.77056277 694 62.82420749 695 62.73381295 696 62.64367816 697 62.55380201 698 62.46418338 699 62.51788269 700 62.57142857 701 62.48216833 702 62.53561254 703 62.58890469 704 62.64204545 705 62.69503546 706 62.74787535 707 62.80056577 708 62.85310734 709 62.90550071 710 62.95774648 711 63.00984529 712 63.06179775 713 63.11360449 714 63.02521008 715 62.93706294 716 62.98882682 717 62.90097629 718 62.81337047 719 62.72600834 720 62.77777778 721 62.82940361 722 62.88088643 723 62.93222683 724 62.84530387 725 62.89655172 726 62.80991736 727 62.72352132 728 62.77472527 729 62.82578875 730 62.87671233 731 62.92749658 732 62.97814208 733 63.02864939 734 63.07901907 735 63.12925170 736 63.17934783 737 63.09362280 738 63.14363144 739 63.05818674 740 62.97297297 741 63.02294197 742 62.93800539 743 62.98788694 744 62.90322581 745 62.95302013 746 63.00268097 747 62.91834003 748 62.83422460 749 62.75033378 750 62.80000000 Final result: 62.8000 +/- 1.7661 Random chance: 25.0083 +/- 1.5824