common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 869 tasks in prompt multiple_choice_score: selecting 750 random tasks from 869 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 100.00000000 3 100.00000000 4 75.00000000 5 80.00000000 6 83.33333333 7 85.71428571 8 87.50000000 9 77.77777778 10 80.00000000 11 72.72727273 12 75.00000000 13 69.23076923 14 71.42857143 15 73.33333333 16 75.00000000 17 76.47058824 18 72.22222222 19 73.68421053 20 70.00000000 21 71.42857143 22 72.72727273 23 69.56521739 24 70.83333333 25 68.00000000 26 65.38461538 27 66.66666667 28 64.28571429 29 65.51724138 30 66.66666667 31 64.51612903 32 65.62500000 33 63.63636364 34 61.76470588 35 62.85714286 36 61.11111111 37 62.16216216 38 60.52631579 39 61.53846154 40 62.50000000 41 63.41463415 42 61.90476190 43 60.46511628 44 59.09090909 45 60.00000000 46 60.86956522 47 61.70212766 48 62.50000000 49 63.26530612 50 62.00000000 51 60.78431373 52 61.53846154 53 60.37735849 54 59.25925926 55 58.18181818 56 58.92857143 57 59.64912281 58 60.34482759 59 61.01694915 60 61.66666667 61 62.29508197 62 61.29032258 63 61.90476190 64 62.50000000 65 61.53846154 66 62.12121212 67 62.68656716 68 63.23529412 69 63.76811594 70 62.85714286 71 61.97183099 72 61.11111111 73 61.64383562 74 62.16216216 75 62.66666667 76 63.15789474 77 63.63636364 78 64.10256410 79 63.29113924 80 63.75000000 81 62.96296296 82 63.41463415 83 62.65060241 84 63.09523810 85 63.52941176 86 63.95348837 87 63.21839080 88 63.63636364 89 64.04494382 90 64.44444444 91 64.83516484 92 65.21739130 93 64.51612903 94 63.82978723 95 64.21052632 96 64.58333333 97 63.91752577 98 63.26530612 99 63.63636364 100 64.00000000 101 63.36633663 102 63.72549020 103 63.10679612 104 62.50000000 105 62.85714286 106 62.26415094 107 62.61682243 108 62.03703704 109 61.46788991 110 61.81818182 111 62.16216216 112 62.50000000 113 61.94690265 114 62.28070175 115 62.60869565 116 62.93103448 117 63.24786325 118 63.55932203 119 63.02521008 120 63.33333333 121 62.80991736 122 63.11475410 123 63.41463415 124 62.90322581 125 63.20000000 126 63.49206349 127 63.77952756 128 64.06250000 129 63.56589147 130 63.07692308 131 63.35877863 132 63.63636364 133 63.90977444 134 63.43283582 135 62.96296296 136 62.50000000 137 62.77372263 138 63.04347826 139 63.30935252 140 63.57142857 141 63.12056738 142 63.38028169 143 63.63636364 144 63.88888889 145 64.13793103 146 63.69863014 147 63.94557823 148 64.18918919 149 64.42953020 150 64.66666667 151 64.90066225 152 65.13157895 153 65.35947712 154 64.93506494 155 65.16129032 156 65.38461538 157 65.60509554 158 65.18987342 159 65.40880503 160 65.00000000 161 65.21739130 162 65.43209877 163 65.03067485 164 64.63414634 165 64.84848485 166 65.06024096 167 64.67065868 168 64.88095238 169 64.49704142 170 64.11764706 171 64.32748538 172 63.95348837 173 64.16184971 174 64.36781609 175 64.00000000 176 64.20454545 177 64.40677966 178 64.60674157 179 64.80446927 180 65.00000000 181 64.64088398 182 64.83516484 183 64.48087432 184 64.67391304 185 64.86486486 186 65.05376344 187 64.70588235 188 64.36170213 189 64.02116402 190 63.68421053 191 63.87434555 192 64.06250000 193 64.24870466 194 64.43298969 195 64.61538462 196 64.79591837 197 64.97461929 198 65.15151515 199 64.82412060 200 65.00000000 201 65.17412935 202 64.85148515 203 65.02463054 204 65.19607843 205 64.87804878 206 64.56310680 207 64.25120773 208 64.42307692 209 64.59330144 210 64.76190476 211 64.92890995 212 64.62264151 213 64.78873239 214 64.95327103 215 65.11627907 216 64.81481481 217 64.51612903 218 64.22018349 219 64.38356164 220 64.09090909 221 63.80090498 222 63.51351351 223 63.22869955 224 63.39285714 225 63.11111111 226 63.27433628 227 63.43612335 228 63.59649123 229 63.75545852 230 63.91304348 231 63.63636364 232 63.36206897 233 63.51931330 234 63.67521368 235 63.40425532 236 63.13559322 237 62.86919831 238 63.02521008 239 63.17991632 240 63.33333333 241 63.48547718 242 63.63636364 243 63.37448560 244 63.11475410 245 63.26530612 246 63.41463415 247 63.56275304 248 63.70967742 249 63.45381526 250 63.60000000 251 63.74501992 252 63.88888889 253 64.03162055 254 63.77952756 255 63.92156863 256 63.67187500 257 63.42412451 258 63.56589147 259 63.32046332 260 63.46153846 261 63.60153257 262 63.74045802 263 63.87832700 264 64.01515152 265 63.77358491 266 63.53383459 267 63.67041199 268 63.80597015 269 63.94052045 270 64.07407407 271 64.20664207 272 63.97058824 273 63.73626374 274 63.50364964 275 63.27272727 276 63.40579710 277 63.53790614 278 63.30935252 279 63.44086022 280 63.57142857 281 63.70106762 282 63.82978723 283 63.60424028 284 63.73239437 285 63.85964912 286 63.98601399 287 64.11149826 288 64.23611111 289 64.01384083 290 64.13793103 291 64.26116838 292 64.38356164 293 64.50511945 294 64.28571429 295 64.40677966 296 64.52702703 297 64.64646465 298 64.76510067 299 64.88294314 300 65.00000000 301 64.78405316 302 64.90066225 303 65.01650165 304 64.80263158 305 64.91803279 306 64.70588235 307 64.82084691 308 64.93506494 309 65.04854369 310 64.83870968 311 64.95176849 312 65.06410256 313 65.17571885 314 64.96815287 315 65.07936508 316 65.18987342 317 65.29968454 318 65.09433962 319 64.89028213 320 64.68750000 321 64.79750779 322 64.90683230 323 65.01547988 324 65.12345679 325 65.23076923 326 65.33742331 327 65.13761468 328 65.24390244 329 65.34954407 330 65.15151515 331 65.25679758 332 65.36144578 333 65.46546547 334 65.56886228 335 65.37313433 336 65.47619048 337 65.57863501 338 65.38461538 339 65.48672566 340 65.58823529 341 65.68914956 342 65.78947368 343 65.88921283 344 65.98837209 345 65.79710145 346 65.89595376 347 65.70605187 348 65.51724138 349 65.61604585 350 65.42857143 351 65.52706553 352 65.62500000 353 65.72237960 354 65.53672316 355 65.35211268 356 65.44943820 357 65.54621849 358 65.64245810 359 65.73816156 360 65.83333333 361 65.92797784 362 66.02209945 363 65.84022039 364 65.65934066 365 65.75342466 366 65.57377049 367 65.39509537 368 65.48913043 369 65.58265583 370 65.67567568 371 65.76819407 372 65.86021505 373 65.95174263 374 66.04278075 375 65.86666667 376 65.69148936 377 65.51724138 378 65.60846561 379 65.43535620 380 65.52631579 381 65.35433071 382 65.18324607 383 65.01305483 384 65.10416667 385 65.19480519 386 65.28497409 387 65.37467700 388 65.20618557 389 65.29562982 390 65.12820513 391 65.21739130 392 65.30612245 393 65.39440204 394 65.48223350 395 65.56962025 396 65.40404040 397 65.49118388 398 65.32663317 399 65.41353383 400 65.25000000 401 65.08728180 402 65.17412935 403 65.26054591 404 65.34653465 405 65.43209877 406 65.51724138 407 65.60196560 408 65.68627451 409 65.52567237 410 65.60975610 411 65.69343066 412 65.77669903 413 65.61743341 414 65.70048309 415 65.54216867 416 65.62500000 417 65.46762590 418 65.55023923 419 65.39379475 420 65.47619048 421 65.32066508 422 65.40284360 423 65.48463357 424 65.33018868 425 65.41176471 426 65.25821596 427 65.33957845 428 65.18691589 429 65.26806527 430 65.11627907 431 65.19721578 432 65.27777778 433 65.12702079 434 65.20737327 435 65.28735632 436 65.36697248 437 65.44622426 438 65.29680365 439 65.37585421 440 65.22727273 441 65.07936508 442 64.93212670 443 64.78555305 444 64.63963964 445 64.71910112 446 64.79820628 447 64.65324385 448 64.73214286 449 64.81069042 450 64.66666667 451 64.74501109 452 64.82300885 453 64.67991170 454 64.75770925 455 64.61538462 456 64.69298246 457 64.77024070 458 64.84716157 459 64.92374728 460 65.00000000 461 65.07592191 462 65.15151515 463 65.01079914 464 64.87068966 465 64.94623656 466 65.02145923 467 65.09635974 468 65.17094017 469 65.03198294 470 65.10638298 471 65.18046709 472 65.04237288 473 64.90486258 474 64.97890295 475 65.05263158 476 65.12605042 477 65.19916143 478 65.27196653 479 65.34446764 480 65.41666667 481 65.48856549 482 65.56016598 483 65.63146998 484 65.70247934 485 65.77319588 486 65.84362140 487 65.70841889 488 65.77868852 489 65.84867076 490 65.71428571 491 65.58044807 492 65.65040650 493 65.51724138 494 65.58704453 495 65.45454545 496 65.52419355 497 65.39235412 498 65.46184739 499 65.53106212 500 65.60000000 501 65.46906188 502 65.53784861 503 65.60636183 504 65.67460317 505 65.74257426 506 65.61264822 507 65.48323471 508 65.55118110 509 65.42239686 510 65.29411765 511 65.16634051 512 65.23437500 513 65.10721248 514 65.17509728 515 65.24271845 516 65.31007752 517 65.37717602 518 65.44401544 519 65.51059730 520 65.38461538 521 65.45105566 522 65.51724138 523 65.39196941 524 65.26717557 525 65.14285714 526 65.01901141 527 64.89563567 528 64.77272727 529 64.65028355 530 64.71698113 531 64.78342750 532 64.84962406 533 64.91557223 534 64.79400749 535 64.67289720 536 64.73880597 537 64.80446927 538 64.86988848 539 64.93506494 540 64.81481481 541 64.69500924 542 64.76014760 543 64.64088398 544 64.70588235 545 64.77064220 546 64.65201465 547 64.53382084 548 64.59854015 549 64.66302368 550 64.72727273 551 64.60980036 552 64.49275362 553 64.55696203 554 64.62093863 555 64.68468468 556 64.56834532 557 64.63195691 558 64.51612903 559 64.57960644 560 64.64285714 561 64.70588235 562 64.76868327 563 64.83126110 564 64.89361702 565 64.77876106 566 64.84098940 567 64.72663139 568 64.78873239 569 64.67486819 570 64.56140351 571 64.62346760 572 64.51048951 573 64.57242583 574 64.63414634 575 64.52173913 576 64.58333333 577 64.64471404 578 64.70588235 579 64.76683938 580 64.82758621 581 64.71600688 582 64.60481100 583 64.66552316 584 64.72602740 585 64.78632479 586 64.67576792 587 64.73594549 588 64.79591837 589 64.85568761 590 64.91525424 591 64.97461929 592 64.86486486 593 64.75548061 594 64.81481481 595 64.87394958 596 64.76510067 597 64.65661642 598 64.54849498 599 64.60767947 600 64.50000000 601 64.55906822 602 64.45182724 603 64.34494196 604 64.40397351 605 64.46280992 606 64.52145215 607 64.57990115 608 64.63815789 609 64.53201970 610 64.59016393 611 64.48445172 612 64.54248366 613 64.43719413 614 64.49511401 615 64.39024390 616 64.28571429 617 64.34359806 618 64.40129450 619 64.29725363 620 64.35483871 621 64.41223833 622 64.30868167 623 64.20545746 624 64.10256410 625 64.00000000 626 63.89776358 627 63.95534290 628 64.01273885 629 63.91096979 630 63.96825397 631 64.02535658 632 63.92405063 633 63.82306477 634 63.88012618 635 63.93700787 636 63.99371069 637 63.89324961 638 63.94984326 639 64.00625978 640 63.90625000 641 63.96255850 642 63.86292835 643 63.76360809 644 63.81987578 645 63.72093023 646 63.77708978 647 63.83307573 648 63.73456790 649 63.63636364 650 63.69230769 651 63.74807988 652 63.65030675 653 63.55283308 654 63.60856269 655 63.51145038 656 63.41463415 657 63.31811263 658 63.22188450 659 63.12594841 660 63.18181818 661 63.08623298 662 63.14199396 663 63.19758673 664 63.25301205 665 63.30827068 666 63.21321321 667 63.26836582 668 63.17365269 669 63.07922272 670 63.13432836 671 63.04023845 672 63.09523810 673 63.00148588 674 62.90801187 675 62.81481481 676 62.86982249 677 62.92466765 678 62.97935103 679 63.03387334 680 63.08823529 681 62.99559471 682 62.90322581 683 62.81112738 684 62.86549708 685 62.91970803 686 62.82798834 687 62.73653566 688 62.79069767 689 62.69956459 690 62.75362319 691 62.80752533 692 62.86127168 693 62.91486291 694 62.96829971 695 62.87769784 696 62.78735632 697 62.69727403 698 62.60744986 699 62.66094421 700 62.71428571 701 62.62482168 702 62.67806268 703 62.73115220 704 62.78409091 705 62.83687943 706 62.88951841 707 62.94200849 708 62.99435028 709 63.04654443 710 63.09859155 711 63.15049226 712 63.20224719 713 63.25385694 714 63.16526611 715 63.07692308 716 63.12849162 717 63.17991632 718 63.09192201 719 63.00417246 720 63.05555556 721 63.10679612 722 63.15789474 723 63.20885201 724 63.12154696 725 63.17241379 726 63.08539945 727 62.99862448 728 63.04945055 729 63.10013717 730 63.15068493 731 63.20109439 732 63.25136612 733 63.30150068 734 63.35149864 735 63.40136054 736 63.45108696 737 63.36499322 738 63.41463415 739 63.32882273 740 63.24324324 741 63.29284750 742 63.20754717 743 63.25706595 744 63.17204301 745 63.22147651 746 63.27077748 747 63.18607764 748 63.10160428 749 63.01735648 750 63.06666667 Final result: 63.0667 +/- 1.7635 Random chance: 25.0083 +/- 1.5824