common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 817 tasks in prompt multiple_choice_score: selecting 750 random tasks from 817 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 25.00000000 5 20.00000000 6 16.66666667 7 14.28571429 8 12.50000000 9 11.11111111 10 10.00000000 11 9.09090909 12 16.66666667 13 23.07692308 14 21.42857143 15 26.66666667 16 31.25000000 17 29.41176471 18 27.77777778 19 31.57894737 20 30.00000000 21 28.57142857 22 27.27272727 23 26.08695652 24 29.16666667 25 28.00000000 26 26.92307692 27 25.92592593 28 25.00000000 29 27.58620690 30 26.66666667 31 25.80645161 32 28.12500000 33 30.30303030 34 29.41176471 35 28.57142857 36 27.77777778 37 29.72972973 38 28.94736842 39 30.76923077 40 30.00000000 41 29.26829268 42 28.57142857 43 30.23255814 44 29.54545455 45 31.11111111 46 30.43478261 47 31.91489362 48 33.33333333 49 32.65306122 50 34.00000000 51 33.33333333 52 32.69230769 53 32.07547170 54 31.48148148 55 32.72727273 56 32.14285714 57 31.57894737 58 31.03448276 59 32.20338983 60 33.33333333 61 32.78688525 62 33.87096774 63 33.33333333 64 34.37500000 65 33.84615385 66 34.84848485 67 34.32835821 68 33.82352941 69 34.78260870 70 34.28571429 71 33.80281690 72 34.72222222 73 35.61643836 74 35.13513514 75 34.66666667 76 34.21052632 77 33.76623377 78 33.33333333 79 32.91139241 80 33.75000000 81 33.33333333 82 34.14634146 83 33.73493976 84 33.33333333 85 32.94117647 86 32.55813953 87 32.18390805 88 31.81818182 89 32.58426966 90 32.22222222 91 31.86813187 92 31.52173913 93 31.18279570 94 30.85106383 95 30.52631579 96 31.25000000 97 31.95876289 98 31.63265306 99 32.32323232 100 33.00000000 101 33.66336634 102 33.33333333 103 33.00970874 104 32.69230769 105 32.38095238 106 32.07547170 107 32.71028037 108 33.33333333 109 33.02752294 110 33.63636364 111 33.33333333 112 33.92857143 113 33.62831858 114 34.21052632 115 33.91304348 116 34.48275862 117 34.18803419 118 33.89830508 119 33.61344538 120 33.33333333 121 33.88429752 122 33.60655738 123 33.33333333 124 33.87096774 125 34.40000000 126 34.12698413 127 33.85826772 128 33.59375000 129 33.33333333 130 33.84615385 131 33.58778626 132 33.33333333 133 33.08270677 134 32.83582090 135 33.33333333 136 33.08823529 137 33.57664234 138 34.05797101 139 34.53237410 140 34.28571429 141 34.04255319 142 33.80281690 143 33.56643357 144 34.02777778 145 33.79310345 146 33.56164384 147 33.33333333 148 33.10810811 149 32.88590604 150 32.66666667 151 33.11258278 152 32.89473684 153 32.67973856 154 32.46753247 155 32.25806452 156 32.69230769 157 32.48407643 158 32.91139241 159 33.33333333 160 33.12500000 161 33.54037267 162 33.95061728 163 33.74233129 164 33.53658537 165 33.93939394 166 34.33734940 167 34.13173653 168 33.92857143 169 33.72781065 170 33.52941176 171 33.33333333 172 33.13953488 173 32.94797688 174 33.33333333 175 33.71428571 176 34.09090909 177 33.89830508 178 33.70786517 179 34.07821229 180 34.44444444 181 34.25414365 182 34.06593407 183 33.87978142 184 33.69565217 185 33.51351351 186 33.33333333 187 33.15508021 188 32.97872340 189 33.33333333 190 33.68421053 191 33.50785340 192 33.33333333 193 33.16062176 194 33.50515464 195 33.33333333 196 33.16326531 197 33.50253807 198 33.83838384 199 33.66834171 200 34.00000000 201 33.83084577 202 34.15841584 203 33.99014778 204 33.82352941 205 33.65853659 206 33.49514563 207 33.81642512 208 33.65384615 209 33.49282297 210 33.33333333 211 33.17535545 212 33.01886792 213 32.86384977 214 32.71028037 215 32.55813953 216 32.87037037 217 33.17972350 218 33.48623853 219 33.33333333 220 33.18181818 221 33.03167421 222 32.88288288 223 33.18385650 224 33.48214286 225 33.33333333 226 33.18584071 227 33.48017621 228 33.33333333 229 33.18777293 230 33.04347826 231 32.90043290 232 32.75862069 233 32.61802575 234 32.90598291 235 32.76595745 236 33.05084746 237 33.33333333 238 33.61344538 239 33.47280335 240 33.33333333 241 33.19502075 242 33.05785124 243 32.92181070 244 32.78688525 245 32.65306122 246 32.52032520 247 32.79352227 248 32.66129032 249 32.93172691 250 32.80000000 251 33.06772908 252 33.33333333 253 33.20158103 254 33.07086614 255 32.94117647 256 32.81250000 257 32.68482490 258 32.55813953 259 32.43243243 260 32.30769231 261 32.56704981 262 32.44274809 263 32.31939163 264 32.57575758 265 32.83018868 266 32.70676692 267 32.58426966 268 32.46268657 269 32.34200743 270 32.22222222 271 32.10332103 272 31.98529412 273 32.23443223 274 32.11678832 275 32.00000000 276 31.88405797 277 31.76895307 278 31.65467626 279 31.54121864 280 31.78571429 281 32.02846975 282 31.91489362 283 32.15547703 284 32.39436620 285 32.28070175 286 32.51748252 287 32.75261324 288 32.98611111 289 32.87197232 290 32.75862069 291 32.98969072 292 32.87671233 293 33.10580205 294 32.99319728 295 33.22033898 296 33.44594595 297 33.33333333 298 33.22147651 299 33.11036789 300 33.00000000 301 32.89036545 302 32.78145695 303 32.67326733 304 32.56578947 305 32.45901639 306 32.35294118 307 32.24755700 308 32.46753247 309 32.36245955 310 32.25806452 311 32.15434084 312 32.37179487 313 32.26837061 314 32.16560510 315 32.06349206 316 31.96202532 317 31.86119874 318 31.76100629 319 31.66144201 320 31.56250000 321 31.46417445 322 31.67701863 323 31.57894737 324 31.48148148 325 31.38461538 326 31.28834356 327 31.49847095 328 31.70731707 329 31.61094225 330 31.81818182 331 31.72205438 332 31.92771084 333 31.83183183 334 31.73652695 335 31.94029851 336 32.14285714 337 32.04747774 338 31.95266272 339 32.15339233 340 32.05882353 341 32.25806452 342 32.45614035 343 32.36151603 344 32.55813953 345 32.75362319 346 32.65895954 347 32.85302594 348 32.75862069 349 32.66475645 350 32.57142857 351 32.47863248 352 32.67045455 353 32.57790368 354 32.48587571 355 32.67605634 356 32.58426966 357 32.77310924 358 32.96089385 359 32.86908078 360 32.77777778 361 32.96398892 362 32.87292818 363 33.05785124 364 32.96703297 365 32.87671233 366 32.78688525 367 32.97002725 368 32.88043478 369 32.79132791 370 32.70270270 371 32.61455526 372 32.79569892 373 32.70777480 374 32.88770053 375 32.80000000 376 32.71276596 377 32.89124668 378 32.80423280 379 32.71767810 380 32.89473684 381 32.80839895 382 32.72251309 383 32.63707572 384 32.55208333 385 32.46753247 386 32.38341969 387 32.29974160 388 32.47422680 389 32.39074550 390 32.30769231 391 32.48081841 392 32.39795918 393 32.56997455 394 32.74111675 395 32.65822785 396 32.82828283 397 32.99748111 398 32.91457286 399 32.83208020 400 32.75000000 401 32.91770574 402 33.08457711 403 33.25062035 404 33.16831683 405 33.08641975 406 33.00492611 407 32.92383292 408 32.84313725 409 32.76283619 410 32.68292683 411 32.60340633 412 32.76699029 413 32.68765133 414 32.85024155 415 33.01204819 416 32.93269231 417 32.85371703 418 32.77511962 419 32.69689737 420 32.85714286 421 33.01662708 422 32.93838863 423 32.86052009 424 32.78301887 425 32.70588235 426 32.62910798 427 32.55269321 428 32.47663551 429 32.40093240 430 32.32558140 431 32.25058005 432 32.40740741 433 32.56351039 434 32.71889401 435 32.64367816 436 32.56880734 437 32.49427918 438 32.64840183 439 32.80182232 440 32.72727273 441 32.65306122 442 32.57918552 443 32.50564334 444 32.65765766 445 32.58426966 446 32.51121076 447 32.43847875 448 32.36607143 449 32.51670379 450 32.66666667 451 32.59423503 452 32.52212389 453 32.67108168 454 32.59911894 455 32.52747253 456 32.45614035 457 32.38512035 458 32.31441048 459 32.24400871 460 32.17391304 461 32.10412148 462 32.25108225 463 32.39740821 464 32.54310345 465 32.68817204 466 32.61802575 467 32.54817987 468 32.69230769 469 32.62260128 470 32.55319149 471 32.69639066 472 32.83898305 473 32.76955603 474 32.70042194 475 32.84210526 476 32.77310924 477 32.70440252 478 32.63598326 479 32.56784969 480 32.50000000 481 32.43243243 482 32.57261411 483 32.50517598 484 32.64462810 485 32.78350515 486 32.71604938 487 32.85420945 488 32.78688525 489 32.92433538 490 33.06122449 491 33.19755601 492 33.33333333 493 33.26572008 494 33.40080972 495 33.33333333 496 33.26612903 497 33.19919517 498 33.13253012 499 33.06613226 500 33.00000000 501 32.93413174 502 32.86852590 503 32.80318091 504 32.93650794 505 32.87128713 506 32.80632411 507 32.93885602 508 33.07086614 509 33.20235756 510 33.33333333 511 33.26810176 512 33.20312500 513 33.13840156 514 33.07392996 515 33.20388350 516 33.13953488 517 33.26885880 518 33.20463320 519 33.14065511 520 33.07692308 521 33.01343570 522 32.95019157 523 32.88718929 524 32.82442748 525 32.76190476 526 32.69961977 527 32.63757116 528 32.57575758 529 32.70321361 530 32.64150943 531 32.58003766 532 32.51879699 533 32.64540338 534 32.58426966 535 32.52336449 536 32.46268657 537 32.58845438 538 32.52788104 539 32.46753247 540 32.40740741 541 32.34750462 542 32.28782288 543 32.22836096 544 32.16911765 545 32.11009174 546 32.05128205 547 32.17550274 548 32.29927007 549 32.24043716 550 32.18181818 551 32.12341198 552 32.24637681 553 32.36889693 554 32.31046931 555 32.25225225 556 32.19424460 557 32.13644524 558 32.07885305 559 32.02146691 560 32.14285714 561 32.26381462 562 32.20640569 563 32.14920071 564 32.09219858 565 32.21238938 566 32.33215548 567 32.27513228 568 32.39436620 569 32.51318102 570 32.45614035 571 32.57443082 572 32.51748252 573 32.46073298 574 32.57839721 575 32.69565217 576 32.81250000 577 32.92894281 578 33.04498270 579 33.16062176 580 33.27586207 581 33.21858864 582 33.33333333 583 33.27615780 584 33.39041096 585 33.33333333 586 33.44709898 587 33.39011925 588 33.33333333 589 33.27674024 590 33.22033898 591 33.16412860 592 33.10810811 593 33.22091062 594 33.33333333 595 33.27731092 596 33.22147651 597 33.16582915 598 33.11036789 599 33.05509182 600 33.00000000 601 33.11148087 602 33.22259136 603 33.33333333 604 33.27814570 605 33.22314050 606 33.33333333 607 33.27841845 608 33.22368421 609 33.16912972 610 33.11475410 611 33.06055646 612 33.16993464 613 33.11582382 614 33.06188925 615 33.00813008 616 33.11688312 617 33.22528363 618 33.17152104 619 33.27948304 620 33.22580645 621 33.17230274 622 33.27974277 623 33.22632424 624 33.17307692 625 33.12000000 626 33.06709265 627 33.17384370 628 33.28025478 629 33.22734499 630 33.17460317 631 33.28050713 632 33.22784810 633 33.17535545 634 33.28075710 635 33.22834646 636 33.17610063 637 33.12401884 638 33.07210031 639 33.02034429 640 33.12500000 641 33.07332293 642 33.17757009 643 33.12597201 644 33.07453416 645 33.17829457 646 33.12693498 647 33.23029366 648 33.33333333 649 33.28197227 650 33.38461538 651 33.48694316 652 33.58895706 653 33.69065850 654 33.63914373 655 33.58778626 656 33.68902439 657 33.78995434 658 33.73860182 659 33.83915023 660 33.78787879 661 33.73676248 662 33.68580060 663 33.63499246 664 33.58433735 665 33.68421053 666 33.78378378 667 33.88305847 668 33.83233533 669 33.78176383 670 33.88059701 671 33.83010432 672 33.77976190 673 33.72956909 674 33.67952522 675 33.77777778 676 33.87573964 677 33.82570162 678 33.77581121 679 33.72606775 680 33.67647059 681 33.62701909 682 33.72434018 683 33.67496340 684 33.77192982 685 33.72262774 686 33.67346939 687 33.62445415 688 33.57558140 689 33.67198839 690 33.62318841 691 33.71924747 692 33.81502890 693 33.76623377 694 33.71757925 695 33.66906475 696 33.76436782 697 33.71592539 698 33.66762178 699 33.61945637 700 33.71428571 701 33.66619116 702 33.61823362 703 33.57041252 704 33.52272727 705 33.47517730 706 33.56940510 707 33.66336634 708 33.61581921 709 33.56840621 710 33.52112676 711 33.47398031 712 33.56741573 713 33.52033661 714 33.47338936 715 33.56643357 716 33.51955307 717 33.61227336 718 33.56545961 719 33.65785814 720 33.61111111 721 33.56449376 722 33.51800554 723 33.47164592 724 33.42541436 725 33.37931034 726 33.47107438 727 33.42503439 728 33.37912088 729 33.47050754 730 33.56164384 731 33.51573187 732 33.60655738 733 33.56070941 734 33.51498638 735 33.46938776 736 33.42391304 737 33.51424695 738 33.46883469 739 33.42354533 740 33.37837838 741 33.46828610 742 33.42318059 743 33.37819650 744 33.46774194 745 33.42281879 746 33.51206434 747 33.60107095 748 33.55614973 749 33.64485981 750 33.73333333 Final result: 33.7333 +/- 1.7276 Random chance: 19.8992 +/- 1.4588