| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 817 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 817 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 0.00000000 | |
| 2 0.00000000 | |
| 3 0.00000000 | |
| 4 0.00000000 | |
| 5 0.00000000 | |
| 6 16.66666667 | |
| 7 14.28571429 | |
| 8 25.00000000 | |
| 9 22.22222222 | |
| 10 20.00000000 | |
| 11 18.18181818 | |
| 12 25.00000000 | |
| 13 23.07692308 | |
| 14 21.42857143 | |
| 15 20.00000000 | |
| 16 25.00000000 | |
| 17 29.41176471 | |
| 18 27.77777778 | |
| 19 26.31578947 | |
| 20 25.00000000 | |
| 21 23.80952381 | |
| 22 22.72727273 | |
| 23 21.73913043 | |
| 24 25.00000000 | |
| 25 24.00000000 | |
| 26 23.07692308 | |
| 27 22.22222222 | |
| 28 21.42857143 | |
| 29 24.13793103 | |
| 30 23.33333333 | |
| 31 22.58064516 | |
| 32 25.00000000 | |
| 33 27.27272727 | |
| 34 26.47058824 | |
| 35 25.71428571 | |
| 36 25.00000000 | |
| 37 24.32432432 | |
| 38 23.68421053 | |
| 39 23.07692308 | |
| 40 25.00000000 | |
| 41 26.82926829 | |
| 42 26.19047619 | |
| 43 27.90697674 | |
| 44 27.27272727 | |
| 45 28.88888889 | |
| 46 30.43478261 | |
| 47 31.91489362 | |
| 48 33.33333333 | |
| 49 32.65306122 | |
| 50 34.00000000 | |
| 51 35.29411765 | |
| 52 34.61538462 | |
| 53 35.84905660 | |
| 54 35.18518519 | |
| 55 36.36363636 | |
| 56 35.71428571 | |
| 57 35.08771930 | |
| 58 34.48275862 | |
| 59 33.89830508 | |
| 60 35.00000000 | |
| 61 34.42622951 | |
| 62 35.48387097 | |
| 63 34.92063492 | |
| 64 34.37500000 | |
| 65 33.84615385 | |
| 66 33.33333333 | |
| 67 32.83582090 | |
| 68 32.35294118 | |
| 69 33.33333333 | |
| 70 32.85714286 | |
| 71 33.80281690 | |
| 72 34.72222222 | |
| 73 34.24657534 | |
| 74 33.78378378 | |
| 75 33.33333333 | |
| 76 32.89473684 | |
| 77 32.46753247 | |
| 78 32.05128205 | |
| 79 31.64556962 | |
| 80 32.50000000 | |
| 81 32.09876543 | |
| 82 32.92682927 | |
| 83 32.53012048 | |
| 84 32.14285714 | |
| 85 31.76470588 | |
| 86 31.39534884 | |
| 87 31.03448276 | |
| 88 30.68181818 | |
| 89 31.46067416 | |
| 90 31.11111111 | |
| 91 31.86813187 | |
| 92 31.52173913 | |
| 93 31.18279570 | |
| 94 30.85106383 | |
| 95 30.52631579 | |
| 96 31.25000000 | |
| 97 31.95876289 | |
| 98 31.63265306 | |
| 99 32.32323232 | |
| 100 32.00000000 | |
| 101 31.68316832 | |
| 102 31.37254902 | |
| 103 31.06796117 | |
| 104 30.76923077 | |
| 105 30.47619048 | |
| 106 30.18867925 | |
| 107 30.84112150 | |
| 108 31.48148148 | |
| 109 31.19266055 | |
| 110 31.81818182 | |
| 111 31.53153153 | |
| 112 32.14285714 | |
| 113 32.74336283 | |
| 114 32.45614035 | |
| 115 32.17391304 | |
| 116 32.75862069 | |
| 117 32.47863248 | |
| 118 32.20338983 | |
| 119 31.93277311 | |
| 120 31.66666667 | |
| 121 32.23140496 | |
| 122 31.96721311 | |
| 123 32.52032520 | |
| 124 32.25806452 | |
| 125 32.00000000 | |
| 126 32.53968254 | |
| 127 32.28346457 | |
| 128 32.03125000 | |
| 129 31.78294574 | |
| 130 31.53846154 | |
| 131 31.29770992 | |
| 132 31.06060606 | |
| 133 30.82706767 | |
| 134 31.34328358 | |
| 135 31.85185185 | |
| 136 31.61764706 | |
| 137 32.11678832 | |
| 138 32.60869565 | |
| 139 33.09352518 | |
| 140 32.85714286 | |
| 141 32.62411348 | |
| 142 32.39436620 | |
| 143 32.16783217 | |
| 144 32.63888889 | |
| 145 32.41379310 | |
| 146 32.19178082 | |
| 147 31.97278912 | |
| 148 31.75675676 | |
| 149 31.54362416 | |
| 150 31.33333333 | |
| 151 31.78807947 | |
| 152 31.57894737 | |
| 153 31.37254902 | |
| 154 31.16883117 | |
| 155 30.96774194 | |
| 156 31.41025641 | |
| 157 31.21019108 | |
| 158 31.64556962 | |
| 159 32.07547170 | |
| 160 31.87500000 | |
| 161 32.29813665 | |
| 162 32.09876543 | |
| 163 31.90184049 | |
| 164 31.70731707 | |
| 165 31.51515152 | |
| 166 31.92771084 | |
| 167 31.73652695 | |
| 168 31.54761905 | |
| 169 31.36094675 | |
| 170 31.17647059 | |
| 171 30.99415205 | |
| 172 30.81395349 | |
| 173 30.63583815 | |
| 174 31.03448276 | |
| 175 30.85714286 | |
| 176 31.25000000 | |
| 177 31.07344633 | |
| 178 30.89887640 | |
| 179 30.72625698 | |
| 180 30.55555556 | |
| 181 30.38674033 | |
| 182 30.21978022 | |
| 183 30.05464481 | |
| 184 29.89130435 | |
| 185 29.72972973 | |
| 186 30.10752688 | |
| 187 29.94652406 | |
| 188 29.78723404 | |
| 189 30.15873016 | |
| 190 30.00000000 | |
| 191 29.84293194 | |
| 192 29.68750000 | |
| 193 29.53367876 | |
| 194 29.89690722 | |
| 195 29.74358974 | |
| 196 29.59183673 | |
| 197 29.94923858 | |
| 198 30.30303030 | |
| 199 30.15075377 | |
| 200 30.50000000 | |
| 201 30.34825871 | |
| 202 30.69306931 | |
| 203 30.54187192 | |
| 204 30.39215686 | |
| 205 30.73170732 | |
| 206 30.58252427 | |
| 207 30.91787440 | |
| 208 30.76923077 | |
| 209 30.62200957 | |
| 210 30.47619048 | |
| 211 30.33175355 | |
| 212 30.18867925 | |
| 213 30.04694836 | |
| 214 29.90654206 | |
| 215 29.76744186 | |
| 216 30.09259259 | |
| 217 30.41474654 | |
| 218 30.73394495 | |
| 219 30.59360731 | |
| 220 30.45454545 | |
| 221 30.31674208 | |
| 222 30.18018018 | |
| 223 30.04484305 | |
| 224 30.35714286 | |
| 225 30.66666667 | |
| 226 30.53097345 | |
| 227 30.83700441 | |
| 228 30.70175439 | |
| 229 30.56768559 | |
| 230 30.43478261 | |
| 231 30.30303030 | |
| 232 30.17241379 | |
| 233 30.04291845 | |
| 234 30.34188034 | |
| 235 30.21276596 | |
| 236 30.50847458 | |
| 237 30.80168776 | |
| 238 31.09243697 | |
| 239 31.38075314 | |
| 240 31.25000000 | |
| 241 31.12033195 | |
| 242 30.99173554 | |
| 243 30.86419753 | |
| 244 30.73770492 | |
| 245 30.61224490 | |
| 246 30.48780488 | |
| 247 30.36437247 | |
| 248 30.24193548 | |
| 249 30.52208835 | |
| 250 30.40000000 | |
| 251 30.67729084 | |
| 252 30.55555556 | |
| 253 30.43478261 | |
| 254 30.31496063 | |
| 255 30.19607843 | |
| 256 30.07812500 | |
| 257 29.96108949 | |
| 258 29.84496124 | |
| 259 29.72972973 | |
| 260 29.61538462 | |
| 261 29.88505747 | |
| 262 30.15267176 | |
| 263 30.03802281 | |
| 264 30.30303030 | |
| 265 30.56603774 | |
| 266 30.45112782 | |
| 267 30.33707865 | |
| 268 30.59701493 | |
| 269 30.48327138 | |
| 270 30.37037037 | |
| 271 30.25830258 | |
| 272 30.14705882 | |
| 273 30.40293040 | |
| 274 30.65693431 | |
| 275 30.54545455 | |
| 276 30.43478261 | |
| 277 30.32490975 | |
| 278 30.21582734 | |
| 279 30.10752688 | |
| 280 30.35714286 | |
| 281 30.60498221 | |
| 282 30.49645390 | |
| 283 30.38869258 | |
| 284 30.28169014 | |
| 285 30.17543860 | |
| 286 30.41958042 | |
| 287 30.66202091 | |
| 288 30.55555556 | |
| 289 30.44982699 | |
| 290 30.34482759 | |
| 291 30.58419244 | |
| 292 30.47945205 | |
| 293 30.37542662 | |
| 294 30.27210884 | |
| 295 30.50847458 | |
| 296 30.74324324 | |
| 297 30.63973064 | |
| 298 30.53691275 | |
| 299 30.43478261 | |
| 300 30.33333333 | |
| 301 30.23255814 | |
| 302 30.13245033 | |
| 303 30.36303630 | |
| 304 30.26315789 | |
| 305 30.16393443 | |
| 306 30.06535948 | |
| 307 29.96742671 | |
| 308 30.19480519 | |
| 309 30.09708738 | |
| 310 30.00000000 | |
| 311 29.90353698 | |
| 312 30.12820513 | |
| 313 30.03194888 | |
| 314 29.93630573 | |
| 315 29.84126984 | |
| 316 29.74683544 | |
| 317 29.65299685 | |
| 318 29.55974843 | |
| 319 29.46708464 | |
| 320 29.37500000 | |
| 321 29.28348910 | |
| 322 29.50310559 | |
| 323 29.41176471 | |
| 324 29.32098765 | |
| 325 29.23076923 | |
| 326 29.14110429 | |
| 327 29.35779817 | |
| 328 29.57317073 | |
| 329 29.48328267 | |
| 330 29.69696970 | |
| 331 29.60725076 | |
| 332 29.81927711 | |
| 333 30.03003003 | |
| 334 29.94011976 | |
| 335 30.14925373 | |
| 336 30.35714286 | |
| 337 30.26706231 | |
| 338 30.17751479 | |
| 339 30.38348083 | |
| 340 30.29411765 | |
| 341 30.20527859 | |
| 342 30.40935673 | |
| 343 30.32069971 | |
| 344 30.52325581 | |
| 345 30.72463768 | |
| 346 30.63583815 | |
| 347 30.83573487 | |
| 348 30.74712644 | |
| 349 30.65902579 | |
| 350 30.57142857 | |
| 351 30.48433048 | |
| 352 30.68181818 | |
| 353 30.59490085 | |
| 354 30.50847458 | |
| 355 30.70422535 | |
| 356 30.61797753 | |
| 357 30.81232493 | |
| 358 30.72625698 | |
| 359 30.64066852 | |
| 360 30.55555556 | |
| 361 30.74792244 | |
| 362 30.66298343 | |
| 363 30.85399449 | |
| 364 30.76923077 | |
| 365 30.68493151 | |
| 366 30.60109290 | |
| 367 30.79019074 | |
| 368 30.70652174 | |
| 369 30.62330623 | |
| 370 30.54054054 | |
| 371 30.45822102 | |
| 372 30.64516129 | |
| 373 30.56300268 | |
| 374 30.74866310 | |
| 375 30.66666667 | |
| 376 30.58510638 | |
| 377 30.76923077 | |
| 378 30.68783069 | |
| 379 30.87071240 | |
| 380 31.05263158 | |
| 381 30.97112861 | |
| 382 30.89005236 | |
| 383 30.80939948 | |
| 384 30.72916667 | |
| 385 30.64935065 | |
| 386 30.56994819 | |
| 387 30.49095607 | |
| 388 30.41237113 | |
| 389 30.33419023 | |
| 390 30.51282051 | |
| 391 30.69053708 | |
| 392 30.61224490 | |
| 393 30.78880407 | |
| 394 30.96446701 | |
| 395 30.88607595 | |
| 396 30.80808081 | |
| 397 30.98236776 | |
| 398 30.90452261 | |
| 399 30.82706767 | |
| 400 30.75000000 | |
| 401 30.92269327 | |
| 402 31.09452736 | |
| 403 31.26550868 | |
| 404 31.18811881 | |
| 405 31.11111111 | |
| 406 31.03448276 | |
| 407 30.95823096 | |
| 408 30.88235294 | |
| 409 30.80684597 | |
| 410 30.73170732 | |
| 411 30.65693431 | |
| 412 30.82524272 | |
| 413 30.99273608 | |
| 414 31.15942029 | |
| 415 31.08433735 | |
| 416 31.00961538 | |
| 417 30.93525180 | |
| 418 30.86124402 | |
| 419 30.78758950 | |
| 420 30.95238095 | |
| 421 31.11638955 | |
| 422 31.04265403 | |
| 423 30.96926714 | |
| 424 30.89622642 | |
| 425 30.82352941 | |
| 426 30.75117371 | |
| 427 30.67915691 | |
| 428 30.60747664 | |
| 429 30.53613054 | |
| 430 30.46511628 | |
| 431 30.39443155 | |
| 432 30.55555556 | |
| 433 30.71593533 | |
| 434 30.87557604 | |
| 435 30.80459770 | |
| 436 30.73394495 | |
| 437 30.66361556 | |
| 438 30.59360731 | |
| 439 30.75170843 | |
| 440 30.90909091 | |
| 441 30.83900227 | |
| 442 30.76923077 | |
| 443 30.69977427 | |
| 444 30.85585586 | |
| 445 30.78651685 | |
| 446 30.71748879 | |
| 447 30.64876957 | |
| 448 30.58035714 | |
| 449 30.73496659 | |
| 450 30.66666667 | |
| 451 30.59866962 | |
| 452 30.53097345 | |
| 453 30.68432671 | |
| 454 30.61674009 | |
| 455 30.54945055 | |
| 456 30.48245614 | |
| 457 30.63457330 | |
| 458 30.56768559 | |
| 459 30.50108932 | |
| 460 30.43478261 | |
| 461 30.36876356 | |
| 462 30.30303030 | |
| 463 30.45356371 | |
| 464 30.60344828 | |
| 465 30.75268817 | |
| 466 30.68669528 | |
| 467 30.62098501 | |
| 468 30.76923077 | |
| 469 30.70362473 | |
| 470 30.63829787 | |
| 471 30.78556263 | |
| 472 30.93220339 | |
| 473 30.86680761 | |
| 474 30.80168776 | |
| 475 30.94736842 | |
| 476 30.88235294 | |
| 477 31.02725367 | |
| 478 30.96234310 | |
| 479 30.89770355 | |
| 480 30.83333333 | |
| 481 30.76923077 | |
| 482 30.91286307 | |
| 483 30.84886128 | |
| 484 30.99173554 | |
| 485 31.13402062 | |
| 486 31.06995885 | |
| 487 31.21149897 | |
| 488 31.14754098 | |
| 489 31.08384458 | |
| 490 31.22448980 | |
| 491 31.36456212 | |
| 492 31.50406504 | |
| 493 31.44016227 | |
| 494 31.57894737 | |
| 495 31.71717172 | |
| 496 31.65322581 | |
| 497 31.58953722 | |
| 498 31.52610442 | |
| 499 31.46292585 | |
| 500 31.40000000 | |
| 501 31.53692615 | |
| 502 31.47410359 | |
| 503 31.41153082 | |
| 504 31.54761905 | |
| 505 31.48514851 | |
| 506 31.42292490 | |
| 507 31.55818540 | |
| 508 31.49606299 | |
| 509 31.63064833 | |
| 510 31.76470588 | |
| 511 31.89823875 | |
| 512 31.83593750 | |
| 513 31.77387914 | |
| 514 31.71206226 | |
| 515 31.84466019 | |
| 516 31.78294574 | |
| 517 31.91489362 | |
| 518 31.85328185 | |
| 519 31.79190751 | |
| 520 31.73076923 | |
| 521 31.86180422 | |
| 522 31.80076628 | |
| 523 31.73996176 | |
| 524 31.67938931 | |
| 525 31.80952381 | |
| 526 31.74904943 | |
| 527 31.68880455 | |
| 528 31.62878788 | |
| 529 31.75803403 | |
| 530 31.69811321 | |
| 531 31.82674200 | |
| 532 31.95488722 | |
| 533 32.08255159 | |
| 534 32.02247191 | |
| 535 31.96261682 | |
| 536 31.90298507 | |
| 537 32.02979516 | |
| 538 31.97026022 | |
| 539 31.91094620 | |
| 540 32.03703704 | |
| 541 31.97781885 | |
| 542 31.91881919 | |
| 543 31.86003683 | |
| 544 31.80147059 | |
| 545 31.74311927 | |
| 546 31.68498168 | |
| 547 31.80987203 | |
| 548 31.93430657 | |
| 549 31.87613843 | |
| 550 31.81818182 | |
| 551 31.76043557 | |
| 552 31.88405797 | |
| 553 32.00723327 | |
| 554 31.94945848 | |
| 555 32.07207207 | |
| 556 32.19424460 | |
| 557 32.13644524 | |
| 558 32.07885305 | |
| 559 32.02146691 | |
| 560 32.14285714 | |
| 561 32.08556150 | |
| 562 32.02846975 | |
| 563 31.97158082 | |
| 564 32.09219858 | |
| 565 32.21238938 | |
| 566 32.33215548 | |
| 567 32.27513228 | |
| 568 32.21830986 | |
| 569 32.16168717 | |
| 570 32.10526316 | |
| 571 32.22416813 | |
| 572 32.16783217 | |
| 573 32.11169284 | |
| 574 32.22996516 | |
| 575 32.34782609 | |
| 576 32.46527778 | |
| 577 32.58232236 | |
| 578 32.69896194 | |
| 579 32.81519862 | |
| 580 32.75862069 | |
| 581 32.70223752 | |
| 582 32.81786942 | |
| 583 32.76157804 | |
| 584 32.87671233 | |
| 585 32.82051282 | |
| 586 32.76450512 | |
| 587 32.87904600 | |
| 588 32.82312925 | |
| 589 32.76740238 | |
| 590 32.71186441 | |
| 591 32.65651438 | |
| 592 32.60135135 | |
| 593 32.54637437 | |
| 594 32.65993266 | |
| 595 32.60504202 | |
| 596 32.55033557 | |
| 597 32.66331658 | |
| 598 32.60869565 | |
| 599 32.55425710 | |
| 600 32.50000000 | |
| 601 32.61231281 | |
| 602 32.72425249 | |
| 603 32.83582090 | |
| 604 32.78145695 | |
| 605 32.72727273 | |
| 606 32.83828383 | |
| 607 32.78418451 | |
| 608 32.73026316 | |
| 609 32.84072250 | |
| 610 32.78688525 | |
| 611 32.73322422 | |
| 612 32.84313725 | |
| 613 32.78955954 | |
| 614 32.73615635 | |
| 615 32.68292683 | |
| 616 32.79220779 | |
| 617 32.90113452 | |
| 618 32.84789644 | |
| 619 32.95638126 | |
| 620 32.90322581 | |
| 621 32.85024155 | |
| 622 32.95819936 | |
| 623 32.90529695 | |
| 624 32.85256410 | |
| 625 32.80000000 | |
| 626 32.74760383 | |
| 627 32.85486443 | |
| 628 32.96178344 | |
| 629 32.90937997 | |
| 630 32.85714286 | |
| 631 32.80507132 | |
| 632 32.91139241 | |
| 633 32.85939968 | |
| 634 32.80757098 | |
| 635 32.75590551 | |
| 636 32.70440252 | |
| 637 32.65306122 | |
| 638 32.60188088 | |
| 639 32.55086072 | |
| 640 32.65625000 | |
| 641 32.60530421 | |
| 642 32.55451713 | |
| 643 32.50388802 | |
| 644 32.60869565 | |
| 645 32.71317829 | |
| 646 32.66253870 | |
| 647 32.61205564 | |
| 648 32.71604938 | |
| 649 32.66563945 | |
| 650 32.76923077 | |
| 651 32.87250384 | |
| 652 32.82208589 | |
| 653 32.92496172 | |
| 654 32.87461774 | |
| 655 32.82442748 | |
| 656 32.77439024 | |
| 657 32.87671233 | |
| 658 32.82674772 | |
| 659 32.92867982 | |
| 660 32.87878788 | |
| 661 32.82904690 | |
| 662 32.77945619 | |
| 663 32.73001508 | |
| 664 32.68072289 | |
| 665 32.78195489 | |
| 666 32.73273273 | |
| 667 32.68365817 | |
| 668 32.63473054 | |
| 669 32.73542601 | |
| 670 32.83582090 | |
| 671 32.78688525 | |
| 672 32.73809524 | |
| 673 32.68945022 | |
| 674 32.64094955 | |
| 675 32.74074074 | |
| 676 32.84023669 | |
| 677 32.79172821 | |
| 678 32.74336283 | |
| 679 32.69513991 | |
| 680 32.64705882 | |
| 681 32.74596182 | |
| 682 32.69794721 | |
| 683 32.65007321 | |
| 684 32.74853801 | |
| 685 32.70072993 | |
| 686 32.65306122 | |
| 687 32.60553130 | |
| 688 32.70348837 | |
| 689 32.80116110 | |
| 690 32.75362319 | |
| 691 32.85094067 | |
| 692 32.94797688 | |
| 693 33.04473304 | |
| 694 32.99711816 | |
| 695 32.94964029 | |
| 696 33.04597701 | |
| 697 32.99856528 | |
| 698 32.95128940 | |
| 699 32.90414878 | |
| 700 33.00000000 | |
| 701 32.95292439 | |
| 702 32.90598291 | |
| 703 32.85917496 | |
| 704 32.81250000 | |
| 705 32.76595745 | |
| 706 32.86118980 | |
| 707 32.95615276 | |
| 708 32.90960452 | |
| 709 32.86318759 | |
| 710 32.81690141 | |
| 711 32.77074543 | |
| 712 32.72471910 | |
| 713 32.67882188 | |
| 714 32.63305322 | |
| 715 32.58741259 | |
| 716 32.54189944 | |
| 717 32.49651325 | |
| 718 32.59052925 | |
| 719 32.68428373 | |
| 720 32.63888889 | |
| 721 32.59361997 | |
| 722 32.54847645 | |
| 723 32.50345781 | |
| 724 32.45856354 | |
| 725 32.41379310 | |
| 726 32.50688705 | |
| 727 32.46217331 | |
| 728 32.55494505 | |
| 729 32.64746228 | |
| 730 32.73972603 | |
| 731 32.69493844 | |
| 732 32.78688525 | |
| 733 32.74215553 | |
| 734 32.69754768 | |
| 735 32.65306122 | |
| 736 32.60869565 | |
| 737 32.70013569 | |
| 738 32.65582656 | |
| 739 32.74695535 | |
| 740 32.70270270 | |
| 741 32.65856950 | |
| 742 32.61455526 | |
| 743 32.57065949 | |
| 744 32.66129032 | |
| 745 32.61744966 | |
| 746 32.57372654 | |
| 747 32.66398929 | |
| 748 32.62032086 | |
| 749 32.71028037 | |
| 750 32.80000000 | |
| Final result: 32.8000 +/- 1.7155 | |
| Random chance: 19.8992 +/- 1.4588 | |