| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 1548 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 1548 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 50.00000000 | |
| 5 40.00000000 | |
| 6 33.33333333 | |
| 7 42.85714286 | |
| 8 50.00000000 | |
| 9 44.44444444 | |
| 10 40.00000000 | |
| 11 36.36363636 | |
| 12 33.33333333 | |
| 13 38.46153846 | |
| 14 35.71428571 | |
| 15 33.33333333 | |
| 16 37.50000000 | |
| 17 41.17647059 | |
| 18 38.88888889 | |
| 19 42.10526316 | |
| 20 45.00000000 | |
| 21 47.61904762 | |
| 22 45.45454545 | |
| 23 43.47826087 | |
| 24 41.66666667 | |
| 25 44.00000000 | |
| 26 42.30769231 | |
| 27 40.74074074 | |
| 28 39.28571429 | |
| 29 41.37931034 | |
| 30 43.33333333 | |
| 31 45.16129032 | |
| 32 46.87500000 | |
| 33 45.45454545 | |
| 34 44.11764706 | |
| 35 45.71428571 | |
| 36 44.44444444 | |
| 37 43.24324324 | |
| 38 42.10526316 | |
| 39 41.02564103 | |
| 40 40.00000000 | |
| 41 41.46341463 | |
| 42 40.47619048 | |
| 43 41.86046512 | |
| 44 40.90909091 | |
| 45 42.22222222 | |
| 46 41.30434783 | |
| 47 40.42553191 | |
| 48 41.66666667 | |
| 49 42.85714286 | |
| 50 42.00000000 | |
| 51 41.17647059 | |
| 52 42.30769231 | |
| 53 41.50943396 | |
| 54 40.74074074 | |
| 55 41.81818182 | |
| 56 42.85714286 | |
| 57 43.85964912 | |
| 58 43.10344828 | |
| 59 42.37288136 | |
| 60 41.66666667 | |
| 61 42.62295082 | |
| 62 41.93548387 | |
| 63 41.26984127 | |
| 64 40.62500000 | |
| 65 40.00000000 | |
| 66 40.90909091 | |
| 67 40.29850746 | |
| 68 39.70588235 | |
| 69 39.13043478 | |
| 70 40.00000000 | |
| 71 40.84507042 | |
| 72 40.27777778 | |
| 73 39.72602740 | |
| 74 40.54054054 | |
| 75 41.33333333 | |
| 76 42.10526316 | |
| 77 42.85714286 | |
| 78 43.58974359 | |
| 79 43.03797468 | |
| 80 42.50000000 | |
| 81 43.20987654 | |
| 82 42.68292683 | |
| 83 43.37349398 | |
| 84 42.85714286 | |
| 85 42.35294118 | |
| 86 41.86046512 | |
| 87 42.52873563 | |
| 88 43.18181818 | |
| 89 42.69662921 | |
| 90 42.22222222 | |
| 91 41.75824176 | |
| 92 41.30434783 | |
| 93 40.86021505 | |
| 94 40.42553191 | |
| 95 41.05263158 | |
| 96 41.66666667 | |
| 97 41.23711340 | |
| 98 40.81632653 | |
| 99 40.40404040 | |
| 100 40.00000000 | |
| 101 40.59405941 | |
| 102 40.19607843 | |
| 103 39.80582524 | |
| 104 40.38461538 | |
| 105 40.95238095 | |
| 106 40.56603774 | |
| 107 40.18691589 | |
| 108 40.74074074 | |
| 109 40.36697248 | |
| 110 40.90909091 | |
| 111 40.54054054 | |
| 112 40.17857143 | |
| 113 40.70796460 | |
| 114 41.22807018 | |
| 115 41.73913043 | |
| 116 42.24137931 | |
| 117 41.88034188 | |
| 118 41.52542373 | |
| 119 42.01680672 | |
| 120 42.50000000 | |
| 121 42.97520661 | |
| 122 42.62295082 | |
| 123 42.27642276 | |
| 124 41.93548387 | |
| 125 42.40000000 | |
| 126 42.06349206 | |
| 127 41.73228346 | |
| 128 41.40625000 | |
| 129 41.08527132 | |
| 130 40.76923077 | |
| 131 41.22137405 | |
| 132 41.66666667 | |
| 133 41.35338346 | |
| 134 41.04477612 | |
| 135 40.74074074 | |
| 136 40.44117647 | |
| 137 40.14598540 | |
| 138 40.57971014 | |
| 139 40.28776978 | |
| 140 40.71428571 | |
| 141 41.13475177 | |
| 142 41.54929577 | |
| 143 41.95804196 | |
| 144 41.66666667 | |
| 145 42.06896552 | |
| 146 41.78082192 | |
| 147 41.49659864 | |
| 148 41.89189189 | |
| 149 41.61073826 | |
| 150 42.00000000 | |
| 151 41.72185430 | |
| 152 41.44736842 | |
| 153 41.83006536 | |
| 154 41.55844156 | |
| 155 41.29032258 | |
| 156 41.02564103 | |
| 157 41.40127389 | |
| 158 41.13924051 | |
| 159 40.88050314 | |
| 160 41.25000000 | |
| 161 40.99378882 | |
| 162 40.74074074 | |
| 163 41.10429448 | |
| 164 41.46341463 | |
| 165 41.81818182 | |
| 166 41.56626506 | |
| 167 41.91616766 | |
| 168 41.66666667 | |
| 169 42.01183432 | |
| 170 41.76470588 | |
| 171 42.10526316 | |
| 172 41.86046512 | |
| 173 41.61849711 | |
| 174 41.37931034 | |
| 175 41.14285714 | |
| 176 40.90909091 | |
| 177 40.67796610 | |
| 178 40.44943820 | |
| 179 40.22346369 | |
| 180 40.00000000 | |
| 181 39.77900552 | |
| 182 39.56043956 | |
| 183 39.89071038 | |
| 184 39.67391304 | |
| 185 40.00000000 | |
| 186 40.32258065 | |
| 187 40.10695187 | |
| 188 40.42553191 | |
| 189 40.74074074 | |
| 190 40.52631579 | |
| 191 40.83769634 | |
| 192 40.62500000 | |
| 193 40.41450777 | |
| 194 40.72164948 | |
| 195 41.02564103 | |
| 196 41.32653061 | |
| 197 41.11675127 | |
| 198 41.41414141 | |
| 199 41.20603015 | |
| 200 41.00000000 | |
| 201 40.79601990 | |
| 202 40.59405941 | |
| 203 40.39408867 | |
| 204 40.19607843 | |
| 205 40.48780488 | |
| 206 40.29126214 | |
| 207 40.57971014 | |
| 208 40.38461538 | |
| 209 40.19138756 | |
| 210 40.00000000 | |
| 211 40.28436019 | |
| 212 40.09433962 | |
| 213 39.90610329 | |
| 214 39.71962617 | |
| 215 40.00000000 | |
| 216 39.81481481 | |
| 217 40.09216590 | |
| 218 39.90825688 | |
| 219 40.18264840 | |
| 220 40.45454545 | |
| 221 40.27149321 | |
| 222 40.54054054 | |
| 223 40.80717489 | |
| 224 40.62500000 | |
| 225 40.88888889 | |
| 226 40.70796460 | |
| 227 40.52863436 | |
| 228 40.78947368 | |
| 229 40.61135371 | |
| 230 40.43478261 | |
| 231 40.25974026 | |
| 232 40.51724138 | |
| 233 40.77253219 | |
| 234 41.02564103 | |
| 235 40.85106383 | |
| 236 40.67796610 | |
| 237 40.50632911 | |
| 238 40.33613445 | |
| 239 40.16736402 | |
| 240 40.00000000 | |
| 241 39.83402490 | |
| 242 40.08264463 | |
| 243 40.32921811 | |
| 244 40.16393443 | |
| 245 40.40816327 | |
| 246 40.24390244 | |
| 247 40.48582996 | |
| 248 40.72580645 | |
| 249 40.56224900 | |
| 250 40.40000000 | |
| 251 40.63745020 | |
| 252 40.47619048 | |
| 253 40.31620553 | |
| 254 40.15748031 | |
| 255 40.00000000 | |
| 256 40.23437500 | |
| 257 40.46692607 | |
| 258 40.31007752 | |
| 259 40.15444015 | |
| 260 40.00000000 | |
| 261 39.84674330 | |
| 262 39.69465649 | |
| 263 39.54372624 | |
| 264 39.39393939 | |
| 265 39.24528302 | |
| 266 39.09774436 | |
| 267 38.95131086 | |
| 268 38.80597015 | |
| 269 39.03345725 | |
| 270 38.88888889 | |
| 271 38.74538745 | |
| 272 38.97058824 | |
| 273 38.82783883 | |
| 274 38.68613139 | |
| 275 38.54545455 | |
| 276 38.76811594 | |
| 277 38.98916968 | |
| 278 38.84892086 | |
| 279 39.06810036 | |
| 280 39.28571429 | |
| 281 39.14590747 | |
| 282 39.36170213 | |
| 283 39.22261484 | |
| 284 39.08450704 | |
| 285 38.94736842 | |
| 286 38.81118881 | |
| 287 39.02439024 | |
| 288 38.88888889 | |
| 289 38.75432526 | |
| 290 38.96551724 | |
| 291 39.17525773 | |
| 292 39.38356164 | |
| 293 39.24914676 | |
| 294 39.11564626 | |
| 295 38.98305085 | |
| 296 38.85135135 | |
| 297 38.72053872 | |
| 298 38.92617450 | |
| 299 38.79598662 | |
| 300 39.00000000 | |
| 301 38.87043189 | |
| 302 39.07284768 | |
| 303 38.94389439 | |
| 304 39.14473684 | |
| 305 39.34426230 | |
| 306 39.21568627 | |
| 307 39.41368078 | |
| 308 39.28571429 | |
| 309 39.15857605 | |
| 310 39.03225806 | |
| 311 38.90675241 | |
| 312 39.10256410 | |
| 313 39.29712460 | |
| 314 39.17197452 | |
| 315 39.04761905 | |
| 316 38.92405063 | |
| 317 39.11671924 | |
| 318 39.30817610 | |
| 319 39.49843260 | |
| 320 39.37500000 | |
| 321 39.25233645 | |
| 322 39.13043478 | |
| 323 39.00928793 | |
| 324 38.88888889 | |
| 325 39.07692308 | |
| 326 38.95705521 | |
| 327 39.14373089 | |
| 328 39.02439024 | |
| 329 39.20972644 | |
| 330 39.09090909 | |
| 331 38.97280967 | |
| 332 38.85542169 | |
| 333 39.03903904 | |
| 334 38.92215569 | |
| 335 39.10447761 | |
| 336 38.98809524 | |
| 337 38.87240356 | |
| 338 38.75739645 | |
| 339 38.64306785 | |
| 340 38.82352941 | |
| 341 39.00293255 | |
| 342 39.18128655 | |
| 343 39.06705539 | |
| 344 38.95348837 | |
| 345 38.84057971 | |
| 346 39.01734104 | |
| 347 39.19308357 | |
| 348 39.08045977 | |
| 349 38.96848138 | |
| 350 38.85714286 | |
| 351 38.74643875 | |
| 352 38.63636364 | |
| 353 38.81019830 | |
| 354 38.70056497 | |
| 355 38.59154930 | |
| 356 38.76404494 | |
| 357 38.93557423 | |
| 358 38.82681564 | |
| 359 38.71866295 | |
| 360 38.88888889 | |
| 361 38.78116343 | |
| 362 38.95027624 | |
| 363 39.11845730 | |
| 364 39.01098901 | |
| 365 38.90410959 | |
| 366 39.07103825 | |
| 367 38.96457766 | |
| 368 38.85869565 | |
| 369 38.75338753 | |
| 370 38.91891892 | |
| 371 38.81401617 | |
| 372 38.70967742 | |
| 373 38.60589812 | |
| 374 38.77005348 | |
| 375 38.93333333 | |
| 376 38.82978723 | |
| 377 38.99204244 | |
| 378 39.15343915 | |
| 379 39.05013193 | |
| 380 38.94736842 | |
| 381 38.84514436 | |
| 382 38.74345550 | |
| 383 38.90339426 | |
| 384 38.80208333 | |
| 385 38.70129870 | |
| 386 38.60103627 | |
| 387 38.50129199 | |
| 388 38.40206186 | |
| 389 38.56041131 | |
| 390 38.71794872 | |
| 391 38.87468031 | |
| 392 39.03061224 | |
| 393 38.93129771 | |
| 394 38.83248731 | |
| 395 38.98734177 | |
| 396 39.14141414 | |
| 397 39.29471033 | |
| 398 39.19597990 | |
| 399 39.34837093 | |
| 400 39.25000000 | |
| 401 39.40149626 | |
| 402 39.55223881 | |
| 403 39.45409429 | |
| 404 39.35643564 | |
| 405 39.50617284 | |
| 406 39.40886700 | |
| 407 39.31203931 | |
| 408 39.21568627 | |
| 409 39.36430318 | |
| 410 39.26829268 | |
| 411 39.17274939 | |
| 412 39.07766990 | |
| 413 38.98305085 | |
| 414 38.88888889 | |
| 415 39.03614458 | |
| 416 39.18269231 | |
| 417 39.08872902 | |
| 418 39.23444976 | |
| 419 39.37947494 | |
| 420 39.28571429 | |
| 421 39.19239905 | |
| 422 39.09952607 | |
| 423 39.00709220 | |
| 424 38.91509434 | |
| 425 38.82352941 | |
| 426 38.73239437 | |
| 427 38.64168618 | |
| 428 38.55140187 | |
| 429 38.69463869 | |
| 430 38.60465116 | |
| 431 38.51508121 | |
| 432 38.65740741 | |
| 433 38.56812933 | |
| 434 38.47926267 | |
| 435 38.39080460 | |
| 436 38.30275229 | |
| 437 38.44393593 | |
| 438 38.58447489 | |
| 439 38.72437358 | |
| 440 38.63636364 | |
| 441 38.77551020 | |
| 442 38.68778281 | |
| 443 38.60045147 | |
| 444 38.51351351 | |
| 445 38.42696629 | |
| 446 38.34080717 | |
| 447 38.25503356 | |
| 448 38.16964286 | |
| 449 38.08463252 | |
| 450 38.00000000 | |
| 451 37.91574279 | |
| 452 37.83185841 | |
| 453 37.96909492 | |
| 454 37.88546256 | |
| 455 37.80219780 | |
| 456 37.93859649 | |
| 457 37.85557987 | |
| 458 37.77292576 | |
| 459 37.69063181 | |
| 460 37.82608696 | |
| 461 37.74403471 | |
| 462 37.66233766 | |
| 463 37.79697624 | |
| 464 37.93103448 | |
| 465 37.84946237 | |
| 466 37.76824034 | |
| 467 37.90149893 | |
| 468 37.82051282 | |
| 469 37.95309168 | |
| 470 37.87234043 | |
| 471 37.79193206 | |
| 472 37.71186441 | |
| 473 37.84355180 | |
| 474 37.76371308 | |
| 475 37.68421053 | |
| 476 37.81512605 | |
| 477 37.73584906 | |
| 478 37.86610879 | |
| 479 37.78705637 | |
| 480 37.70833333 | |
| 481 37.62993763 | |
| 482 37.55186722 | |
| 483 37.68115942 | |
| 484 37.80991736 | |
| 485 37.73195876 | |
| 486 37.65432099 | |
| 487 37.78234086 | |
| 488 37.70491803 | |
| 489 37.62781186 | |
| 490 37.75510204 | |
| 491 37.67820774 | |
| 492 37.60162602 | |
| 493 37.52535497 | |
| 494 37.65182186 | |
| 495 37.77777778 | |
| 496 37.90322581 | |
| 497 37.82696177 | |
| 498 37.75100402 | |
| 499 37.67535070 | |
| 500 37.60000000 | |
| 501 37.52495010 | |
| 502 37.45019920 | |
| 503 37.37574553 | |
| 504 37.50000000 | |
| 505 37.42574257 | |
| 506 37.54940711 | |
| 507 37.47534517 | |
| 508 37.59842520 | |
| 509 37.52455796 | |
| 510 37.45098039 | |
| 511 37.37769080 | |
| 512 37.30468750 | |
| 513 37.23196881 | |
| 514 37.15953307 | |
| 515 37.28155340 | |
| 516 37.40310078 | |
| 517 37.52417795 | |
| 518 37.64478764 | |
| 519 37.57225434 | |
| 520 37.50000000 | |
| 521 37.42802303 | |
| 522 37.54789272 | |
| 523 37.47609943 | |
| 524 37.59541985 | |
| 525 37.52380952 | |
| 526 37.45247148 | |
| 527 37.38140417 | |
| 528 37.31060606 | |
| 529 37.42911153 | |
| 530 37.54716981 | |
| 531 37.47645951 | |
| 532 37.40601504 | |
| 533 37.52345216 | |
| 534 37.64044944 | |
| 535 37.75700935 | |
| 536 37.68656716 | |
| 537 37.80260708 | |
| 538 37.91821561 | |
| 539 37.84786642 | |
| 540 37.77777778 | |
| 541 37.70794824 | |
| 542 37.82287823 | |
| 543 37.75322284 | |
| 544 37.68382353 | |
| 545 37.61467890 | |
| 546 37.72893773 | |
| 547 37.65996344 | |
| 548 37.77372263 | |
| 549 37.70491803 | |
| 550 37.63636364 | |
| 551 37.74954628 | |
| 552 37.68115942 | |
| 553 37.61301989 | |
| 554 37.54512635 | |
| 555 37.47747748 | |
| 556 37.58992806 | |
| 557 37.70197487 | |
| 558 37.81362007 | |
| 559 37.92486583 | |
| 560 37.85714286 | |
| 561 37.78966132 | |
| 562 37.90035587 | |
| 563 38.01065719 | |
| 564 37.94326241 | |
| 565 38.05309735 | |
| 566 38.16254417 | |
| 567 38.09523810 | |
| 568 38.02816901 | |
| 569 37.96133568 | |
| 570 37.89473684 | |
| 571 37.82837128 | |
| 572 37.76223776 | |
| 573 37.69633508 | |
| 574 37.63066202 | |
| 575 37.56521739 | |
| 576 37.67361111 | |
| 577 37.78162912 | |
| 578 37.71626298 | |
| 579 37.65112263 | |
| 580 37.75862069 | |
| 581 37.86574871 | |
| 582 37.80068729 | |
| 583 37.90737564 | |
| 584 38.01369863 | |
| 585 38.11965812 | |
| 586 38.22525597 | |
| 587 38.16013629 | |
| 588 38.09523810 | |
| 589 38.03056027 | |
| 590 37.96610169 | |
| 591 37.90186125 | |
| 592 37.83783784 | |
| 593 37.94266442 | |
| 594 38.04713805 | |
| 595 37.98319328 | |
| 596 38.08724832 | |
| 597 38.02345059 | |
| 598 37.95986622 | |
| 599 38.06343907 | |
| 600 38.00000000 | |
| 601 38.10316140 | |
| 602 38.03986711 | |
| 603 38.14262023 | |
| 604 38.07947020 | |
| 605 38.01652893 | |
| 606 37.95379538 | |
| 607 38.05601318 | |
| 608 37.99342105 | |
| 609 37.93103448 | |
| 610 37.86885246 | |
| 611 37.97054010 | |
| 612 37.90849673 | |
| 613 37.84665579 | |
| 614 37.78501629 | |
| 615 37.72357724 | |
| 616 37.66233766 | |
| 617 37.76337115 | |
| 618 37.70226537 | |
| 619 37.64135703 | |
| 620 37.58064516 | |
| 621 37.68115942 | |
| 622 37.62057878 | |
| 623 37.56019262 | |
| 624 37.66025641 | |
| 625 37.60000000 | |
| 626 37.69968051 | |
| 627 37.63955343 | |
| 628 37.57961783 | |
| 629 37.67885533 | |
| 630 37.77777778 | |
| 631 37.71790808 | |
| 632 37.81645570 | |
| 633 37.75671406 | |
| 634 37.69716088 | |
| 635 37.63779528 | |
| 636 37.57861635 | |
| 637 37.51962323 | |
| 638 37.46081505 | |
| 639 37.55868545 | |
| 640 37.65625000 | |
| 641 37.59750390 | |
| 642 37.53894081 | |
| 643 37.48055988 | |
| 644 37.57763975 | |
| 645 37.51937984 | |
| 646 37.46130031 | |
| 647 37.40340031 | |
| 648 37.34567901 | |
| 649 37.28813559 | |
| 650 37.23076923 | |
| 651 37.17357911 | |
| 652 37.11656442 | |
| 653 37.05972435 | |
| 654 37.15596330 | |
| 655 37.09923664 | |
| 656 37.19512195 | |
| 657 37.13850837 | |
| 658 37.23404255 | |
| 659 37.32928680 | |
| 660 37.27272727 | |
| 661 37.21633888 | |
| 662 37.16012085 | |
| 663 37.10407240 | |
| 664 37.04819277 | |
| 665 36.99248120 | |
| 666 36.93693694 | |
| 667 36.88155922 | |
| 668 36.97604790 | |
| 669 37.07025411 | |
| 670 37.01492537 | |
| 671 36.95976155 | |
| 672 36.90476190 | |
| 673 36.84992571 | |
| 674 36.79525223 | |
| 675 36.74074074 | |
| 676 36.68639053 | |
| 677 36.63220089 | |
| 678 36.57817109 | |
| 679 36.52430044 | |
| 680 36.61764706 | |
| 681 36.71071953 | |
| 682 36.65689150 | |
| 683 36.60322108 | |
| 684 36.69590643 | |
| 685 36.64233577 | |
| 686 36.58892128 | |
| 687 36.53566230 | |
| 688 36.48255814 | |
| 689 36.57474601 | |
| 690 36.52173913 | |
| 691 36.46888567 | |
| 692 36.41618497 | |
| 693 36.36363636 | |
| 694 36.31123919 | |
| 695 36.25899281 | |
| 696 36.35057471 | |
| 697 36.29842181 | |
| 698 36.24641834 | |
| 699 36.33762518 | |
| 700 36.28571429 | |
| 701 36.37660485 | |
| 702 36.46723647 | |
| 703 36.41536273 | |
| 704 36.36363636 | |
| 705 36.31205674 | |
| 706 36.26062323 | |
| 707 36.20933522 | |
| 708 36.15819209 | |
| 709 36.10719323 | |
| 710 36.05633803 | |
| 711 36.14627286 | |
| 712 36.09550562 | |
| 713 36.18513324 | |
| 714 36.27450980 | |
| 715 36.22377622 | |
| 716 36.17318436 | |
| 717 36.26220363 | |
| 718 36.21169916 | |
| 719 36.30041725 | |
| 720 36.25000000 | |
| 721 36.19972261 | |
| 722 36.14958449 | |
| 723 36.23789765 | |
| 724 36.32596685 | |
| 725 36.41379310 | |
| 726 36.36363636 | |
| 727 36.31361761 | |
| 728 36.40109890 | |
| 729 36.35116598 | |
| 730 36.30136986 | |
| 731 36.25170999 | |
| 732 36.20218579 | |
| 733 36.28922237 | |
| 734 36.37602180 | |
| 735 36.32653061 | |
| 736 36.27717391 | |
| 737 36.36363636 | |
| 738 36.31436314 | |
| 739 36.40054127 | |
| 740 36.35135135 | |
| 741 36.30229420 | |
| 742 36.38814016 | |
| 743 36.47375505 | |
| 744 36.55913978 | |
| 745 36.51006711 | |
| 746 36.46112601 | |
| 747 36.41231593 | |
| 748 36.36363636 | |
| 749 36.31508678 | |
| 750 36.40000000 | |
| Final result: 36.4000 +/- 1.7581 | |
| Random chance: 25.0000 +/- 1.5822 | |