| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 817 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 817 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 25.00000000 | |
| 5 20.00000000 | |
| 6 16.66666667 | |
| 7 14.28571429 | |
| 8 25.00000000 | |
| 9 22.22222222 | |
| 10 20.00000000 | |
| 11 18.18181818 | |
| 12 25.00000000 | |
| 13 30.76923077 | |
| 14 28.57142857 | |
| 15 26.66666667 | |
| 16 31.25000000 | |
| 17 29.41176471 | |
| 18 27.77777778 | |
| 19 26.31578947 | |
| 20 25.00000000 | |
| 21 23.80952381 | |
| 22 22.72727273 | |
| 23 21.73913043 | |
| 24 25.00000000 | |
| 25 24.00000000 | |
| 26 23.07692308 | |
| 27 22.22222222 | |
| 28 21.42857143 | |
| 29 24.13793103 | |
| 30 23.33333333 | |
| 31 22.58064516 | |
| 32 25.00000000 | |
| 33 27.27272727 | |
| 34 26.47058824 | |
| 35 25.71428571 | |
| 36 25.00000000 | |
| 37 27.02702703 | |
| 38 26.31578947 | |
| 39 28.20512821 | |
| 40 27.50000000 | |
| 41 26.82926829 | |
| 42 26.19047619 | |
| 43 27.90697674 | |
| 44 27.27272727 | |
| 45 28.88888889 | |
| 46 28.26086957 | |
| 47 29.78723404 | |
| 48 31.25000000 | |
| 49 30.61224490 | |
| 50 32.00000000 | |
| 51 31.37254902 | |
| 52 30.76923077 | |
| 53 30.18867925 | |
| 54 29.62962963 | |
| 55 30.90909091 | |
| 56 30.35714286 | |
| 57 29.82456140 | |
| 58 29.31034483 | |
| 59 30.50847458 | |
| 60 31.66666667 | |
| 61 31.14754098 | |
| 62 32.25806452 | |
| 63 31.74603175 | |
| 64 31.25000000 | |
| 65 30.76923077 | |
| 66 31.81818182 | |
| 67 31.34328358 | |
| 68 30.88235294 | |
| 69 31.88405797 | |
| 70 31.42857143 | |
| 71 30.98591549 | |
| 72 31.94444444 | |
| 73 31.50684932 | |
| 74 31.08108108 | |
| 75 30.66666667 | |
| 76 31.57894737 | |
| 77 31.16883117 | |
| 78 30.76923077 | |
| 79 30.37974684 | |
| 80 31.25000000 | |
| 81 30.86419753 | |
| 82 31.70731707 | |
| 83 31.32530120 | |
| 84 30.95238095 | |
| 85 30.58823529 | |
| 86 30.23255814 | |
| 87 29.88505747 | |
| 88 29.54545455 | |
| 89 30.33707865 | |
| 90 30.00000000 | |
| 91 29.67032967 | |
| 92 29.34782609 | |
| 93 29.03225806 | |
| 94 28.72340426 | |
| 95 29.47368421 | |
| 96 30.20833333 | |
| 97 30.92783505 | |
| 98 30.61224490 | |
| 99 31.31313131 | |
| 100 32.00000000 | |
| 101 32.67326733 | |
| 102 32.35294118 | |
| 103 32.03883495 | |
| 104 31.73076923 | |
| 105 31.42857143 | |
| 106 31.13207547 | |
| 107 31.77570093 | |
| 108 32.40740741 | |
| 109 32.11009174 | |
| 110 32.72727273 | |
| 111 32.43243243 | |
| 112 33.03571429 | |
| 113 33.62831858 | |
| 114 34.21052632 | |
| 115 33.91304348 | |
| 116 33.62068966 | |
| 117 33.33333333 | |
| 118 33.05084746 | |
| 119 32.77310924 | |
| 120 33.33333333 | |
| 121 33.88429752 | |
| 122 33.60655738 | |
| 123 33.33333333 | |
| 124 33.87096774 | |
| 125 34.40000000 | |
| 126 34.92063492 | |
| 127 34.64566929 | |
| 128 34.37500000 | |
| 129 34.88372093 | |
| 130 35.38461538 | |
| 131 35.11450382 | |
| 132 34.84848485 | |
| 133 34.58646617 | |
| 134 34.32835821 | |
| 135 34.81481481 | |
| 136 34.55882353 | |
| 137 35.03649635 | |
| 138 35.50724638 | |
| 139 35.97122302 | |
| 140 35.71428571 | |
| 141 35.46099291 | |
| 142 35.21126761 | |
| 143 34.96503497 | |
| 144 35.41666667 | |
| 145 35.17241379 | |
| 146 34.93150685 | |
| 147 35.37414966 | |
| 148 35.13513514 | |
| 149 34.89932886 | |
| 150 34.66666667 | |
| 151 35.09933775 | |
| 152 34.86842105 | |
| 153 34.64052288 | |
| 154 34.41558442 | |
| 155 34.83870968 | |
| 156 35.25641026 | |
| 157 35.03184713 | |
| 158 35.44303797 | |
| 159 35.22012579 | |
| 160 35.00000000 | |
| 161 34.78260870 | |
| 162 35.18518519 | |
| 163 34.96932515 | |
| 164 34.75609756 | |
| 165 35.15151515 | |
| 166 35.54216867 | |
| 167 35.32934132 | |
| 168 35.71428571 | |
| 169 35.50295858 | |
| 170 35.29411765 | |
| 171 35.08771930 | |
| 172 34.88372093 | |
| 173 34.68208092 | |
| 174 35.05747126 | |
| 175 34.85714286 | |
| 176 35.22727273 | |
| 177 35.02824859 | |
| 178 34.83146067 | |
| 179 35.19553073 | |
| 180 35.55555556 | |
| 181 35.35911602 | |
| 182 35.16483516 | |
| 183 34.97267760 | |
| 184 34.78260870 | |
| 185 34.59459459 | |
| 186 34.94623656 | |
| 187 34.75935829 | |
| 188 34.57446809 | |
| 189 34.92063492 | |
| 190 34.73684211 | |
| 191 34.55497382 | |
| 192 34.37500000 | |
| 193 34.19689119 | |
| 194 34.53608247 | |
| 195 34.35897436 | |
| 196 34.18367347 | |
| 197 34.51776650 | |
| 198 34.84848485 | |
| 199 34.67336683 | |
| 200 35.00000000 | |
| 201 34.82587065 | |
| 202 35.14851485 | |
| 203 34.97536946 | |
| 204 34.80392157 | |
| 205 34.63414634 | |
| 206 34.46601942 | |
| 207 34.78260870 | |
| 208 34.61538462 | |
| 209 34.44976077 | |
| 210 34.28571429 | |
| 211 34.12322275 | |
| 212 33.96226415 | |
| 213 33.80281690 | |
| 214 33.64485981 | |
| 215 33.48837209 | |
| 216 33.79629630 | |
| 217 34.10138249 | |
| 218 34.40366972 | |
| 219 34.24657534 | |
| 220 34.09090909 | |
| 221 33.93665158 | |
| 222 33.78378378 | |
| 223 33.63228700 | |
| 224 33.92857143 | |
| 225 33.77777778 | |
| 226 34.07079646 | |
| 227 34.36123348 | |
| 228 34.21052632 | |
| 229 34.06113537 | |
| 230 33.91304348 | |
| 231 33.76623377 | |
| 232 33.62068966 | |
| 233 33.47639485 | |
| 234 33.76068376 | |
| 235 33.61702128 | |
| 236 33.89830508 | |
| 237 33.75527426 | |
| 238 34.03361345 | |
| 239 33.89121339 | |
| 240 33.75000000 | |
| 241 33.60995851 | |
| 242 33.47107438 | |
| 243 33.74485597 | |
| 244 33.60655738 | |
| 245 33.46938776 | |
| 246 33.33333333 | |
| 247 33.60323887 | |
| 248 33.46774194 | |
| 249 33.73493976 | |
| 250 33.60000000 | |
| 251 33.86454183 | |
| 252 34.12698413 | |
| 253 33.99209486 | |
| 254 33.85826772 | |
| 255 33.72549020 | |
| 256 33.59375000 | |
| 257 33.46303502 | |
| 258 33.33333333 | |
| 259 33.20463320 | |
| 260 33.07692308 | |
| 261 33.33333333 | |
| 262 33.20610687 | |
| 263 33.07984791 | |
| 264 33.33333333 | |
| 265 33.58490566 | |
| 266 33.45864662 | |
| 267 33.33333333 | |
| 268 33.20895522 | |
| 269 33.08550186 | |
| 270 32.96296296 | |
| 271 32.84132841 | |
| 272 32.72058824 | |
| 273 32.96703297 | |
| 274 32.84671533 | |
| 275 32.72727273 | |
| 276 32.60869565 | |
| 277 32.49097473 | |
| 278 32.37410072 | |
| 279 32.25806452 | |
| 280 32.50000000 | |
| 281 32.74021352 | |
| 282 32.62411348 | |
| 283 32.86219081 | |
| 284 32.74647887 | |
| 285 32.63157895 | |
| 286 32.86713287 | |
| 287 33.10104530 | |
| 288 32.98611111 | |
| 289 32.87197232 | |
| 290 32.75862069 | |
| 291 32.98969072 | |
| 292 32.87671233 | |
| 293 33.10580205 | |
| 294 32.99319728 | |
| 295 33.22033898 | |
| 296 33.44594595 | |
| 297 33.67003367 | |
| 298 33.55704698 | |
| 299 33.44481605 | |
| 300 33.33333333 | |
| 301 33.22259136 | |
| 302 33.11258278 | |
| 303 33.00330033 | |
| 304 32.89473684 | |
| 305 32.78688525 | |
| 306 32.67973856 | |
| 307 32.57328990 | |
| 308 32.79220779 | |
| 309 32.68608414 | |
| 310 32.58064516 | |
| 311 32.47588424 | |
| 312 32.69230769 | |
| 313 32.58785942 | |
| 314 32.48407643 | |
| 315 32.38095238 | |
| 316 32.27848101 | |
| 317 32.17665615 | |
| 318 32.07547170 | |
| 319 31.97492163 | |
| 320 31.87500000 | |
| 321 31.77570093 | |
| 322 31.98757764 | |
| 323 31.88854489 | |
| 324 31.79012346 | |
| 325 31.69230769 | |
| 326 31.59509202 | |
| 327 31.49847095 | |
| 328 31.70731707 | |
| 329 31.61094225 | |
| 330 31.81818182 | |
| 331 31.72205438 | |
| 332 31.92771084 | |
| 333 31.83183183 | |
| 334 31.73652695 | |
| 335 31.94029851 | |
| 336 32.14285714 | |
| 337 32.04747774 | |
| 338 31.95266272 | |
| 339 32.15339233 | |
| 340 32.05882353 | |
| 341 32.25806452 | |
| 342 32.45614035 | |
| 343 32.36151603 | |
| 344 32.55813953 | |
| 345 32.75362319 | |
| 346 32.65895954 | |
| 347 32.85302594 | |
| 348 32.75862069 | |
| 349 32.66475645 | |
| 350 32.57142857 | |
| 351 32.47863248 | |
| 352 32.38636364 | |
| 353 32.29461756 | |
| 354 32.20338983 | |
| 355 32.39436620 | |
| 356 32.30337079 | |
| 357 32.21288515 | |
| 358 32.12290503 | |
| 359 32.03342618 | |
| 360 31.94444444 | |
| 361 32.13296399 | |
| 362 32.04419890 | |
| 363 32.23140496 | |
| 364 32.14285714 | |
| 365 32.05479452 | |
| 366 31.96721311 | |
| 367 32.15258856 | |
| 368 32.06521739 | |
| 369 31.97831978 | |
| 370 31.89189189 | |
| 371 31.80592992 | |
| 372 31.98924731 | |
| 373 31.90348525 | |
| 374 32.08556150 | |
| 375 32.00000000 | |
| 376 31.91489362 | |
| 377 32.09549072 | |
| 378 32.01058201 | |
| 379 31.92612137 | |
| 380 31.84210526 | |
| 381 31.75853018 | |
| 382 31.67539267 | |
| 383 31.59268930 | |
| 384 31.51041667 | |
| 385 31.42857143 | |
| 386 31.34715026 | |
| 387 31.26614987 | |
| 388 31.44329897 | |
| 389 31.36246787 | |
| 390 31.28205128 | |
| 391 31.45780051 | |
| 392 31.37755102 | |
| 393 31.55216285 | |
| 394 31.72588832 | |
| 395 31.64556962 | |
| 396 31.81818182 | |
| 397 31.98992443 | |
| 398 31.90954774 | |
| 399 31.82957393 | |
| 400 31.75000000 | |
| 401 31.67082294 | |
| 402 31.84079602 | |
| 403 31.76178660 | |
| 404 31.68316832 | |
| 405 31.60493827 | |
| 406 31.52709360 | |
| 407 31.44963145 | |
| 408 31.37254902 | |
| 409 31.29584352 | |
| 410 31.21951220 | |
| 411 31.14355231 | |
| 412 31.31067961 | |
| 413 31.47699758 | |
| 414 31.64251208 | |
| 415 31.80722892 | |
| 416 31.73076923 | |
| 417 31.65467626 | |
| 418 31.57894737 | |
| 419 31.50357995 | |
| 420 31.66666667 | |
| 421 31.82897862 | |
| 422 31.75355450 | |
| 423 31.91489362 | |
| 424 31.83962264 | |
| 425 31.76470588 | |
| 426 31.69014085 | |
| 427 31.61592506 | |
| 428 31.54205607 | |
| 429 31.46853147 | |
| 430 31.39534884 | |
| 431 31.32250580 | |
| 432 31.48148148 | |
| 433 31.63972286 | |
| 434 31.79723502 | |
| 435 31.72413793 | |
| 436 31.65137615 | |
| 437 31.57894737 | |
| 438 31.73515982 | |
| 439 31.89066059 | |
| 440 32.04545455 | |
| 441 31.97278912 | |
| 442 31.90045249 | |
| 443 31.82844244 | |
| 444 31.98198198 | |
| 445 31.91011236 | |
| 446 31.83856502 | |
| 447 31.76733781 | |
| 448 31.69642857 | |
| 449 31.84855234 | |
| 450 31.77777778 | |
| 451 31.70731707 | |
| 452 31.63716814 | |
| 453 31.78807947 | |
| 454 31.71806167 | |
| 455 31.64835165 | |
| 456 31.57894737 | |
| 457 31.50984683 | |
| 458 31.44104803 | |
| 459 31.37254902 | |
| 460 31.30434783 | |
| 461 31.23644252 | |
| 462 31.38528139 | |
| 463 31.53347732 | |
| 464 31.68103448 | |
| 465 31.82795699 | |
| 466 31.75965665 | |
| 467 31.69164882 | |
| 468 31.83760684 | |
| 469 31.76972281 | |
| 470 31.70212766 | |
| 471 31.84713376 | |
| 472 31.99152542 | |
| 473 31.92389006 | |
| 474 31.85654008 | |
| 475 32.00000000 | |
| 476 31.93277311 | |
| 477 31.86582809 | |
| 478 31.79916318 | |
| 479 31.73277662 | |
| 480 31.66666667 | |
| 481 31.60083160 | |
| 482 31.74273859 | |
| 483 31.67701863 | |
| 484 31.81818182 | |
| 485 31.95876289 | |
| 486 31.89300412 | |
| 487 31.82751540 | |
| 488 31.76229508 | |
| 489 31.90184049 | |
| 490 32.04081633 | |
| 491 32.17922607 | |
| 492 32.31707317 | |
| 493 32.25152130 | |
| 494 32.38866397 | |
| 495 32.52525253 | |
| 496 32.45967742 | |
| 497 32.39436620 | |
| 498 32.32931727 | |
| 499 32.26452906 | |
| 500 32.20000000 | |
| 501 32.13572854 | |
| 502 32.07171315 | |
| 503 32.00795229 | |
| 504 32.14285714 | |
| 505 32.07920792 | |
| 506 32.01581028 | |
| 507 32.14990138 | |
| 508 32.28346457 | |
| 509 32.41650295 | |
| 510 32.54901961 | |
| 511 32.48532290 | |
| 512 32.42187500 | |
| 513 32.35867446 | |
| 514 32.29571984 | |
| 515 32.42718447 | |
| 516 32.36434109 | |
| 517 32.49516441 | |
| 518 32.43243243 | |
| 519 32.36994220 | |
| 520 32.30769231 | |
| 521 32.24568138 | |
| 522 32.18390805 | |
| 523 32.12237094 | |
| 524 32.06106870 | |
| 525 32.00000000 | |
| 526 31.93916350 | |
| 527 32.06831120 | |
| 528 32.00757576 | |
| 529 32.13610586 | |
| 530 32.07547170 | |
| 531 32.01506591 | |
| 532 31.95488722 | |
| 533 32.08255159 | |
| 534 32.02247191 | |
| 535 31.96261682 | |
| 536 31.90298507 | |
| 537 32.02979516 | |
| 538 31.97026022 | |
| 539 31.91094620 | |
| 540 32.03703704 | |
| 541 31.97781885 | |
| 542 31.91881919 | |
| 543 31.86003683 | |
| 544 31.80147059 | |
| 545 31.74311927 | |
| 546 31.68498168 | |
| 547 31.80987203 | |
| 548 31.93430657 | |
| 549 31.87613843 | |
| 550 31.81818182 | |
| 551 31.76043557 | |
| 552 31.88405797 | |
| 553 32.00723327 | |
| 554 31.94945848 | |
| 555 31.89189189 | |
| 556 31.83453237 | |
| 557 31.77737882 | |
| 558 31.72043011 | |
| 559 31.66368515 | |
| 560 31.78571429 | |
| 561 31.72905526 | |
| 562 31.67259786 | |
| 563 31.61634103 | |
| 564 31.73758865 | |
| 565 31.85840708 | |
| 566 31.97879859 | |
| 567 31.92239859 | |
| 568 32.04225352 | |
| 569 32.16168717 | |
| 570 32.10526316 | |
| 571 32.22416813 | |
| 572 32.16783217 | |
| 573 32.11169284 | |
| 574 32.22996516 | |
| 575 32.34782609 | |
| 576 32.46527778 | |
| 577 32.58232236 | |
| 578 32.69896194 | |
| 579 32.81519862 | |
| 580 32.93103448 | |
| 581 32.87435456 | |
| 582 32.98969072 | |
| 583 32.93310463 | |
| 584 33.04794521 | |
| 585 32.99145299 | |
| 586 33.10580205 | |
| 587 33.04940375 | |
| 588 32.99319728 | |
| 589 32.93718166 | |
| 590 33.05084746 | |
| 591 32.99492386 | |
| 592 32.93918919 | |
| 593 32.88364250 | |
| 594 32.99663300 | |
| 595 32.94117647 | |
| 596 32.88590604 | |
| 597 32.83082077 | |
| 598 32.77591973 | |
| 599 32.72120200 | |
| 600 32.66666667 | |
| 601 32.77870216 | |
| 602 32.89036545 | |
| 603 33.00165837 | |
| 604 32.94701987 | |
| 605 32.89256198 | |
| 606 33.00330033 | |
| 607 32.94892916 | |
| 608 32.89473684 | |
| 609 32.84072250 | |
| 610 32.78688525 | |
| 611 32.73322422 | |
| 612 32.84313725 | |
| 613 32.78955954 | |
| 614 32.73615635 | |
| 615 32.68292683 | |
| 616 32.79220779 | |
| 617 32.90113452 | |
| 618 32.84789644 | |
| 619 32.95638126 | |
| 620 32.90322581 | |
| 621 32.85024155 | |
| 622 32.95819936 | |
| 623 32.90529695 | |
| 624 32.85256410 | |
| 625 32.80000000 | |
| 626 32.74760383 | |
| 627 32.69537480 | |
| 628 32.80254777 | |
| 629 32.75039746 | |
| 630 32.69841270 | |
| 631 32.80507132 | |
| 632 32.75316456 | |
| 633 32.70142180 | |
| 634 32.80757098 | |
| 635 32.75590551 | |
| 636 32.70440252 | |
| 637 32.65306122 | |
| 638 32.60188088 | |
| 639 32.55086072 | |
| 640 32.65625000 | |
| 641 32.60530421 | |
| 642 32.71028037 | |
| 643 32.65940902 | |
| 644 32.60869565 | |
| 645 32.71317829 | |
| 646 32.66253870 | |
| 647 32.61205564 | |
| 648 32.71604938 | |
| 649 32.66563945 | |
| 650 32.76923077 | |
| 651 32.87250384 | |
| 652 32.97546012 | |
| 653 33.07810107 | |
| 654 33.02752294 | |
| 655 32.97709924 | |
| 656 33.07926829 | |
| 657 33.18112633 | |
| 658 33.13069909 | |
| 659 33.23216995 | |
| 660 33.18181818 | |
| 661 33.13161876 | |
| 662 33.08157100 | |
| 663 33.03167421 | |
| 664 32.98192771 | |
| 665 32.93233083 | |
| 666 33.03303303 | |
| 667 32.98350825 | |
| 668 32.93413174 | |
| 669 32.88490284 | |
| 670 32.98507463 | |
| 671 32.93591654 | |
| 672 32.88690476 | |
| 673 32.83803863 | |
| 674 32.78931751 | |
| 675 32.88888889 | |
| 676 32.98816568 | |
| 677 32.93943870 | |
| 678 32.89085546 | |
| 679 32.84241532 | |
| 680 32.79411765 | |
| 681 32.74596182 | |
| 682 32.84457478 | |
| 683 32.79648609 | |
| 684 32.89473684 | |
| 685 32.84671533 | |
| 686 32.79883382 | |
| 687 32.75109170 | |
| 688 32.70348837 | |
| 689 32.80116110 | |
| 690 32.75362319 | |
| 691 32.85094067 | |
| 692 32.94797688 | |
| 693 33.04473304 | |
| 694 32.99711816 | |
| 695 32.94964029 | |
| 696 33.04597701 | |
| 697 32.99856528 | |
| 698 32.95128940 | |
| 699 32.90414878 | |
| 700 33.00000000 | |
| 701 32.95292439 | |
| 702 32.90598291 | |
| 703 32.85917496 | |
| 704 32.81250000 | |
| 705 32.76595745 | |
| 706 32.86118980 | |
| 707 32.95615276 | |
| 708 32.90960452 | |
| 709 32.86318759 | |
| 710 32.95774648 | |
| 711 32.91139241 | |
| 712 33.00561798 | |
| 713 32.95932679 | |
| 714 32.91316527 | |
| 715 33.00699301 | |
| 716 32.96089385 | |
| 717 33.05439331 | |
| 718 33.14763231 | |
| 719 33.24061196 | |
| 720 33.19444444 | |
| 721 33.14840499 | |
| 722 33.10249307 | |
| 723 33.05670816 | |
| 724 33.14917127 | |
| 725 33.10344828 | |
| 726 33.19559229 | |
| 727 33.14993122 | |
| 728 33.10439560 | |
| 729 33.19615912 | |
| 730 33.15068493 | |
| 731 33.10533516 | |
| 732 33.19672131 | |
| 733 33.15143247 | |
| 734 33.10626703 | |
| 735 33.06122449 | |
| 736 33.01630435 | |
| 737 33.10719132 | |
| 738 33.06233062 | |
| 739 33.01759134 | |
| 740 32.97297297 | |
| 741 33.06342780 | |
| 742 33.01886792 | |
| 743 32.97442799 | |
| 744 33.06451613 | |
| 745 33.02013423 | |
| 746 33.10991957 | |
| 747 33.19946452 | |
| 748 33.15508021 | |
| 749 33.24432577 | |
| 750 33.33333333 | |
| Final result: 33.3333 +/- 1.7225 | |
| Random chance: 19.8992 +/- 1.4588 | |