| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 869 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 869 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 0.00000000 | |
| 2 0.00000000 | |
| 3 0.00000000 | |
| 4 0.00000000 | |
| 5 20.00000000 | |
| 6 16.66666667 | |
| 7 28.57142857 | |
| 8 25.00000000 | |
| 9 22.22222222 | |
| 10 30.00000000 | |
| 11 27.27272727 | |
| 12 33.33333333 | |
| 13 30.76923077 | |
| 14 28.57142857 | |
| 15 33.33333333 | |
| 16 31.25000000 | |
| 17 35.29411765 | |
| 18 38.88888889 | |
| 19 42.10526316 | |
| 20 40.00000000 | |
| 21 42.85714286 | |
| 22 45.45454545 | |
| 23 43.47826087 | |
| 24 45.83333333 | |
| 25 44.00000000 | |
| 26 42.30769231 | |
| 27 44.44444444 | |
| 28 42.85714286 | |
| 29 44.82758621 | |
| 30 46.66666667 | |
| 31 45.16129032 | |
| 32 46.87500000 | |
| 33 45.45454545 | |
| 34 44.11764706 | |
| 35 42.85714286 | |
| 36 41.66666667 | |
| 37 43.24324324 | |
| 38 42.10526316 | |
| 39 41.02564103 | |
| 40 42.50000000 | |
| 41 41.46341463 | |
| 42 40.47619048 | |
| 43 39.53488372 | |
| 44 38.63636364 | |
| 45 40.00000000 | |
| 46 41.30434783 | |
| 47 42.55319149 | |
| 48 41.66666667 | |
| 49 42.85714286 | |
| 50 44.00000000 | |
| 51 45.09803922 | |
| 52 46.15384615 | |
| 53 47.16981132 | |
| 54 46.29629630 | |
| 55 45.45454545 | |
| 56 44.64285714 | |
| 57 45.61403509 | |
| 58 46.55172414 | |
| 59 47.45762712 | |
| 60 48.33333333 | |
| 61 49.18032787 | |
| 62 48.38709677 | |
| 63 49.20634921 | |
| 64 50.00000000 | |
| 65 49.23076923 | |
| 66 50.00000000 | |
| 67 50.74626866 | |
| 68 51.47058824 | |
| 69 50.72463768 | |
| 70 50.00000000 | |
| 71 49.29577465 | |
| 72 48.61111111 | |
| 73 49.31506849 | |
| 74 48.64864865 | |
| 75 49.33333333 | |
| 76 50.00000000 | |
| 77 49.35064935 | |
| 78 50.00000000 | |
| 79 49.36708861 | |
| 80 48.75000000 | |
| 81 48.14814815 | |
| 82 48.78048780 | |
| 83 48.19277108 | |
| 84 47.61904762 | |
| 85 48.23529412 | |
| 86 47.67441860 | |
| 87 47.12643678 | |
| 88 47.72727273 | |
| 89 48.31460674 | |
| 90 48.88888889 | |
| 91 49.45054945 | |
| 92 50.00000000 | |
| 93 49.46236559 | |
| 94 48.93617021 | |
| 95 49.47368421 | |
| 96 50.00000000 | |
| 97 49.48453608 | |
| 98 48.97959184 | |
| 99 49.49494949 | |
| 100 49.00000000 | |
| 101 49.50495050 | |
| 102 50.00000000 | |
| 103 49.51456311 | |
| 104 50.00000000 | |
| 105 50.47619048 | |
| 106 50.00000000 | |
| 107 50.46728972 | |
| 108 50.00000000 | |
| 109 49.54128440 | |
| 110 50.00000000 | |
| 111 50.45045045 | |
| 112 50.89285714 | |
| 113 50.44247788 | |
| 114 50.87719298 | |
| 115 51.30434783 | |
| 116 51.72413793 | |
| 117 51.28205128 | |
| 118 51.69491525 | |
| 119 51.26050420 | |
| 120 50.83333333 | |
| 121 50.41322314 | |
| 122 50.81967213 | |
| 123 51.21951220 | |
| 124 50.80645161 | |
| 125 51.20000000 | |
| 126 51.58730159 | |
| 127 51.18110236 | |
| 128 51.56250000 | |
| 129 51.16279070 | |
| 130 50.76923077 | |
| 131 51.14503817 | |
| 132 50.75757576 | |
| 133 51.12781955 | |
| 134 50.74626866 | |
| 135 50.37037037 | |
| 136 50.00000000 | |
| 137 50.36496350 | |
| 138 50.72463768 | |
| 139 51.07913669 | |
| 140 50.71428571 | |
| 141 50.35460993 | |
| 142 50.70422535 | |
| 143 51.04895105 | |
| 144 51.38888889 | |
| 145 51.72413793 | |
| 146 51.36986301 | |
| 147 51.70068027 | |
| 148 51.35135135 | |
| 149 51.67785235 | |
| 150 52.00000000 | |
| 151 52.31788079 | |
| 152 52.63157895 | |
| 153 52.28758170 | |
| 154 51.94805195 | |
| 155 51.61290323 | |
| 156 51.92307692 | |
| 157 52.22929936 | |
| 158 51.89873418 | |
| 159 52.20125786 | |
| 160 51.87500000 | |
| 161 51.55279503 | |
| 162 51.23456790 | |
| 163 51.53374233 | |
| 164 51.21951220 | |
| 165 51.51515152 | |
| 166 51.80722892 | |
| 167 51.49700599 | |
| 168 51.78571429 | |
| 169 51.47928994 | |
| 170 51.17647059 | |
| 171 51.46198830 | |
| 172 51.74418605 | |
| 173 52.02312139 | |
| 174 52.29885057 | |
| 175 52.00000000 | |
| 176 51.70454545 | |
| 177 51.97740113 | |
| 178 52.24719101 | |
| 179 51.95530726 | |
| 180 51.66666667 | |
| 181 51.38121547 | |
| 182 51.64835165 | |
| 183 51.36612022 | |
| 184 51.08695652 | |
| 185 51.35135135 | |
| 186 51.61290323 | |
| 187 51.87165775 | |
| 188 51.59574468 | |
| 189 51.32275132 | |
| 190 51.05263158 | |
| 191 51.30890052 | |
| 192 51.04166667 | |
| 193 50.77720207 | |
| 194 51.03092784 | |
| 195 51.28205128 | |
| 196 51.53061224 | |
| 197 51.77664975 | |
| 198 51.51515152 | |
| 199 51.75879397 | |
| 200 52.00000000 | |
| 201 52.23880597 | |
| 202 51.98019802 | |
| 203 51.72413793 | |
| 204 51.96078431 | |
| 205 51.70731707 | |
| 206 51.45631068 | |
| 207 51.69082126 | |
| 208 51.44230769 | |
| 209 51.19617225 | |
| 210 51.42857143 | |
| 211 51.65876777 | |
| 212 51.41509434 | |
| 213 51.64319249 | |
| 214 51.86915888 | |
| 215 52.09302326 | |
| 216 51.85185185 | |
| 217 52.07373272 | |
| 218 51.83486239 | |
| 219 51.59817352 | |
| 220 51.36363636 | |
| 221 51.58371041 | |
| 222 51.80180180 | |
| 223 51.56950673 | |
| 224 51.78571429 | |
| 225 51.55555556 | |
| 226 51.76991150 | |
| 227 51.98237885 | |
| 228 52.19298246 | |
| 229 52.40174672 | |
| 230 52.60869565 | |
| 231 52.38095238 | |
| 232 52.15517241 | |
| 233 51.93133047 | |
| 234 51.70940171 | |
| 235 51.48936170 | |
| 236 51.27118644 | |
| 237 51.05485232 | |
| 238 51.26050420 | |
| 239 51.46443515 | |
| 240 51.66666667 | |
| 241 51.86721992 | |
| 242 51.65289256 | |
| 243 51.85185185 | |
| 244 51.63934426 | |
| 245 51.83673469 | |
| 246 52.03252033 | |
| 247 52.22672065 | |
| 248 52.41935484 | |
| 249 52.20883534 | |
| 250 52.40000000 | |
| 251 52.58964143 | |
| 252 52.77777778 | |
| 253 52.96442688 | |
| 254 52.75590551 | |
| 255 52.94117647 | |
| 256 53.12500000 | |
| 257 53.30739300 | |
| 258 53.10077519 | |
| 259 52.89575290 | |
| 260 52.69230769 | |
| 261 52.87356322 | |
| 262 52.67175573 | |
| 263 52.85171103 | |
| 264 52.65151515 | |
| 265 52.45283019 | |
| 266 52.25563910 | |
| 267 52.43445693 | |
| 268 52.61194030 | |
| 269 52.41635688 | |
| 270 52.59259259 | |
| 271 52.76752768 | |
| 272 52.57352941 | |
| 273 52.38095238 | |
| 274 52.18978102 | |
| 275 52.00000000 | |
| 276 51.81159420 | |
| 277 51.98555957 | |
| 278 51.79856115 | |
| 279 51.97132616 | |
| 280 52.14285714 | |
| 281 52.31316726 | |
| 282 52.48226950 | |
| 283 52.29681979 | |
| 284 52.46478873 | |
| 285 52.63157895 | |
| 286 52.44755245 | |
| 287 52.26480836 | |
| 288 52.43055556 | |
| 289 52.24913495 | |
| 290 52.41379310 | |
| 291 52.23367698 | |
| 292 52.39726027 | |
| 293 52.55972696 | |
| 294 52.38095238 | |
| 295 52.54237288 | |
| 296 52.70270270 | |
| 297 52.86195286 | |
| 298 53.02013423 | |
| 299 52.84280936 | |
| 300 53.00000000 | |
| 301 52.82392027 | |
| 302 52.98013245 | |
| 303 53.13531353 | |
| 304 52.96052632 | |
| 305 53.11475410 | |
| 306 52.94117647 | |
| 307 53.09446254 | |
| 308 53.24675325 | |
| 309 53.39805825 | |
| 310 53.54838710 | |
| 311 53.69774920 | |
| 312 53.52564103 | |
| 313 53.67412141 | |
| 314 53.50318471 | |
| 315 53.33333333 | |
| 316 53.48101266 | |
| 317 53.62776025 | |
| 318 53.45911950 | |
| 319 53.29153605 | |
| 320 53.12500000 | |
| 321 52.95950156 | |
| 322 53.10559006 | |
| 323 52.94117647 | |
| 324 52.77777778 | |
| 325 52.92307692 | |
| 326 53.06748466 | |
| 327 52.90519878 | |
| 328 53.04878049 | |
| 329 52.88753799 | |
| 330 52.72727273 | |
| 331 52.87009063 | |
| 332 53.01204819 | |
| 333 53.15315315 | |
| 334 53.29341317 | |
| 335 53.13432836 | |
| 336 53.27380952 | |
| 337 53.11572700 | |
| 338 52.95857988 | |
| 339 53.09734513 | |
| 340 53.23529412 | |
| 341 53.07917889 | |
| 342 52.92397661 | |
| 343 53.06122449 | |
| 344 53.19767442 | |
| 345 53.04347826 | |
| 346 53.17919075 | |
| 347 53.02593660 | |
| 348 52.87356322 | |
| 349 52.72206304 | |
| 350 52.85714286 | |
| 351 52.99145299 | |
| 352 52.84090909 | |
| 353 52.69121813 | |
| 354 52.54237288 | |
| 355 52.39436620 | |
| 356 52.52808989 | |
| 357 52.38095238 | |
| 358 52.51396648 | |
| 359 52.36768802 | |
| 360 52.50000000 | |
| 361 52.63157895 | |
| 362 52.76243094 | |
| 363 52.89256198 | |
| 364 52.74725275 | |
| 365 52.60273973 | |
| 366 52.45901639 | |
| 367 52.58855586 | |
| 368 52.44565217 | |
| 369 52.30352304 | |
| 370 52.43243243 | |
| 371 52.29110512 | |
| 372 52.15053763 | |
| 373 52.01072386 | |
| 374 52.13903743 | |
| 375 52.26666667 | |
| 376 52.12765957 | |
| 377 52.25464191 | |
| 378 52.38095238 | |
| 379 52.24274406 | |
| 380 52.36842105 | |
| 381 52.23097113 | |
| 382 52.09424084 | |
| 383 51.95822454 | |
| 384 51.82291667 | |
| 385 51.94805195 | |
| 386 51.81347150 | |
| 387 51.67958656 | |
| 388 51.54639175 | |
| 389 51.41388175 | |
| 390 51.28205128 | |
| 391 51.15089514 | |
| 392 51.27551020 | |
| 393 51.14503817 | |
| 394 51.01522843 | |
| 395 51.13924051 | |
| 396 51.01010101 | |
| 397 50.88161209 | |
| 398 50.75376884 | |
| 399 50.87719298 | |
| 400 51.00000000 | |
| 401 50.87281796 | |
| 402 50.99502488 | |
| 403 51.11662531 | |
| 404 50.99009901 | |
| 405 50.86419753 | |
| 406 50.98522167 | |
| 407 51.10565111 | |
| 408 51.22549020 | |
| 409 51.10024450 | |
| 410 51.21951220 | |
| 411 51.09489051 | |
| 412 51.21359223 | |
| 413 51.08958838 | |
| 414 51.20772947 | |
| 415 51.08433735 | |
| 416 50.96153846 | |
| 417 50.83932854 | |
| 418 50.71770335 | |
| 419 50.59665871 | |
| 420 50.47619048 | |
| 421 50.59382423 | |
| 422 50.71090047 | |
| 423 50.82742317 | |
| 424 50.70754717 | |
| 425 50.58823529 | |
| 426 50.46948357 | |
| 427 50.58548009 | |
| 428 50.46728972 | |
| 429 50.58275058 | |
| 430 50.46511628 | |
| 431 50.34802784 | |
| 432 50.46296296 | |
| 433 50.34642032 | |
| 434 50.46082949 | |
| 435 50.57471264 | |
| 436 50.68807339 | |
| 437 50.57208238 | |
| 438 50.45662100 | |
| 439 50.56947608 | |
| 440 50.45454545 | |
| 441 50.56689342 | |
| 442 50.45248869 | |
| 443 50.33860045 | |
| 444 50.22522523 | |
| 445 50.33707865 | |
| 446 50.44843049 | |
| 447 50.33557047 | |
| 448 50.22321429 | |
| 449 50.33407572 | |
| 450 50.22222222 | |
| 451 50.11086475 | |
| 452 50.22123894 | |
| 453 50.33112583 | |
| 454 50.22026432 | |
| 455 50.32967033 | |
| 456 50.21929825 | |
| 457 50.10940919 | |
| 458 50.00000000 | |
| 459 50.10893246 | |
| 460 50.00000000 | |
| 461 49.89154013 | |
| 462 50.00000000 | |
| 463 49.89200864 | |
| 464 49.78448276 | |
| 465 49.89247312 | |
| 466 49.78540773 | |
| 467 49.89293362 | |
| 468 50.00000000 | |
| 469 49.89339019 | |
| 470 49.78723404 | |
| 471 49.89384289 | |
| 472 50.00000000 | |
| 473 49.89429175 | |
| 474 50.00000000 | |
| 475 49.89473684 | |
| 476 49.78991597 | |
| 477 49.89517820 | |
| 478 50.00000000 | |
| 479 50.10438413 | |
| 480 50.20833333 | |
| 481 50.10395010 | |
| 482 50.20746888 | |
| 483 50.31055901 | |
| 484 50.41322314 | |
| 485 50.51546392 | |
| 486 50.41152263 | |
| 487 50.30800821 | |
| 488 50.40983607 | |
| 489 50.30674847 | |
| 490 50.20408163 | |
| 491 50.10183299 | |
| 492 50.00000000 | |
| 493 49.89858012 | |
| 494 50.00000000 | |
| 495 49.89898990 | |
| 496 49.79838710 | |
| 497 49.89939638 | |
| 498 50.00000000 | |
| 499 50.10020040 | |
| 500 50.00000000 | |
| 501 49.90019960 | |
| 502 50.00000000 | |
| 503 50.09940358 | |
| 504 50.00000000 | |
| 505 50.09900990 | |
| 506 50.00000000 | |
| 507 49.90138067 | |
| 508 50.00000000 | |
| 509 49.90176817 | |
| 510 49.80392157 | |
| 511 49.90215264 | |
| 512 49.80468750 | |
| 513 49.70760234 | |
| 514 49.80544747 | |
| 515 49.90291262 | |
| 516 49.80620155 | |
| 517 49.70986460 | |
| 518 49.80694981 | |
| 519 49.71098266 | |
| 520 49.61538462 | |
| 521 49.71209213 | |
| 522 49.61685824 | |
| 523 49.52198853 | |
| 524 49.42748092 | |
| 525 49.33333333 | |
| 526 49.23954373 | |
| 527 49.14611006 | |
| 528 49.05303030 | |
| 529 48.96030246 | |
| 530 49.05660377 | |
| 531 49.15254237 | |
| 532 49.24812030 | |
| 533 49.34333959 | |
| 534 49.25093633 | |
| 535 49.34579439 | |
| 536 49.44029851 | |
| 537 49.53445065 | |
| 538 49.44237918 | |
| 539 49.53617811 | |
| 540 49.44444444 | |
| 541 49.35304991 | |
| 542 49.44649446 | |
| 543 49.53959484 | |
| 544 49.63235294 | |
| 545 49.54128440 | |
| 546 49.63369963 | |
| 547 49.54296161 | |
| 548 49.63503650 | |
| 549 49.54462659 | |
| 550 49.63636364 | |
| 551 49.54627949 | |
| 552 49.63768116 | |
| 553 49.72875226 | |
| 554 49.63898917 | |
| 555 49.72972973 | |
| 556 49.64028777 | |
| 557 49.73070018 | |
| 558 49.64157706 | |
| 559 49.73166369 | |
| 560 49.82142857 | |
| 561 49.91087344 | |
| 562 50.00000000 | |
| 563 49.91119005 | |
| 564 49.82269504 | |
| 565 49.73451327 | |
| 566 49.82332155 | |
| 567 49.73544974 | |
| 568 49.82394366 | |
| 569 49.73637961 | |
| 570 49.64912281 | |
| 571 49.73730298 | |
| 572 49.65034965 | |
| 573 49.56369983 | |
| 574 49.65156794 | |
| 575 49.56521739 | |
| 576 49.65277778 | |
| 577 49.74003466 | |
| 578 49.82698962 | |
| 579 49.91364421 | |
| 580 50.00000000 | |
| 581 49.91394148 | |
| 582 50.00000000 | |
| 583 50.08576329 | |
| 584 50.17123288 | |
| 585 50.25641026 | |
| 586 50.17064846 | |
| 587 50.08517888 | |
| 588 50.17006803 | |
| 589 50.08488964 | |
| 590 50.16949153 | |
| 591 50.25380711 | |
| 592 50.33783784 | |
| 593 50.25295110 | |
| 594 50.33670034 | |
| 595 50.25210084 | |
| 596 50.16778523 | |
| 597 50.08375209 | |
| 598 50.00000000 | |
| 599 50.08347245 | |
| 600 50.16666667 | |
| 601 50.24958403 | |
| 602 50.16611296 | |
| 603 50.08291874 | |
| 604 50.00000000 | |
| 605 49.91735537 | |
| 606 50.00000000 | |
| 607 50.08237232 | |
| 608 50.16447368 | |
| 609 50.08210181 | |
| 610 50.16393443 | |
| 611 50.08183306 | |
| 612 50.16339869 | |
| 613 50.08156607 | |
| 614 50.00000000 | |
| 615 50.08130081 | |
| 616 50.00000000 | |
| 617 50.08103728 | |
| 618 50.16181230 | |
| 619 50.08077544 | |
| 620 50.16129032 | |
| 621 50.24154589 | |
| 622 50.16077170 | |
| 623 50.24077047 | |
| 624 50.16025641 | |
| 625 50.24000000 | |
| 626 50.15974441 | |
| 627 50.23923445 | |
| 628 50.31847134 | |
| 629 50.23847377 | |
| 630 50.31746032 | |
| 631 50.23771791 | |
| 632 50.15822785 | |
| 633 50.07898894 | |
| 634 50.00000000 | |
| 635 50.07874016 | |
| 636 50.15723270 | |
| 637 50.07849294 | |
| 638 50.15673981 | |
| 639 50.23474178 | |
| 640 50.15625000 | |
| 641 50.23400936 | |
| 642 50.15576324 | |
| 643 50.07776050 | |
| 644 50.15527950 | |
| 645 50.07751938 | |
| 646 50.15479876 | |
| 647 50.07727975 | |
| 648 50.00000000 | |
| 649 49.92295840 | |
| 650 50.00000000 | |
| 651 50.07680492 | |
| 652 50.00000000 | |
| 653 49.92343032 | |
| 654 49.84709480 | |
| 655 49.77099237 | |
| 656 49.69512195 | |
| 657 49.61948250 | |
| 658 49.69604863 | |
| 659 49.77238240 | |
| 660 49.84848485 | |
| 661 49.77307110 | |
| 662 49.84894260 | |
| 663 49.92458522 | |
| 664 50.00000000 | |
| 665 50.07518797 | |
| 666 50.00000000 | |
| 667 50.07496252 | |
| 668 50.00000000 | |
| 669 49.92526158 | |
| 670 50.00000000 | |
| 671 50.07451565 | |
| 672 50.00000000 | |
| 673 49.92570579 | |
| 674 49.85163205 | |
| 675 49.77777778 | |
| 676 49.70414201 | |
| 677 49.77843427 | |
| 678 49.85250737 | |
| 679 49.77908689 | |
| 680 49.85294118 | |
| 681 49.77973568 | |
| 682 49.85337243 | |
| 683 49.78038067 | |
| 684 49.70760234 | |
| 685 49.78102190 | |
| 686 49.70845481 | |
| 687 49.63609898 | |
| 688 49.70930233 | |
| 689 49.63715530 | |
| 690 49.56521739 | |
| 691 49.49348770 | |
| 692 49.56647399 | |
| 693 49.49494949 | |
| 694 49.42363112 | |
| 695 49.35251799 | |
| 696 49.28160920 | |
| 697 49.21090387 | |
| 698 49.14040115 | |
| 699 49.07010014 | |
| 700 49.14285714 | |
| 701 49.07275321 | |
| 702 49.14529915 | |
| 703 49.21763869 | |
| 704 49.28977273 | |
| 705 49.36170213 | |
| 706 49.29178470 | |
| 707 49.36350778 | |
| 708 49.43502825 | |
| 709 49.50634697 | |
| 710 49.57746479 | |
| 711 49.64838256 | |
| 712 49.71910112 | |
| 713 49.64936886 | |
| 714 49.57983193 | |
| 715 49.51048951 | |
| 716 49.44134078 | |
| 717 49.37238494 | |
| 718 49.30362117 | |
| 719 49.23504868 | |
| 720 49.30555556 | |
| 721 49.23717060 | |
| 722 49.30747922 | |
| 723 49.37759336 | |
| 724 49.44751381 | |
| 725 49.51724138 | |
| 726 49.44903581 | |
| 727 49.38101788 | |
| 728 49.31318681 | |
| 729 49.24554184 | |
| 730 49.31506849 | |
| 731 49.38440492 | |
| 732 49.31693989 | |
| 733 49.38608458 | |
| 734 49.45504087 | |
| 735 49.38775510 | |
| 736 49.45652174 | |
| 737 49.38941655 | |
| 738 49.45799458 | |
| 739 49.39106901 | |
| 740 49.32432432 | |
| 741 49.39271255 | |
| 742 49.46091644 | |
| 743 49.52893674 | |
| 744 49.46236559 | |
| 745 49.39597315 | |
| 746 49.32975871 | |
| 747 49.26372155 | |
| 748 49.19786096 | |
| 749 49.13217623 | |
| 750 49.20000000 | |
| Final result: 49.2000 +/- 1.8267 | |
| Random chance: 25.0083 +/- 1.5824 | |