| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 1548 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 1548 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 50.00000000 | |
| 5 40.00000000 | |
| 6 33.33333333 | |
| 7 42.85714286 | |
| 8 50.00000000 | |
| 9 44.44444444 | |
| 10 50.00000000 | |
| 11 45.45454545 | |
| 12 41.66666667 | |
| 13 38.46153846 | |
| 14 35.71428571 | |
| 15 33.33333333 | |
| 16 37.50000000 | |
| 17 35.29411765 | |
| 18 33.33333333 | |
| 19 36.84210526 | |
| 20 40.00000000 | |
| 21 38.09523810 | |
| 22 36.36363636 | |
| 23 34.78260870 | |
| 24 33.33333333 | |
| 25 36.00000000 | |
| 26 34.61538462 | |
| 27 37.03703704 | |
| 28 35.71428571 | |
| 29 34.48275862 | |
| 30 36.66666667 | |
| 31 38.70967742 | |
| 32 40.62500000 | |
| 33 39.39393939 | |
| 34 41.17647059 | |
| 35 42.85714286 | |
| 36 41.66666667 | |
| 37 40.54054054 | |
| 38 39.47368421 | |
| 39 38.46153846 | |
| 40 40.00000000 | |
| 41 41.46341463 | |
| 42 40.47619048 | |
| 43 41.86046512 | |
| 44 40.90909091 | |
| 45 42.22222222 | |
| 46 41.30434783 | |
| 47 40.42553191 | |
| 48 41.66666667 | |
| 49 40.81632653 | |
| 50 42.00000000 | |
| 51 41.17647059 | |
| 52 40.38461538 | |
| 53 39.62264151 | |
| 54 38.88888889 | |
| 55 40.00000000 | |
| 56 41.07142857 | |
| 57 42.10526316 | |
| 58 43.10344828 | |
| 59 42.37288136 | |
| 60 41.66666667 | |
| 61 42.62295082 | |
| 62 43.54838710 | |
| 63 42.85714286 | |
| 64 42.18750000 | |
| 65 41.53846154 | |
| 66 42.42424242 | |
| 67 43.28358209 | |
| 68 42.64705882 | |
| 69 42.02898551 | |
| 70 42.85714286 | |
| 71 43.66197183 | |
| 72 43.05555556 | |
| 73 42.46575342 | |
| 74 41.89189189 | |
| 75 42.66666667 | |
| 76 43.42105263 | |
| 77 44.15584416 | |
| 78 44.87179487 | |
| 79 44.30379747 | |
| 80 43.75000000 | |
| 81 44.44444444 | |
| 82 43.90243902 | |
| 83 44.57831325 | |
| 84 44.04761905 | |
| 85 43.52941176 | |
| 86 43.02325581 | |
| 87 43.67816092 | |
| 88 44.31818182 | |
| 89 43.82022472 | |
| 90 43.33333333 | |
| 91 42.85714286 | |
| 92 42.39130435 | |
| 93 41.93548387 | |
| 94 41.48936170 | |
| 95 42.10526316 | |
| 96 42.70833333 | |
| 97 42.26804124 | |
| 98 41.83673469 | |
| 99 41.41414141 | |
| 100 41.00000000 | |
| 101 41.58415842 | |
| 102 41.17647059 | |
| 103 40.77669903 | |
| 104 41.34615385 | |
| 105 41.90476190 | |
| 106 41.50943396 | |
| 107 41.12149533 | |
| 108 40.74074074 | |
| 109 40.36697248 | |
| 110 40.00000000 | |
| 111 39.63963964 | |
| 112 39.28571429 | |
| 113 39.82300885 | |
| 114 39.47368421 | |
| 115 40.00000000 | |
| 116 39.65517241 | |
| 117 40.17094017 | |
| 118 39.83050847 | |
| 119 40.33613445 | |
| 120 40.00000000 | |
| 121 40.49586777 | |
| 122 40.16393443 | |
| 123 39.83739837 | |
| 124 39.51612903 | |
| 125 40.00000000 | |
| 126 39.68253968 | |
| 127 39.37007874 | |
| 128 39.06250000 | |
| 129 38.75968992 | |
| 130 38.46153846 | |
| 131 38.16793893 | |
| 132 38.63636364 | |
| 133 38.34586466 | |
| 134 38.05970149 | |
| 135 37.77777778 | |
| 136 37.50000000 | |
| 137 37.22627737 | |
| 138 37.68115942 | |
| 139 37.41007194 | |
| 140 37.14285714 | |
| 141 37.58865248 | |
| 142 38.02816901 | |
| 143 38.46153846 | |
| 144 38.19444444 | |
| 145 38.62068966 | |
| 146 38.35616438 | |
| 147 38.09523810 | |
| 148 38.51351351 | |
| 149 38.25503356 | |
| 150 38.00000000 | |
| 151 37.74834437 | |
| 152 37.50000000 | |
| 153 37.25490196 | |
| 154 37.01298701 | |
| 155 36.77419355 | |
| 156 36.53846154 | |
| 157 36.94267516 | |
| 158 36.70886076 | |
| 159 36.47798742 | |
| 160 36.87500000 | |
| 161 36.64596273 | |
| 162 36.41975309 | |
| 163 36.80981595 | |
| 164 37.19512195 | |
| 165 37.57575758 | |
| 166 37.34939759 | |
| 167 37.72455090 | |
| 168 37.50000000 | |
| 169 37.86982249 | |
| 170 37.64705882 | |
| 171 38.01169591 | |
| 172 37.79069767 | |
| 173 37.57225434 | |
| 174 37.35632184 | |
| 175 37.14285714 | |
| 176 36.93181818 | |
| 177 36.72316384 | |
| 178 36.51685393 | |
| 179 36.87150838 | |
| 180 36.66666667 | |
| 181 36.46408840 | |
| 182 36.26373626 | |
| 183 36.61202186 | |
| 184 36.41304348 | |
| 185 36.75675676 | |
| 186 37.09677419 | |
| 187 36.89839572 | |
| 188 37.23404255 | |
| 189 37.03703704 | |
| 190 36.84210526 | |
| 191 36.64921466 | |
| 192 36.45833333 | |
| 193 36.26943005 | |
| 194 36.59793814 | |
| 195 36.92307692 | |
| 196 36.73469388 | |
| 197 36.54822335 | |
| 198 36.36363636 | |
| 199 36.18090452 | |
| 200 36.00000000 | |
| 201 35.82089552 | |
| 202 35.64356436 | |
| 203 35.46798030 | |
| 204 35.29411765 | |
| 205 35.60975610 | |
| 206 35.43689320 | |
| 207 35.74879227 | |
| 208 35.57692308 | |
| 209 35.40669856 | |
| 210 35.23809524 | |
| 211 35.54502370 | |
| 212 35.37735849 | |
| 213 35.21126761 | |
| 214 35.04672897 | |
| 215 35.34883721 | |
| 216 35.18518519 | |
| 217 35.02304147 | |
| 218 34.86238532 | |
| 219 34.70319635 | |
| 220 34.54545455 | |
| 221 34.38914027 | |
| 222 34.23423423 | |
| 223 34.52914798 | |
| 224 34.82142857 | |
| 225 35.11111111 | |
| 226 34.95575221 | |
| 227 34.80176211 | |
| 228 35.08771930 | |
| 229 35.37117904 | |
| 230 35.21739130 | |
| 231 35.06493506 | |
| 232 35.34482759 | |
| 233 35.62231760 | |
| 234 35.89743590 | |
| 235 35.74468085 | |
| 236 35.59322034 | |
| 237 35.44303797 | |
| 238 35.29411765 | |
| 239 35.14644351 | |
| 240 35.00000000 | |
| 241 34.85477178 | |
| 242 35.12396694 | |
| 243 35.39094650 | |
| 244 35.24590164 | |
| 245 35.51020408 | |
| 246 35.36585366 | |
| 247 35.62753036 | |
| 248 35.48387097 | |
| 249 35.34136546 | |
| 250 35.20000000 | |
| 251 35.45816733 | |
| 252 35.31746032 | |
| 253 35.17786561 | |
| 254 35.03937008 | |
| 255 34.90196078 | |
| 256 35.15625000 | |
| 257 35.01945525 | |
| 258 34.88372093 | |
| 259 34.74903475 | |
| 260 35.00000000 | |
| 261 34.86590038 | |
| 262 34.73282443 | |
| 263 34.98098859 | |
| 264 34.84848485 | |
| 265 34.71698113 | |
| 266 34.58646617 | |
| 267 34.45692884 | |
| 268 34.32835821 | |
| 269 34.57249071 | |
| 270 34.44444444 | |
| 271 34.31734317 | |
| 272 34.55882353 | |
| 273 34.43223443 | |
| 274 34.30656934 | |
| 275 34.18181818 | |
| 276 34.42028986 | |
| 277 34.65703971 | |
| 278 34.53237410 | |
| 279 34.40860215 | |
| 280 34.64285714 | |
| 281 34.51957295 | |
| 282 34.39716312 | |
| 283 34.27561837 | |
| 284 34.15492958 | |
| 285 34.38596491 | |
| 286 34.26573427 | |
| 287 34.14634146 | |
| 288 34.02777778 | |
| 289 33.91003460 | |
| 290 34.13793103 | |
| 291 34.36426117 | |
| 292 34.58904110 | |
| 293 34.47098976 | |
| 294 34.35374150 | |
| 295 34.23728814 | |
| 296 34.12162162 | |
| 297 34.00673401 | |
| 298 33.89261745 | |
| 299 33.77926421 | |
| 300 34.00000000 | |
| 301 34.21926910 | |
| 302 34.43708609 | |
| 303 34.32343234 | |
| 304 34.53947368 | |
| 305 34.42622951 | |
| 306 34.31372549 | |
| 307 34.52768730 | |
| 308 34.41558442 | |
| 309 34.30420712 | |
| 310 34.19354839 | |
| 311 34.40514469 | |
| 312 34.61538462 | |
| 313 34.82428115 | |
| 314 34.71337580 | |
| 315 34.60317460 | |
| 316 34.49367089 | |
| 317 34.70031546 | |
| 318 34.59119497 | |
| 319 34.79623824 | |
| 320 34.68750000 | |
| 321 34.57943925 | |
| 322 34.47204969 | |
| 323 34.36532508 | |
| 324 34.25925926 | |
| 325 34.46153846 | |
| 326 34.35582822 | |
| 327 34.55657492 | |
| 328 34.45121951 | |
| 329 34.34650456 | |
| 330 34.24242424 | |
| 331 34.13897281 | |
| 332 34.33734940 | |
| 333 34.53453453 | |
| 334 34.43113772 | |
| 335 34.62686567 | |
| 336 34.52380952 | |
| 337 34.42136499 | |
| 338 34.31952663 | |
| 339 34.21828909 | |
| 340 34.41176471 | |
| 341 34.60410557 | |
| 342 34.79532164 | |
| 343 34.69387755 | |
| 344 34.59302326 | |
| 345 34.49275362 | |
| 346 34.68208092 | |
| 347 34.87031700 | |
| 348 34.77011494 | |
| 349 34.95702006 | |
| 350 34.85714286 | |
| 351 34.75783476 | |
| 352 34.65909091 | |
| 353 34.84419263 | |
| 354 34.74576271 | |
| 355 34.64788732 | |
| 356 34.83146067 | |
| 357 35.01400560 | |
| 358 34.91620112 | |
| 359 34.81894150 | |
| 360 35.00000000 | |
| 361 35.18005540 | |
| 362 35.35911602 | |
| 363 35.53719008 | |
| 364 35.43956044 | |
| 365 35.34246575 | |
| 366 35.51912568 | |
| 367 35.42234332 | |
| 368 35.32608696 | |
| 369 35.23035230 | |
| 370 35.40540541 | |
| 371 35.30997305 | |
| 372 35.21505376 | |
| 373 35.12064343 | |
| 374 35.29411765 | |
| 375 35.46666667 | |
| 376 35.37234043 | |
| 377 35.54376658 | |
| 378 35.71428571 | |
| 379 35.62005277 | |
| 380 35.52631579 | |
| 381 35.43307087 | |
| 382 35.34031414 | |
| 383 35.50913838 | |
| 384 35.41666667 | |
| 385 35.32467532 | |
| 386 35.23316062 | |
| 387 35.14211886 | |
| 388 35.05154639 | |
| 389 35.21850900 | |
| 390 35.38461538 | |
| 391 35.54987212 | |
| 392 35.71428571 | |
| 393 35.62340967 | |
| 394 35.53299492 | |
| 395 35.69620253 | |
| 396 35.85858586 | |
| 397 35.76826196 | |
| 398 35.67839196 | |
| 399 35.83959900 | |
| 400 35.75000000 | |
| 401 35.91022444 | |
| 402 36.06965174 | |
| 403 35.98014888 | |
| 404 35.89108911 | |
| 405 36.04938272 | |
| 406 35.96059113 | |
| 407 35.87223587 | |
| 408 35.78431373 | |
| 409 35.94132029 | |
| 410 35.85365854 | |
| 411 35.76642336 | |
| 412 35.67961165 | |
| 413 35.59322034 | |
| 414 35.50724638 | |
| 415 35.66265060 | |
| 416 35.81730769 | |
| 417 35.73141487 | |
| 418 35.88516746 | |
| 419 36.03818616 | |
| 420 35.95238095 | |
| 421 35.86698337 | |
| 422 35.78199052 | |
| 423 35.69739953 | |
| 424 35.84905660 | |
| 425 35.76470588 | |
| 426 35.68075117 | |
| 427 35.59718970 | |
| 428 35.51401869 | |
| 429 35.66433566 | |
| 430 35.58139535 | |
| 431 35.73085847 | |
| 432 35.87962963 | |
| 433 35.79676674 | |
| 434 35.71428571 | |
| 435 35.63218391 | |
| 436 35.55045872 | |
| 437 35.69794050 | |
| 438 35.84474886 | |
| 439 35.99088838 | |
| 440 35.90909091 | |
| 441 36.05442177 | |
| 442 35.97285068 | |
| 443 35.89164786 | |
| 444 35.81081081 | |
| 445 35.73033708 | |
| 446 35.65022422 | |
| 447 35.57046980 | |
| 448 35.49107143 | |
| 449 35.41202673 | |
| 450 35.33333333 | |
| 451 35.25498891 | |
| 452 35.17699115 | |
| 453 35.32008830 | |
| 454 35.24229075 | |
| 455 35.16483516 | |
| 456 35.30701754 | |
| 457 35.22975930 | |
| 458 35.15283843 | |
| 459 35.07625272 | |
| 460 35.21739130 | |
| 461 35.14099783 | |
| 462 35.28138528 | |
| 463 35.42116631 | |
| 464 35.56034483 | |
| 465 35.48387097 | |
| 466 35.62231760 | |
| 467 35.76017131 | |
| 468 35.89743590 | |
| 469 35.82089552 | |
| 470 35.74468085 | |
| 471 35.66878981 | |
| 472 35.59322034 | |
| 473 35.72938689 | |
| 474 35.65400844 | |
| 475 35.57894737 | |
| 476 35.71428571 | |
| 477 35.63941300 | |
| 478 35.77405858 | |
| 479 35.69937370 | |
| 480 35.83333333 | |
| 481 35.75883576 | |
| 482 35.68464730 | |
| 483 35.61076605 | |
| 484 35.53719008 | |
| 485 35.46391753 | |
| 486 35.39094650 | |
| 487 35.52361396 | |
| 488 35.45081967 | |
| 489 35.37832311 | |
| 490 35.51020408 | |
| 491 35.43788187 | |
| 492 35.36585366 | |
| 493 35.29411765 | |
| 494 35.42510121 | |
| 495 35.55555556 | |
| 496 35.68548387 | |
| 497 35.61368209 | |
| 498 35.54216867 | |
| 499 35.67134269 | |
| 500 35.80000000 | |
| 501 35.72854291 | |
| 502 35.65737052 | |
| 503 35.58648111 | |
| 504 35.51587302 | |
| 505 35.44554455 | |
| 506 35.57312253 | |
| 507 35.50295858 | |
| 508 35.62992126 | |
| 509 35.55992141 | |
| 510 35.49019608 | |
| 511 35.42074364 | |
| 512 35.35156250 | |
| 513 35.28265107 | |
| 514 35.21400778 | |
| 515 35.33980583 | |
| 516 35.46511628 | |
| 517 35.39651838 | |
| 518 35.52123552 | |
| 519 35.45279383 | |
| 520 35.38461538 | |
| 521 35.31669866 | |
| 522 35.44061303 | |
| 523 35.37284895 | |
| 524 35.49618321 | |
| 525 35.42857143 | |
| 526 35.36121673 | |
| 527 35.29411765 | |
| 528 35.22727273 | |
| 529 35.34971645 | |
| 530 35.47169811 | |
| 531 35.40489642 | |
| 532 35.52631579 | |
| 533 35.64727955 | |
| 534 35.76779026 | |
| 535 35.88785047 | |
| 536 35.82089552 | |
| 537 35.94040968 | |
| 538 36.05947955 | |
| 539 35.99257885 | |
| 540 35.92592593 | |
| 541 35.85951941 | |
| 542 35.97785978 | |
| 543 35.91160221 | |
| 544 35.84558824 | |
| 545 35.77981651 | |
| 546 35.71428571 | |
| 547 35.64899452 | |
| 548 35.76642336 | |
| 549 35.70127505 | |
| 550 35.63636364 | |
| 551 35.75317604 | |
| 552 35.68840580 | |
| 553 35.62386980 | |
| 554 35.55956679 | |
| 555 35.49549550 | |
| 556 35.61151079 | |
| 557 35.72710952 | |
| 558 35.84229391 | |
| 559 35.95706619 | |
| 560 35.89285714 | |
| 561 35.82887701 | |
| 562 35.94306050 | |
| 563 36.05683837 | |
| 564 35.99290780 | |
| 565 36.10619469 | |
| 566 36.21908127 | |
| 567 36.15520282 | |
| 568 36.09154930 | |
| 569 36.02811951 | |
| 570 36.14035088 | |
| 571 36.25218914 | |
| 572 36.18881119 | |
| 573 36.12565445 | |
| 574 36.06271777 | |
| 575 36.00000000 | |
| 576 36.11111111 | |
| 577 36.22183709 | |
| 578 36.15916955 | |
| 579 36.09671848 | |
| 580 36.20689655 | |
| 581 36.14457831 | |
| 582 36.08247423 | |
| 583 36.19210978 | |
| 584 36.30136986 | |
| 585 36.41025641 | |
| 586 36.34812287 | |
| 587 36.28620102 | |
| 588 36.22448980 | |
| 589 36.16298812 | |
| 590 36.10169492 | |
| 591 36.04060914 | |
| 592 35.97972973 | |
| 593 35.91905565 | |
| 594 36.02693603 | |
| 595 35.96638655 | |
| 596 36.07382550 | |
| 597 36.01340034 | |
| 598 35.95317726 | |
| 599 36.06010017 | |
| 600 36.00000000 | |
| 601 36.10648918 | |
| 602 36.04651163 | |
| 603 36.15257048 | |
| 604 36.09271523 | |
| 605 36.03305785 | |
| 606 35.97359736 | |
| 607 36.07907743 | |
| 608 36.01973684 | |
| 609 35.96059113 | |
| 610 35.90163934 | |
| 611 35.84288052 | |
| 612 35.78431373 | |
| 613 35.72593801 | |
| 614 35.83061889 | |
| 615 35.77235772 | |
| 616 35.71428571 | |
| 617 35.65640194 | |
| 618 35.59870550 | |
| 619 35.54119548 | |
| 620 35.48387097 | |
| 621 35.58776167 | |
| 622 35.53054662 | |
| 623 35.47351525 | |
| 624 35.57692308 | |
| 625 35.52000000 | |
| 626 35.62300319 | |
| 627 35.56618820 | |
| 628 35.50955414 | |
| 629 35.61208267 | |
| 630 35.71428571 | |
| 631 35.65768621 | |
| 632 35.75949367 | |
| 633 35.70300158 | |
| 634 35.64668770 | |
| 635 35.59055118 | |
| 636 35.53459119 | |
| 637 35.47880691 | |
| 638 35.42319749 | |
| 639 35.52425665 | |
| 640 35.62500000 | |
| 641 35.56942278 | |
| 642 35.51401869 | |
| 643 35.45878694 | |
| 644 35.55900621 | |
| 645 35.50387597 | |
| 646 35.44891641 | |
| 647 35.39412674 | |
| 648 35.33950617 | |
| 649 35.28505393 | |
| 650 35.23076923 | |
| 651 35.17665131 | |
| 652 35.12269939 | |
| 653 35.06891271 | |
| 654 35.16819572 | |
| 655 35.11450382 | |
| 656 35.21341463 | |
| 657 35.15981735 | |
| 658 35.25835866 | |
| 659 35.35660091 | |
| 660 35.30303030 | |
| 661 35.24962179 | |
| 662 35.34743202 | |
| 663 35.29411765 | |
| 664 35.24096386 | |
| 665 35.33834586 | |
| 666 35.28528529 | |
| 667 35.23238381 | |
| 668 35.32934132 | |
| 669 35.42600897 | |
| 670 35.37313433 | |
| 671 35.32041729 | |
| 672 35.26785714 | |
| 673 35.21545319 | |
| 674 35.16320475 | |
| 675 35.11111111 | |
| 676 35.05917160 | |
| 677 35.00738552 | |
| 678 34.95575221 | |
| 679 34.90427099 | |
| 680 35.00000000 | |
| 681 35.09544787 | |
| 682 35.04398827 | |
| 683 34.99267936 | |
| 684 35.08771930 | |
| 685 35.18248175 | |
| 686 35.13119534 | |
| 687 35.08005822 | |
| 688 35.02906977 | |
| 689 35.12336720 | |
| 690 35.07246377 | |
| 691 35.02170767 | |
| 692 34.97109827 | |
| 693 34.92063492 | |
| 694 34.87031700 | |
| 695 34.82014388 | |
| 696 34.91379310 | |
| 697 34.86370158 | |
| 698 34.81375358 | |
| 699 34.90701001 | |
| 700 34.85714286 | |
| 701 34.95007133 | |
| 702 35.04273504 | |
| 703 34.99288762 | |
| 704 35.08522727 | |
| 705 35.03546099 | |
| 706 34.98583569 | |
| 707 34.93635078 | |
| 708 34.88700565 | |
| 709 34.97884344 | |
| 710 34.92957746 | |
| 711 35.02109705 | |
| 712 34.97191011 | |
| 713 35.06311360 | |
| 714 35.15406162 | |
| 715 35.10489510 | |
| 716 35.05586592 | |
| 717 35.14644351 | |
| 718 35.09749304 | |
| 719 35.18776078 | |
| 720 35.13888889 | |
| 721 35.09015257 | |
| 722 35.04155125 | |
| 723 35.13139696 | |
| 724 35.08287293 | |
| 725 35.17241379 | |
| 726 35.12396694 | |
| 727 35.07565337 | |
| 728 35.16483516 | |
| 729 35.11659808 | |
| 730 35.06849315 | |
| 731 35.02051984 | |
| 732 34.97267760 | |
| 733 35.06139154 | |
| 734 35.14986376 | |
| 735 35.10204082 | |
| 736 35.05434783 | |
| 737 35.14246947 | |
| 738 35.09485095 | |
| 739 35.18267930 | |
| 740 35.13513514 | |
| 741 35.08771930 | |
| 742 35.04043127 | |
| 743 35.12786003 | |
| 744 35.21505376 | |
| 745 35.16778523 | |
| 746 35.12064343 | |
| 747 35.07362784 | |
| 748 35.02673797 | |
| 749 34.97997330 | |
| 750 35.06666667 | |
| Final result: 35.0667 +/- 1.7436 | |
| Random chance: 25.0000 +/- 1.5822 | |