JoshMe1 commited on
Commit
be755b1
·
verified ·
1 Parent(s): 40364d0

Training in progress, step 200, checkpoint

Browse files
last-checkpoint/adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:3ce02f701af666e8574f34bbece43cf678e6c43828c6401013c7adaee7f3e44c
3
  size 639691872
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:262adc43b1549747f8133c78c1f3cd45fcd802b97965c2e197dc99a7ceaf6d3b
3
  size 639691872
last-checkpoint/optimizer.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:8575e904c1cf22935656eef06768f9eada06da0066a043880265c8ccc56d8ae8
3
  size 1279529146
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d21352c173c6a408300739dc463c59f3dba7ad1fb915511917540a408a158f27
3
  size 1279529146
last-checkpoint/scheduler.pt CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:46c96054ea697f2f09206927300764c4f5f88c9d18ac53664cf1661a0b6f2665
3
  size 1064
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:50a635eef07ba685f548ffceaa8f54b55dd3ba9de5e267cb2673b0a5d4d9b18e
3
  size 1064
last-checkpoint/trainer_state.json CHANGED
@@ -1,9 +1,9 @@
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
- "epoch": 0.02502111156288118,
5
  "eval_steps": 500,
6
- "global_step": 100,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
@@ -1507,6 +1507,1506 @@
1507
  "rewards/margins": 0.3506268858909607,
1508
  "rewards/rejected": -0.010964250192046165,
1509
  "step": 100
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1510
  }
1511
  ],
1512
  "logging_steps": 1,
@@ -1521,7 +3021,7 @@
1521
  "should_evaluate": false,
1522
  "should_log": false,
1523
  "should_save": true,
1524
- "should_training_stop": false
1525
  },
1526
  "attributes": {}
1527
  }
 
1
  {
2
  "best_metric": null,
3
  "best_model_checkpoint": null,
4
+ "epoch": 0.05004222312576236,
5
  "eval_steps": 500,
6
+ "global_step": 200,
7
  "is_hyper_param_search": false,
8
  "is_local_process_zero": true,
9
  "is_world_process_zero": true,
 
1507
  "rewards/margins": 0.3506268858909607,
1508
  "rewards/rejected": -0.010964250192046165,
1509
  "step": 100
1510
+ },
1511
+ {
1512
+ "epoch": 0.025271322678509994,
1513
+ "grad_norm": 168.01998901367188,
1514
+ "learning_rate": 2.5250000000000004e-06,
1515
+ "logits/chosen": 0.5338684320449829,
1516
+ "logits/rejected": 0.5581062436103821,
1517
+ "logps/chosen": -830.5277099609375,
1518
+ "logps/rejected": -750.9749145507812,
1519
+ "loss": 5.1281,
1520
+ "rewards/accuracies": 0.6875,
1521
+ "rewards/chosen": 0.29842013120651245,
1522
+ "rewards/margins": 0.12503623962402344,
1523
+ "rewards/rejected": 0.1733839064836502,
1524
+ "step": 101
1525
+ },
1526
+ {
1527
+ "epoch": 0.025521533794138806,
1528
+ "grad_norm": 195.6433868408203,
1529
+ "learning_rate": 2.55e-06,
1530
+ "logits/chosen": 0.5473726987838745,
1531
+ "logits/rejected": 0.2889501452445984,
1532
+ "logps/chosen": -904.6567993164062,
1533
+ "logps/rejected": -989.4647216796875,
1534
+ "loss": 4.9998,
1535
+ "rewards/accuracies": 0.75,
1536
+ "rewards/chosen": 0.41574910283088684,
1537
+ "rewards/margins": 0.19607900083065033,
1538
+ "rewards/rejected": 0.21967005729675293,
1539
+ "step": 102
1540
+ },
1541
+ {
1542
+ "epoch": 0.025771744909767618,
1543
+ "grad_norm": 170.56797790527344,
1544
+ "learning_rate": 2.5750000000000003e-06,
1545
+ "logits/chosen": 0.8648879528045654,
1546
+ "logits/rejected": 0.5083369612693787,
1547
+ "logps/chosen": -743.2974243164062,
1548
+ "logps/rejected": -946.9625244140625,
1549
+ "loss": 5.1817,
1550
+ "rewards/accuracies": 0.6875,
1551
+ "rewards/chosen": 0.336568146944046,
1552
+ "rewards/margins": 0.1252390742301941,
1553
+ "rewards/rejected": 0.21132907271385193,
1554
+ "step": 103
1555
+ },
1556
+ {
1557
+ "epoch": 0.026021956025396427,
1558
+ "grad_norm": 179.4681854248047,
1559
+ "learning_rate": 2.6e-06,
1560
+ "logits/chosen": 0.6028002500534058,
1561
+ "logits/rejected": 0.6547164916992188,
1562
+ "logps/chosen": -834.4608154296875,
1563
+ "logps/rejected": -800.7316284179688,
1564
+ "loss": 5.4172,
1565
+ "rewards/accuracies": 0.5,
1566
+ "rewards/chosen": 0.5305736660957336,
1567
+ "rewards/margins": 0.05496921390295029,
1568
+ "rewards/rejected": 0.47560444474220276,
1569
+ "step": 104
1570
+ },
1571
+ {
1572
+ "epoch": 0.02627216714102524,
1573
+ "grad_norm": 177.14212036132812,
1574
+ "learning_rate": 2.6250000000000003e-06,
1575
+ "logits/chosen": 0.633183479309082,
1576
+ "logits/rejected": 0.49183252453804016,
1577
+ "logps/chosen": -769.5543823242188,
1578
+ "logps/rejected": -841.36181640625,
1579
+ "loss": 5.0401,
1580
+ "rewards/accuracies": 0.6875,
1581
+ "rewards/chosen": 0.3643644452095032,
1582
+ "rewards/margins": 0.16701143980026245,
1583
+ "rewards/rejected": 0.1973530352115631,
1584
+ "step": 105
1585
+ },
1586
+ {
1587
+ "epoch": 0.02652237825665405,
1588
+ "grad_norm": 183.1171112060547,
1589
+ "learning_rate": 2.6500000000000005e-06,
1590
+ "logits/chosen": 0.7262644171714783,
1591
+ "logits/rejected": 0.7168549299240112,
1592
+ "logps/chosen": -836.1957397460938,
1593
+ "logps/rejected": -949.1402587890625,
1594
+ "loss": 5.3246,
1595
+ "rewards/accuracies": 0.5,
1596
+ "rewards/chosen": 0.4721420407295227,
1597
+ "rewards/margins": 0.14782388508319855,
1598
+ "rewards/rejected": 0.32431814074516296,
1599
+ "step": 106
1600
+ },
1601
+ {
1602
+ "epoch": 0.026772589372282864,
1603
+ "grad_norm": 183.04190063476562,
1604
+ "learning_rate": 2.6750000000000002e-06,
1605
+ "logits/chosen": 0.45039886236190796,
1606
+ "logits/rejected": 0.23830902576446533,
1607
+ "logps/chosen": -868.3818359375,
1608
+ "logps/rejected": -880.0626220703125,
1609
+ "loss": 5.1669,
1610
+ "rewards/accuracies": 0.625,
1611
+ "rewards/chosen": 0.38201138377189636,
1612
+ "rewards/margins": 0.16360534727573395,
1613
+ "rewards/rejected": 0.2184060662984848,
1614
+ "step": 107
1615
+ },
1616
+ {
1617
+ "epoch": 0.027022800487911676,
1618
+ "grad_norm": 166.98008728027344,
1619
+ "learning_rate": 2.7000000000000004e-06,
1620
+ "logits/chosen": 0.5699177980422974,
1621
+ "logits/rejected": 0.5048665404319763,
1622
+ "logps/chosen": -841.4978637695312,
1623
+ "logps/rejected": -836.6079711914062,
1624
+ "loss": 4.5338,
1625
+ "rewards/accuracies": 0.8125,
1626
+ "rewards/chosen": 0.591092586517334,
1627
+ "rewards/margins": 0.3148172199726105,
1628
+ "rewards/rejected": 0.2762753963470459,
1629
+ "step": 108
1630
+ },
1631
+ {
1632
+ "epoch": 0.027273011603540488,
1633
+ "grad_norm": 174.3636474609375,
1634
+ "learning_rate": 2.7250000000000006e-06,
1635
+ "logits/chosen": 0.6328776478767395,
1636
+ "logits/rejected": 0.506734311580658,
1637
+ "logps/chosen": -1035.760009765625,
1638
+ "logps/rejected": -1037.7685546875,
1639
+ "loss": 4.5938,
1640
+ "rewards/accuracies": 0.8125,
1641
+ "rewards/chosen": 0.5912416577339172,
1642
+ "rewards/margins": 0.30340731143951416,
1643
+ "rewards/rejected": 0.2878343462944031,
1644
+ "step": 109
1645
+ },
1646
+ {
1647
+ "epoch": 0.0275232227191693,
1648
+ "grad_norm": 172.23876953125,
1649
+ "learning_rate": 2.7500000000000004e-06,
1650
+ "logits/chosen": 0.4434925317764282,
1651
+ "logits/rejected": 0.6842514872550964,
1652
+ "logps/chosen": -983.0321044921875,
1653
+ "logps/rejected": -788.3206787109375,
1654
+ "loss": 4.999,
1655
+ "rewards/accuracies": 0.625,
1656
+ "rewards/chosen": 0.3090551793575287,
1657
+ "rewards/margins": 0.17981156706809998,
1658
+ "rewards/rejected": 0.1292436271905899,
1659
+ "step": 110
1660
+ },
1661
+ {
1662
+ "epoch": 0.02777343383479811,
1663
+ "grad_norm": 172.0596466064453,
1664
+ "learning_rate": 2.7750000000000005e-06,
1665
+ "logits/chosen": 0.4155680537223816,
1666
+ "logits/rejected": 0.46463680267333984,
1667
+ "logps/chosen": -881.0513305664062,
1668
+ "logps/rejected": -925.1255493164062,
1669
+ "loss": 4.7598,
1670
+ "rewards/accuracies": 0.625,
1671
+ "rewards/chosen": 0.6203694939613342,
1672
+ "rewards/margins": 0.26839369535446167,
1673
+ "rewards/rejected": 0.35197576880455017,
1674
+ "step": 111
1675
+ },
1676
+ {
1677
+ "epoch": 0.02802364495042692,
1678
+ "grad_norm": 195.08697509765625,
1679
+ "learning_rate": 2.8000000000000003e-06,
1680
+ "logits/chosen": 0.605146586894989,
1681
+ "logits/rejected": 0.7462798357009888,
1682
+ "logps/chosen": -970.7579345703125,
1683
+ "logps/rejected": -969.8851318359375,
1684
+ "loss": 5.0111,
1685
+ "rewards/accuracies": 0.625,
1686
+ "rewards/chosen": 0.3882797658443451,
1687
+ "rewards/margins": 0.22670859098434448,
1688
+ "rewards/rejected": 0.1615711748600006,
1689
+ "step": 112
1690
+ },
1691
+ {
1692
+ "epoch": 0.028273856066055734,
1693
+ "grad_norm": 157.1236114501953,
1694
+ "learning_rate": 2.825e-06,
1695
+ "logits/chosen": 0.6366948485374451,
1696
+ "logits/rejected": 0.717607855796814,
1697
+ "logps/chosen": -820.1522827148438,
1698
+ "logps/rejected": -789.8685913085938,
1699
+ "loss": 5.0085,
1700
+ "rewards/accuracies": 0.625,
1701
+ "rewards/chosen": 0.4857552647590637,
1702
+ "rewards/margins": 0.18623466789722443,
1703
+ "rewards/rejected": 0.2995206117630005,
1704
+ "step": 113
1705
+ },
1706
+ {
1707
+ "epoch": 0.028524067181684546,
1708
+ "grad_norm": 198.33815002441406,
1709
+ "learning_rate": 2.85e-06,
1710
+ "logits/chosen": 0.7510839700698853,
1711
+ "logits/rejected": 0.6683776378631592,
1712
+ "logps/chosen": -951.9930419921875,
1713
+ "logps/rejected": -911.171142578125,
1714
+ "loss": 5.9293,
1715
+ "rewards/accuracies": 0.4375,
1716
+ "rewards/chosen": 0.3041432499885559,
1717
+ "rewards/margins": -0.05183648690581322,
1718
+ "rewards/rejected": 0.3559797406196594,
1719
+ "step": 114
1720
+ },
1721
+ {
1722
+ "epoch": 0.02877427829731336,
1723
+ "grad_norm": 156.49578857421875,
1724
+ "learning_rate": 2.875e-06,
1725
+ "logits/chosen": 0.3662444055080414,
1726
+ "logits/rejected": 0.35427069664001465,
1727
+ "logps/chosen": -725.6800537109375,
1728
+ "logps/rejected": -791.179931640625,
1729
+ "loss": 4.5455,
1730
+ "rewards/accuracies": 0.75,
1731
+ "rewards/chosen": 0.3427468240261078,
1732
+ "rewards/margins": 0.31882816553115845,
1733
+ "rewards/rejected": 0.023918677121400833,
1734
+ "step": 115
1735
+ },
1736
+ {
1737
+ "epoch": 0.02902448941294217,
1738
+ "grad_norm": 162.18934631347656,
1739
+ "learning_rate": 2.9e-06,
1740
+ "logits/chosen": 0.568724513053894,
1741
+ "logits/rejected": 0.4400956928730011,
1742
+ "logps/chosen": -718.4844360351562,
1743
+ "logps/rejected": -824.3160400390625,
1744
+ "loss": 4.7706,
1745
+ "rewards/accuracies": 0.625,
1746
+ "rewards/chosen": 0.5042533874511719,
1747
+ "rewards/margins": 0.26497164368629456,
1748
+ "rewards/rejected": 0.2392817586660385,
1749
+ "step": 116
1750
+ },
1751
+ {
1752
+ "epoch": 0.029274700528570983,
1753
+ "grad_norm": 179.54246520996094,
1754
+ "learning_rate": 2.925e-06,
1755
+ "logits/chosen": 0.5801786184310913,
1756
+ "logits/rejected": 0.549165666103363,
1757
+ "logps/chosen": -898.4237060546875,
1758
+ "logps/rejected": -969.0765380859375,
1759
+ "loss": 4.3671,
1760
+ "rewards/accuracies": 0.75,
1761
+ "rewards/chosen": 0.66265469789505,
1762
+ "rewards/margins": 0.3924952745437622,
1763
+ "rewards/rejected": 0.27015945315361023,
1764
+ "step": 117
1765
+ },
1766
+ {
1767
+ "epoch": 0.029524911644199795,
1768
+ "grad_norm": 181.2903289794922,
1769
+ "learning_rate": 2.95e-06,
1770
+ "logits/chosen": 0.7072390913963318,
1771
+ "logits/rejected": 0.599428653717041,
1772
+ "logps/chosen": -873.1541748046875,
1773
+ "logps/rejected": -932.2781982421875,
1774
+ "loss": 5.0481,
1775
+ "rewards/accuracies": 0.6875,
1776
+ "rewards/chosen": 0.516876220703125,
1777
+ "rewards/margins": 0.1785745620727539,
1778
+ "rewards/rejected": 0.3383016586303711,
1779
+ "step": 118
1780
+ },
1781
+ {
1782
+ "epoch": 0.029775122759828604,
1783
+ "grad_norm": 170.22119140625,
1784
+ "learning_rate": 2.9750000000000003e-06,
1785
+ "logits/chosen": 0.9537469744682312,
1786
+ "logits/rejected": 0.8464532494544983,
1787
+ "logps/chosen": -755.2413330078125,
1788
+ "logps/rejected": -743.6602172851562,
1789
+ "loss": 4.8546,
1790
+ "rewards/accuracies": 0.625,
1791
+ "rewards/chosen": 0.5026987791061401,
1792
+ "rewards/margins": 0.2353692501783371,
1793
+ "rewards/rejected": 0.26732951402664185,
1794
+ "step": 119
1795
+ },
1796
+ {
1797
+ "epoch": 0.030025333875457416,
1798
+ "grad_norm": 144.19140625,
1799
+ "learning_rate": 3e-06,
1800
+ "logits/chosen": 0.6080533266067505,
1801
+ "logits/rejected": 0.5463143587112427,
1802
+ "logps/chosen": -680.61181640625,
1803
+ "logps/rejected": -740.6124877929688,
1804
+ "loss": 4.1267,
1805
+ "rewards/accuracies": 0.875,
1806
+ "rewards/chosen": 0.5357052683830261,
1807
+ "rewards/margins": 0.4544171392917633,
1808
+ "rewards/rejected": 0.08128809928894043,
1809
+ "step": 120
1810
+ },
1811
+ {
1812
+ "epoch": 0.03027554499108623,
1813
+ "grad_norm": 152.67770385742188,
1814
+ "learning_rate": 3.0250000000000003e-06,
1815
+ "logits/chosen": 0.433148592710495,
1816
+ "logits/rejected": 0.657333493232727,
1817
+ "logps/chosen": -992.8467407226562,
1818
+ "logps/rejected": -825.07958984375,
1819
+ "loss": 4.248,
1820
+ "rewards/accuracies": 0.75,
1821
+ "rewards/chosen": 0.5468575358390808,
1822
+ "rewards/margins": 0.4552149772644043,
1823
+ "rewards/rejected": 0.09164253622293472,
1824
+ "step": 121
1825
+ },
1826
+ {
1827
+ "epoch": 0.03052575610671504,
1828
+ "grad_norm": 170.1833038330078,
1829
+ "learning_rate": 3.05e-06,
1830
+ "logits/chosen": 1.1314983367919922,
1831
+ "logits/rejected": 1.0099120140075684,
1832
+ "logps/chosen": -732.5499877929688,
1833
+ "logps/rejected": -709.9703979492188,
1834
+ "loss": 5.4108,
1835
+ "rewards/accuracies": 0.5,
1836
+ "rewards/chosen": 0.35310542583465576,
1837
+ "rewards/margins": 0.10829624533653259,
1838
+ "rewards/rejected": 0.24480919539928436,
1839
+ "step": 122
1840
+ },
1841
+ {
1842
+ "epoch": 0.030775967222343853,
1843
+ "grad_norm": 121.0159912109375,
1844
+ "learning_rate": 3.075e-06,
1845
+ "logits/chosen": 0.9415697455406189,
1846
+ "logits/rejected": 0.8197463154792786,
1847
+ "logps/chosen": -769.3919677734375,
1848
+ "logps/rejected": -631.9843139648438,
1849
+ "loss": 3.956,
1850
+ "rewards/accuracies": 0.875,
1851
+ "rewards/chosen": 0.5862818956375122,
1852
+ "rewards/margins": 0.5186235308647156,
1853
+ "rewards/rejected": 0.0676584541797638,
1854
+ "step": 123
1855
+ },
1856
+ {
1857
+ "epoch": 0.031026178337972665,
1858
+ "grad_norm": 176.4051513671875,
1859
+ "learning_rate": 3.1000000000000004e-06,
1860
+ "logits/chosen": 0.5172820687294006,
1861
+ "logits/rejected": 0.3080348074436188,
1862
+ "logps/chosen": -974.28173828125,
1863
+ "logps/rejected": -859.1981201171875,
1864
+ "loss": 4.7509,
1865
+ "rewards/accuracies": 0.75,
1866
+ "rewards/chosen": 0.5816446542739868,
1867
+ "rewards/margins": 0.2865919768810272,
1868
+ "rewards/rejected": 0.2950526475906372,
1869
+ "step": 124
1870
+ },
1871
+ {
1872
+ "epoch": 0.03127638945360148,
1873
+ "grad_norm": 155.2854461669922,
1874
+ "learning_rate": 3.125e-06,
1875
+ "logits/chosen": 1.1160801649093628,
1876
+ "logits/rejected": 0.9693074226379395,
1877
+ "logps/chosen": -568.8187255859375,
1878
+ "logps/rejected": -709.5980834960938,
1879
+ "loss": 4.9778,
1880
+ "rewards/accuracies": 0.75,
1881
+ "rewards/chosen": 0.40664032101631165,
1882
+ "rewards/margins": 0.20195113122463226,
1883
+ "rewards/rejected": 0.2046891748905182,
1884
+ "step": 125
1885
+ },
1886
+ {
1887
+ "epoch": 0.03152660056923029,
1888
+ "grad_norm": 174.4468536376953,
1889
+ "learning_rate": 3.1500000000000003e-06,
1890
+ "logits/chosen": 0.5777021646499634,
1891
+ "logits/rejected": 0.6705082058906555,
1892
+ "logps/chosen": -1009.0233154296875,
1893
+ "logps/rejected": -1065.4202880859375,
1894
+ "loss": 4.7682,
1895
+ "rewards/accuracies": 0.625,
1896
+ "rewards/chosen": 0.6260993480682373,
1897
+ "rewards/margins": 0.3270193338394165,
1898
+ "rewards/rejected": 0.29908010363578796,
1899
+ "step": 126
1900
+ },
1901
+ {
1902
+ "epoch": 0.0317768116848591,
1903
+ "grad_norm": 135.96408081054688,
1904
+ "learning_rate": 3.175e-06,
1905
+ "logits/chosen": 0.9410853385925293,
1906
+ "logits/rejected": 0.9606730937957764,
1907
+ "logps/chosen": -729.5543212890625,
1908
+ "logps/rejected": -651.3915405273438,
1909
+ "loss": 4.3649,
1910
+ "rewards/accuracies": 0.625,
1911
+ "rewards/chosen": 0.4558297395706177,
1912
+ "rewards/margins": 0.4536476135253906,
1913
+ "rewards/rejected": 0.002182198688387871,
1914
+ "step": 127
1915
+ },
1916
+ {
1917
+ "epoch": 0.032027022800487914,
1918
+ "grad_norm": 170.97409057617188,
1919
+ "learning_rate": 3.2000000000000003e-06,
1920
+ "logits/chosen": 0.5134232640266418,
1921
+ "logits/rejected": 0.6284916400909424,
1922
+ "logps/chosen": -871.0223999023438,
1923
+ "logps/rejected": -807.8709106445312,
1924
+ "loss": 4.8398,
1925
+ "rewards/accuracies": 0.625,
1926
+ "rewards/chosen": 0.44705837965011597,
1927
+ "rewards/margins": 0.23864485323429108,
1928
+ "rewards/rejected": 0.20841355621814728,
1929
+ "step": 128
1930
+ },
1931
+ {
1932
+ "epoch": 0.03227723391611673,
1933
+ "grad_norm": 134.9060821533203,
1934
+ "learning_rate": 3.2250000000000005e-06,
1935
+ "logits/chosen": 0.295400470495224,
1936
+ "logits/rejected": 0.3619856834411621,
1937
+ "logps/chosen": -827.4248046875,
1938
+ "logps/rejected": -775.391357421875,
1939
+ "loss": 4.6104,
1940
+ "rewards/accuracies": 0.75,
1941
+ "rewards/chosen": 0.45551204681396484,
1942
+ "rewards/margins": 0.318456768989563,
1943
+ "rewards/rejected": 0.13705529272556305,
1944
+ "step": 129
1945
+ },
1946
+ {
1947
+ "epoch": 0.03252744503174553,
1948
+ "grad_norm": 139.79693603515625,
1949
+ "learning_rate": 3.2500000000000002e-06,
1950
+ "logits/chosen": 0.7282382845878601,
1951
+ "logits/rejected": 0.6856253743171692,
1952
+ "logps/chosen": -897.2752685546875,
1953
+ "logps/rejected": -857.0909423828125,
1954
+ "loss": 3.9534,
1955
+ "rewards/accuracies": 0.875,
1956
+ "rewards/chosen": 0.6775761842727661,
1957
+ "rewards/margins": 0.49155694246292114,
1958
+ "rewards/rejected": 0.18601924180984497,
1959
+ "step": 130
1960
+ },
1961
+ {
1962
+ "epoch": 0.032777656147374344,
1963
+ "grad_norm": 165.3737030029297,
1964
+ "learning_rate": 3.2750000000000004e-06,
1965
+ "logits/chosen": 0.6580913066864014,
1966
+ "logits/rejected": 0.40577080845832825,
1967
+ "logps/chosen": -777.4276733398438,
1968
+ "logps/rejected": -795.449951171875,
1969
+ "loss": 4.3611,
1970
+ "rewards/accuracies": 0.75,
1971
+ "rewards/chosen": 0.5797842144966125,
1972
+ "rewards/margins": 0.39686450362205505,
1973
+ "rewards/rejected": 0.1829197257757187,
1974
+ "step": 131
1975
+ },
1976
+ {
1977
+ "epoch": 0.033027867263003156,
1978
+ "grad_norm": 148.64382934570312,
1979
+ "learning_rate": 3.3000000000000006e-06,
1980
+ "logits/chosen": 0.7884936928749084,
1981
+ "logits/rejected": 0.8930270075798035,
1982
+ "logps/chosen": -911.4806518554688,
1983
+ "logps/rejected": -826.3285522460938,
1984
+ "loss": 3.9331,
1985
+ "rewards/accuracies": 0.9375,
1986
+ "rewards/chosen": 0.6084189414978027,
1987
+ "rewards/margins": 0.5333962440490723,
1988
+ "rewards/rejected": 0.07502269744873047,
1989
+ "step": 132
1990
+ },
1991
+ {
1992
+ "epoch": 0.03327807837863197,
1993
+ "grad_norm": 149.3182373046875,
1994
+ "learning_rate": 3.3250000000000004e-06,
1995
+ "logits/chosen": 0.9997194409370422,
1996
+ "logits/rejected": 0.8741428256034851,
1997
+ "logps/chosen": -750.9700927734375,
1998
+ "logps/rejected": -828.2926635742188,
1999
+ "loss": 4.5261,
2000
+ "rewards/accuracies": 0.75,
2001
+ "rewards/chosen": 0.5206612944602966,
2002
+ "rewards/margins": 0.34372854232788086,
2003
+ "rewards/rejected": 0.17693273723125458,
2004
+ "step": 133
2005
+ },
2006
+ {
2007
+ "epoch": 0.03352828949426078,
2008
+ "grad_norm": 205.8184051513672,
2009
+ "learning_rate": 3.3500000000000005e-06,
2010
+ "logits/chosen": 0.5166525840759277,
2011
+ "logits/rejected": 0.3342657685279846,
2012
+ "logps/chosen": -820.9586181640625,
2013
+ "logps/rejected": -1032.162841796875,
2014
+ "loss": 4.9892,
2015
+ "rewards/accuracies": 0.75,
2016
+ "rewards/chosen": 0.6503936648368835,
2017
+ "rewards/margins": 0.24925732612609863,
2018
+ "rewards/rejected": 0.4011363089084625,
2019
+ "step": 134
2020
+ },
2021
+ {
2022
+ "epoch": 0.03377850060988959,
2023
+ "grad_norm": 148.48712158203125,
2024
+ "learning_rate": 3.3750000000000003e-06,
2025
+ "logits/chosen": 0.3050788342952728,
2026
+ "logits/rejected": 0.27589350938796997,
2027
+ "logps/chosen": -950.6005249023438,
2028
+ "logps/rejected": -985.9448852539062,
2029
+ "loss": 4.3223,
2030
+ "rewards/accuracies": 0.75,
2031
+ "rewards/chosen": 0.6733807325363159,
2032
+ "rewards/margins": 0.41189804673194885,
2033
+ "rewards/rejected": 0.26148271560668945,
2034
+ "step": 135
2035
+ },
2036
+ {
2037
+ "epoch": 0.034028711725518405,
2038
+ "grad_norm": 162.81271362304688,
2039
+ "learning_rate": 3.4000000000000005e-06,
2040
+ "logits/chosen": 0.8448665142059326,
2041
+ "logits/rejected": 0.6966872811317444,
2042
+ "logps/chosen": -743.9957275390625,
2043
+ "logps/rejected": -750.2032470703125,
2044
+ "loss": 5.0515,
2045
+ "rewards/accuracies": 0.5625,
2046
+ "rewards/chosen": 0.3949689567089081,
2047
+ "rewards/margins": 0.22736862301826477,
2048
+ "rewards/rejected": 0.1676003336906433,
2049
+ "step": 136
2050
+ },
2051
+ {
2052
+ "epoch": 0.03427892284114722,
2053
+ "grad_norm": 139.54498291015625,
2054
+ "learning_rate": 3.4250000000000007e-06,
2055
+ "logits/chosen": 0.553779661655426,
2056
+ "logits/rejected": 0.5152968168258667,
2057
+ "logps/chosen": -796.459228515625,
2058
+ "logps/rejected": -836.67822265625,
2059
+ "loss": 4.3161,
2060
+ "rewards/accuracies": 0.75,
2061
+ "rewards/chosen": 0.7378758192062378,
2062
+ "rewards/margins": 0.4128691554069519,
2063
+ "rewards/rejected": 0.3250066041946411,
2064
+ "step": 137
2065
+ },
2066
+ {
2067
+ "epoch": 0.03452913395677603,
2068
+ "grad_norm": 151.59085083007812,
2069
+ "learning_rate": 3.45e-06,
2070
+ "logits/chosen": 0.5669974684715271,
2071
+ "logits/rejected": 0.6564877033233643,
2072
+ "logps/chosen": -815.4319458007812,
2073
+ "logps/rejected": -738.3330078125,
2074
+ "loss": 4.5803,
2075
+ "rewards/accuracies": 0.8125,
2076
+ "rewards/chosen": 0.4881832003593445,
2077
+ "rewards/margins": 0.350455105304718,
2078
+ "rewards/rejected": 0.13772808015346527,
2079
+ "step": 138
2080
+ },
2081
+ {
2082
+ "epoch": 0.03477934507240484,
2083
+ "grad_norm": 148.89613342285156,
2084
+ "learning_rate": 3.475e-06,
2085
+ "logits/chosen": 0.5828185081481934,
2086
+ "logits/rejected": 0.3963644206523895,
2087
+ "logps/chosen": -909.1538696289062,
2088
+ "logps/rejected": -997.1546630859375,
2089
+ "loss": 3.4615,
2090
+ "rewards/accuracies": 0.875,
2091
+ "rewards/chosen": 0.8202193975448608,
2092
+ "rewards/margins": 0.8132323622703552,
2093
+ "rewards/rejected": 0.006987094879150391,
2094
+ "step": 139
2095
+ },
2096
+ {
2097
+ "epoch": 0.035029556188033655,
2098
+ "grad_norm": 153.6221160888672,
2099
+ "learning_rate": 3.5e-06,
2100
+ "logits/chosen": 1.0873308181762695,
2101
+ "logits/rejected": 1.020272970199585,
2102
+ "logps/chosen": -749.4591674804688,
2103
+ "logps/rejected": -737.8289794921875,
2104
+ "loss": 4.6594,
2105
+ "rewards/accuracies": 0.6875,
2106
+ "rewards/chosen": 0.841954231262207,
2107
+ "rewards/margins": 0.32822534441947937,
2108
+ "rewards/rejected": 0.51372891664505,
2109
+ "step": 140
2110
+ },
2111
+ {
2112
+ "epoch": 0.03527976730366247,
2113
+ "grad_norm": 163.4862518310547,
2114
+ "learning_rate": 3.525e-06,
2115
+ "logits/chosen": 0.6650976538658142,
2116
+ "logits/rejected": 0.5198307037353516,
2117
+ "logps/chosen": -900.0272216796875,
2118
+ "logps/rejected": -877.53466796875,
2119
+ "loss": 5.235,
2120
+ "rewards/accuracies": 0.625,
2121
+ "rewards/chosen": 0.7146763801574707,
2122
+ "rewards/margins": 0.21902096271514893,
2123
+ "rewards/rejected": 0.4956553280353546,
2124
+ "step": 141
2125
+ },
2126
+ {
2127
+ "epoch": 0.03552997841929128,
2128
+ "grad_norm": 175.59779357910156,
2129
+ "learning_rate": 3.5500000000000003e-06,
2130
+ "logits/chosen": 0.48263072967529297,
2131
+ "logits/rejected": 0.3903447985649109,
2132
+ "logps/chosen": -1006.64111328125,
2133
+ "logps/rejected": -1018.9843139648438,
2134
+ "loss": 4.2582,
2135
+ "rewards/accuracies": 0.875,
2136
+ "rewards/chosen": 0.7219852805137634,
2137
+ "rewards/margins": 0.40434205532073975,
2138
+ "rewards/rejected": 0.3176431953907013,
2139
+ "step": 142
2140
+ },
2141
+ {
2142
+ "epoch": 0.03578018953492009,
2143
+ "grad_norm": 129.54666137695312,
2144
+ "learning_rate": 3.575e-06,
2145
+ "logits/chosen": 0.9929988384246826,
2146
+ "logits/rejected": 0.9958979487419128,
2147
+ "logps/chosen": -597.6022338867188,
2148
+ "logps/rejected": -560.0516967773438,
2149
+ "loss": 4.6597,
2150
+ "rewards/accuracies": 0.625,
2151
+ "rewards/chosen": 0.4857937693595886,
2152
+ "rewards/margins": 0.47004491090774536,
2153
+ "rewards/rejected": 0.01574888825416565,
2154
+ "step": 143
2155
+ },
2156
+ {
2157
+ "epoch": 0.036030400650548904,
2158
+ "grad_norm": 144.1933135986328,
2159
+ "learning_rate": 3.6000000000000003e-06,
2160
+ "logits/chosen": 0.5605639219284058,
2161
+ "logits/rejected": 0.44549262523651123,
2162
+ "logps/chosen": -890.560302734375,
2163
+ "logps/rejected": -883.78369140625,
2164
+ "loss": 3.4495,
2165
+ "rewards/accuracies": 0.8125,
2166
+ "rewards/chosen": 1.008372187614441,
2167
+ "rewards/margins": 0.813175618648529,
2168
+ "rewards/rejected": 0.1951965093612671,
2169
+ "step": 144
2170
+ },
2171
+ {
2172
+ "epoch": 0.036280611766177716,
2173
+ "grad_norm": 164.4710235595703,
2174
+ "learning_rate": 3.625e-06,
2175
+ "logits/chosen": 0.3835439085960388,
2176
+ "logits/rejected": 0.2644839286804199,
2177
+ "logps/chosen": -756.0498046875,
2178
+ "logps/rejected": -946.8538818359375,
2179
+ "loss": 4.4913,
2180
+ "rewards/accuracies": 0.6875,
2181
+ "rewards/chosen": 0.6012188196182251,
2182
+ "rewards/margins": 0.5210594534873962,
2183
+ "rewards/rejected": 0.08015939593315125,
2184
+ "step": 145
2185
+ },
2186
+ {
2187
+ "epoch": 0.03653082288180652,
2188
+ "grad_norm": 117.87698364257812,
2189
+ "learning_rate": 3.65e-06,
2190
+ "logits/chosen": 0.598304033279419,
2191
+ "logits/rejected": 0.47974368929862976,
2192
+ "logps/chosen": -849.1965942382812,
2193
+ "logps/rejected": -900.4439697265625,
2194
+ "loss": 3.3203,
2195
+ "rewards/accuracies": 0.875,
2196
+ "rewards/chosen": 0.8496798276901245,
2197
+ "rewards/margins": 0.8870599269866943,
2198
+ "rewards/rejected": -0.037380047142505646,
2199
+ "step": 146
2200
+ },
2201
+ {
2202
+ "epoch": 0.03678103399743533,
2203
+ "grad_norm": 164.40049743652344,
2204
+ "learning_rate": 3.6750000000000004e-06,
2205
+ "logits/chosen": 0.6101506948471069,
2206
+ "logits/rejected": 0.6310951113700867,
2207
+ "logps/chosen": -801.3853149414062,
2208
+ "logps/rejected": -883.5050048828125,
2209
+ "loss": 4.7674,
2210
+ "rewards/accuracies": 0.75,
2211
+ "rewards/chosen": 0.7482701539993286,
2212
+ "rewards/margins": 0.41238662600517273,
2213
+ "rewards/rejected": 0.33588361740112305,
2214
+ "step": 147
2215
+ },
2216
+ {
2217
+ "epoch": 0.037031245113064146,
2218
+ "grad_norm": 163.932861328125,
2219
+ "learning_rate": 3.7e-06,
2220
+ "logits/chosen": 1.0259884595870972,
2221
+ "logits/rejected": 1.1341297626495361,
2222
+ "logps/chosen": -858.99169921875,
2223
+ "logps/rejected": -760.257568359375,
2224
+ "loss": 4.0584,
2225
+ "rewards/accuracies": 0.625,
2226
+ "rewards/chosen": 0.6627655029296875,
2227
+ "rewards/margins": 0.5813624858856201,
2228
+ "rewards/rejected": 0.08140295743942261,
2229
+ "step": 148
2230
+ },
2231
+ {
2232
+ "epoch": 0.03728145622869296,
2233
+ "grad_norm": 178.82398986816406,
2234
+ "learning_rate": 3.7250000000000003e-06,
2235
+ "logits/chosen": 0.8155965805053711,
2236
+ "logits/rejected": 0.6099752187728882,
2237
+ "logps/chosen": -817.9868774414062,
2238
+ "logps/rejected": -1045.9879150390625,
2239
+ "loss": 4.7943,
2240
+ "rewards/accuracies": 0.6875,
2241
+ "rewards/chosen": 0.6769387125968933,
2242
+ "rewards/margins": 0.3597002923488617,
2243
+ "rewards/rejected": 0.31723839044570923,
2244
+ "step": 149
2245
+ },
2246
+ {
2247
+ "epoch": 0.03753166734432177,
2248
+ "grad_norm": 169.52249145507812,
2249
+ "learning_rate": 3.7500000000000005e-06,
2250
+ "logits/chosen": 0.5471953749656677,
2251
+ "logits/rejected": 0.5662561058998108,
2252
+ "logps/chosen": -986.181884765625,
2253
+ "logps/rejected": -1016.1942138671875,
2254
+ "loss": 3.7821,
2255
+ "rewards/accuracies": 0.6875,
2256
+ "rewards/chosen": 0.6381077766418457,
2257
+ "rewards/margins": 0.8503923416137695,
2258
+ "rewards/rejected": -0.21228459477424622,
2259
+ "step": 150
2260
+ },
2261
+ {
2262
+ "epoch": 0.03778187845995058,
2263
+ "grad_norm": 182.09280395507812,
2264
+ "learning_rate": 3.7750000000000003e-06,
2265
+ "logits/chosen": 0.6060510873794556,
2266
+ "logits/rejected": 0.4075266420841217,
2267
+ "logps/chosen": -821.666015625,
2268
+ "logps/rejected": -908.2159423828125,
2269
+ "loss": 4.6201,
2270
+ "rewards/accuracies": 0.8125,
2271
+ "rewards/chosen": 0.5977217555046082,
2272
+ "rewards/margins": 0.3014836609363556,
2273
+ "rewards/rejected": 0.29623815417289734,
2274
+ "step": 151
2275
+ },
2276
+ {
2277
+ "epoch": 0.038032089575579395,
2278
+ "grad_norm": 149.020263671875,
2279
+ "learning_rate": 3.8000000000000005e-06,
2280
+ "logits/chosen": 0.9842261075973511,
2281
+ "logits/rejected": 0.8099168539047241,
2282
+ "logps/chosen": -836.44287109375,
2283
+ "logps/rejected": -982.6625366210938,
2284
+ "loss": 3.7042,
2285
+ "rewards/accuracies": 0.9375,
2286
+ "rewards/chosen": 1.0862705707550049,
2287
+ "rewards/margins": 0.7122077941894531,
2288
+ "rewards/rejected": 0.37406274676322937,
2289
+ "step": 152
2290
+ },
2291
+ {
2292
+ "epoch": 0.03828230069120821,
2293
+ "grad_norm": 159.7766571044922,
2294
+ "learning_rate": 3.825000000000001e-06,
2295
+ "logits/chosen": 0.7192105650901794,
2296
+ "logits/rejected": 0.5668576955795288,
2297
+ "logps/chosen": -801.5166015625,
2298
+ "logps/rejected": -979.3016357421875,
2299
+ "loss": 4.1193,
2300
+ "rewards/accuracies": 0.8125,
2301
+ "rewards/chosen": 0.6885128021240234,
2302
+ "rewards/margins": 0.4991152584552765,
2303
+ "rewards/rejected": 0.18939754366874695,
2304
+ "step": 153
2305
+ },
2306
+ {
2307
+ "epoch": 0.03853251180683702,
2308
+ "grad_norm": 166.79933166503906,
2309
+ "learning_rate": 3.85e-06,
2310
+ "logits/chosen": 0.7860902547836304,
2311
+ "logits/rejected": 0.8038786053657532,
2312
+ "logps/chosen": -810.453857421875,
2313
+ "logps/rejected": -723.3671875,
2314
+ "loss": 4.1486,
2315
+ "rewards/accuracies": 0.875,
2316
+ "rewards/chosen": 0.6507831811904907,
2317
+ "rewards/margins": 0.6382051110267639,
2318
+ "rewards/rejected": 0.012578055262565613,
2319
+ "step": 154
2320
+ },
2321
+ {
2322
+ "epoch": 0.03878272292246583,
2323
+ "grad_norm": 173.12013244628906,
2324
+ "learning_rate": 3.875e-06,
2325
+ "logits/chosen": 0.9088850021362305,
2326
+ "logits/rejected": 0.7919357419013977,
2327
+ "logps/chosen": -913.0542602539062,
2328
+ "logps/rejected": -926.7158203125,
2329
+ "loss": 4.406,
2330
+ "rewards/accuracies": 0.6875,
2331
+ "rewards/chosen": 0.8225826621055603,
2332
+ "rewards/margins": 0.5563416481018066,
2333
+ "rewards/rejected": 0.2662409842014313,
2334
+ "step": 155
2335
+ },
2336
+ {
2337
+ "epoch": 0.039032934038094644,
2338
+ "grad_norm": 171.97964477539062,
2339
+ "learning_rate": 3.900000000000001e-06,
2340
+ "logits/chosen": 0.6967225074768066,
2341
+ "logits/rejected": 0.540969967842102,
2342
+ "logps/chosen": -849.8792114257812,
2343
+ "logps/rejected": -913.679443359375,
2344
+ "loss": 4.5975,
2345
+ "rewards/accuracies": 0.6875,
2346
+ "rewards/chosen": 0.7197656035423279,
2347
+ "rewards/margins": 0.35648685693740845,
2348
+ "rewards/rejected": 0.3632788062095642,
2349
+ "step": 156
2350
+ },
2351
+ {
2352
+ "epoch": 0.039283145153723456,
2353
+ "grad_norm": 144.37672424316406,
2354
+ "learning_rate": 3.9250000000000005e-06,
2355
+ "logits/chosen": 0.6681056022644043,
2356
+ "logits/rejected": 0.7472302317619324,
2357
+ "logps/chosen": -798.53759765625,
2358
+ "logps/rejected": -854.90087890625,
2359
+ "loss": 3.7286,
2360
+ "rewards/accuracies": 0.9375,
2361
+ "rewards/chosen": 0.7991371154785156,
2362
+ "rewards/margins": 0.7337710857391357,
2363
+ "rewards/rejected": 0.06536597013473511,
2364
+ "step": 157
2365
+ },
2366
+ {
2367
+ "epoch": 0.03953335626935227,
2368
+ "grad_norm": 141.87095642089844,
2369
+ "learning_rate": 3.95e-06,
2370
+ "logits/chosen": 0.6761986613273621,
2371
+ "logits/rejected": 0.579059362411499,
2372
+ "logps/chosen": -909.6781616210938,
2373
+ "logps/rejected": -862.220458984375,
2374
+ "loss": 3.7281,
2375
+ "rewards/accuracies": 0.9375,
2376
+ "rewards/chosen": 0.5470597743988037,
2377
+ "rewards/margins": 0.6505722999572754,
2378
+ "rewards/rejected": -0.10351258516311646,
2379
+ "step": 158
2380
+ },
2381
+ {
2382
+ "epoch": 0.03978356738498108,
2383
+ "grad_norm": 112.09781646728516,
2384
+ "learning_rate": 3.975000000000001e-06,
2385
+ "logits/chosen": 0.5438903570175171,
2386
+ "logits/rejected": 0.7648389339447021,
2387
+ "logps/chosen": -990.7408447265625,
2388
+ "logps/rejected": -835.4803466796875,
2389
+ "loss": 2.564,
2390
+ "rewards/accuracies": 1.0,
2391
+ "rewards/chosen": 1.2782984972000122,
2392
+ "rewards/margins": 1.3758175373077393,
2393
+ "rewards/rejected": -0.09751906245946884,
2394
+ "step": 159
2395
+ },
2396
+ {
2397
+ "epoch": 0.04003377850060989,
2398
+ "grad_norm": 133.63601684570312,
2399
+ "learning_rate": 4.000000000000001e-06,
2400
+ "logits/chosen": 0.560940146446228,
2401
+ "logits/rejected": 0.672624945640564,
2402
+ "logps/chosen": -807.1983642578125,
2403
+ "logps/rejected": -861.266845703125,
2404
+ "loss": 3.7409,
2405
+ "rewards/accuracies": 0.9375,
2406
+ "rewards/chosen": 0.8021640777587891,
2407
+ "rewards/margins": 0.6544591784477234,
2408
+ "rewards/rejected": 0.14770489931106567,
2409
+ "step": 160
2410
+ },
2411
+ {
2412
+ "epoch": 0.0402839896162387,
2413
+ "grad_norm": 168.26402282714844,
2414
+ "learning_rate": 4.0250000000000004e-06,
2415
+ "logits/chosen": 0.27081358432769775,
2416
+ "logits/rejected": 0.36320167779922485,
2417
+ "logps/chosen": -950.2415161132812,
2418
+ "logps/rejected": -933.5248413085938,
2419
+ "loss": 3.8679,
2420
+ "rewards/accuracies": 0.8125,
2421
+ "rewards/chosen": 0.9346640706062317,
2422
+ "rewards/margins": 0.7012830972671509,
2423
+ "rewards/rejected": 0.23338094353675842,
2424
+ "step": 161
2425
+ },
2426
+ {
2427
+ "epoch": 0.04053420073186751,
2428
+ "grad_norm": 189.97183227539062,
2429
+ "learning_rate": 4.05e-06,
2430
+ "logits/chosen": 0.7229450941085815,
2431
+ "logits/rejected": 0.6292408108711243,
2432
+ "logps/chosen": -815.431396484375,
2433
+ "logps/rejected": -854.240478515625,
2434
+ "loss": 5.1314,
2435
+ "rewards/accuracies": 0.75,
2436
+ "rewards/chosen": 0.556159257888794,
2437
+ "rewards/margins": 0.33854156732559204,
2438
+ "rewards/rejected": 0.2176176756620407,
2439
+ "step": 162
2440
+ },
2441
+ {
2442
+ "epoch": 0.04078441184749632,
2443
+ "grad_norm": 194.52952575683594,
2444
+ "learning_rate": 4.075e-06,
2445
+ "logits/chosen": 0.7778934240341187,
2446
+ "logits/rejected": 0.737458348274231,
2447
+ "logps/chosen": -692.4363403320312,
2448
+ "logps/rejected": -810.5929565429688,
2449
+ "loss": 4.9497,
2450
+ "rewards/accuracies": 0.625,
2451
+ "rewards/chosen": 0.5305599570274353,
2452
+ "rewards/margins": 0.48551517724990845,
2453
+ "rewards/rejected": 0.045044802129268646,
2454
+ "step": 163
2455
+ },
2456
+ {
2457
+ "epoch": 0.041034622963125135,
2458
+ "grad_norm": 143.23312377929688,
2459
+ "learning_rate": 4.1e-06,
2460
+ "logits/chosen": 0.7502599358558655,
2461
+ "logits/rejected": 0.7829670906066895,
2462
+ "logps/chosen": -908.2646484375,
2463
+ "logps/rejected": -864.3381958007812,
2464
+ "loss": 3.4274,
2465
+ "rewards/accuracies": 0.9375,
2466
+ "rewards/chosen": 0.8055950403213501,
2467
+ "rewards/margins": 0.9003482460975647,
2468
+ "rewards/rejected": -0.09475325047969818,
2469
+ "step": 164
2470
+ },
2471
+ {
2472
+ "epoch": 0.04128483407875395,
2473
+ "grad_norm": 157.2545623779297,
2474
+ "learning_rate": 4.125e-06,
2475
+ "logits/chosen": 0.42867469787597656,
2476
+ "logits/rejected": 0.26693516969680786,
2477
+ "logps/chosen": -951.4996948242188,
2478
+ "logps/rejected": -904.564697265625,
2479
+ "loss": 3.9217,
2480
+ "rewards/accuracies": 0.8125,
2481
+ "rewards/chosen": 0.9759159088134766,
2482
+ "rewards/margins": 0.6854816675186157,
2483
+ "rewards/rejected": 0.29043421149253845,
2484
+ "step": 165
2485
+ },
2486
+ {
2487
+ "epoch": 0.04153504519438276,
2488
+ "grad_norm": 137.5034637451172,
2489
+ "learning_rate": 4.15e-06,
2490
+ "logits/chosen": 0.5330204367637634,
2491
+ "logits/rejected": 0.5395705699920654,
2492
+ "logps/chosen": -767.4264526367188,
2493
+ "logps/rejected": -729.0516357421875,
2494
+ "loss": 3.6721,
2495
+ "rewards/accuracies": 0.75,
2496
+ "rewards/chosen": 0.8080553412437439,
2497
+ "rewards/margins": 0.718656063079834,
2498
+ "rewards/rejected": 0.08939927071332932,
2499
+ "step": 166
2500
+ },
2501
+ {
2502
+ "epoch": 0.04178525631001157,
2503
+ "grad_norm": 167.63526916503906,
2504
+ "learning_rate": 4.175e-06,
2505
+ "logits/chosen": 0.5876973867416382,
2506
+ "logits/rejected": 0.5106618404388428,
2507
+ "logps/chosen": -769.3453369140625,
2508
+ "logps/rejected": -762.6663208007812,
2509
+ "loss": 4.5432,
2510
+ "rewards/accuracies": 0.5625,
2511
+ "rewards/chosen": 0.6535158157348633,
2512
+ "rewards/margins": 0.46440136432647705,
2513
+ "rewards/rejected": 0.1891145408153534,
2514
+ "step": 167
2515
+ },
2516
+ {
2517
+ "epoch": 0.042035467425640384,
2518
+ "grad_norm": 153.6588592529297,
2519
+ "learning_rate": 4.2000000000000004e-06,
2520
+ "logits/chosen": 1.0562717914581299,
2521
+ "logits/rejected": 0.8941137790679932,
2522
+ "logps/chosen": -774.98779296875,
2523
+ "logps/rejected": -924.650146484375,
2524
+ "loss": 4.5292,
2525
+ "rewards/accuracies": 0.625,
2526
+ "rewards/chosen": 0.6449779272079468,
2527
+ "rewards/margins": 0.7065171599388123,
2528
+ "rewards/rejected": -0.061539266258478165,
2529
+ "step": 168
2530
+ },
2531
+ {
2532
+ "epoch": 0.042285678541269196,
2533
+ "grad_norm": 135.03091430664062,
2534
+ "learning_rate": 4.225e-06,
2535
+ "logits/chosen": 0.9676786065101624,
2536
+ "logits/rejected": 0.8078603744506836,
2537
+ "logps/chosen": -831.5836791992188,
2538
+ "logps/rejected": -804.6873168945312,
2539
+ "loss": 3.2118,
2540
+ "rewards/accuracies": 0.875,
2541
+ "rewards/chosen": 0.6791290044784546,
2542
+ "rewards/margins": 0.9013684391975403,
2543
+ "rewards/rejected": -0.2222394049167633,
2544
+ "step": 169
2545
+ },
2546
+ {
2547
+ "epoch": 0.04253588965689801,
2548
+ "grad_norm": 130.77609252929688,
2549
+ "learning_rate": 4.25e-06,
2550
+ "logits/chosen": 0.6716828346252441,
2551
+ "logits/rejected": 0.4985966086387634,
2552
+ "logps/chosen": -858.6065063476562,
2553
+ "logps/rejected": -982.9849853515625,
2554
+ "loss": 3.028,
2555
+ "rewards/accuracies": 0.875,
2556
+ "rewards/chosen": 0.8909362554550171,
2557
+ "rewards/margins": 1.2361092567443848,
2558
+ "rewards/rejected": -0.3451731204986572,
2559
+ "step": 170
2560
+ },
2561
+ {
2562
+ "epoch": 0.04278610077252682,
2563
+ "grad_norm": 141.84657287597656,
2564
+ "learning_rate": 4.2750000000000006e-06,
2565
+ "logits/chosen": 0.6962553858757019,
2566
+ "logits/rejected": 0.9387693405151367,
2567
+ "logps/chosen": -731.3770141601562,
2568
+ "logps/rejected": -636.809326171875,
2569
+ "loss": 4.3014,
2570
+ "rewards/accuracies": 0.75,
2571
+ "rewards/chosen": 0.762712836265564,
2572
+ "rewards/margins": 0.5948913097381592,
2573
+ "rewards/rejected": 0.16782140731811523,
2574
+ "step": 171
2575
+ },
2576
+ {
2577
+ "epoch": 0.04303631188815563,
2578
+ "grad_norm": 138.4955596923828,
2579
+ "learning_rate": 4.3e-06,
2580
+ "logits/chosen": 0.44833868741989136,
2581
+ "logits/rejected": 0.3760189712047577,
2582
+ "logps/chosen": -954.2752685546875,
2583
+ "logps/rejected": -976.9806518554688,
2584
+ "loss": 3.0478,
2585
+ "rewards/accuracies": 0.9375,
2586
+ "rewards/chosen": 1.0003539323806763,
2587
+ "rewards/margins": 1.2028504610061646,
2588
+ "rewards/rejected": -0.20249652862548828,
2589
+ "step": 172
2590
+ },
2591
+ {
2592
+ "epoch": 0.043286523003784445,
2593
+ "grad_norm": 125.87564849853516,
2594
+ "learning_rate": 4.325e-06,
2595
+ "logits/chosen": 0.14536720514297485,
2596
+ "logits/rejected": 0.34332674741744995,
2597
+ "logps/chosen": -845.4380493164062,
2598
+ "logps/rejected": -711.8458862304688,
2599
+ "loss": 3.6443,
2600
+ "rewards/accuracies": 0.75,
2601
+ "rewards/chosen": 0.7356263399124146,
2602
+ "rewards/margins": 0.8952075242996216,
2603
+ "rewards/rejected": -0.1595812439918518,
2604
+ "step": 173
2605
+ },
2606
+ {
2607
+ "epoch": 0.04353673411941326,
2608
+ "grad_norm": 153.5838623046875,
2609
+ "learning_rate": 4.350000000000001e-06,
2610
+ "logits/chosen": 0.7080106139183044,
2611
+ "logits/rejected": 0.696812629699707,
2612
+ "logps/chosen": -778.3582153320312,
2613
+ "logps/rejected": -852.6756591796875,
2614
+ "loss": 3.8358,
2615
+ "rewards/accuracies": 0.75,
2616
+ "rewards/chosen": 0.716163694858551,
2617
+ "rewards/margins": 0.6860113143920898,
2618
+ "rewards/rejected": 0.030152369290590286,
2619
+ "step": 174
2620
+ },
2621
+ {
2622
+ "epoch": 0.04378694523504207,
2623
+ "grad_norm": 148.90675354003906,
2624
+ "learning_rate": 4.3750000000000005e-06,
2625
+ "logits/chosen": 0.7510695457458496,
2626
+ "logits/rejected": 0.7831482887268066,
2627
+ "logps/chosen": -791.2515258789062,
2628
+ "logps/rejected": -890.5167846679688,
2629
+ "loss": 3.5614,
2630
+ "rewards/accuracies": 0.75,
2631
+ "rewards/chosen": 0.9652303457260132,
2632
+ "rewards/margins": 0.8300256133079529,
2633
+ "rewards/rejected": 0.13520470261573792,
2634
+ "step": 175
2635
+ },
2636
+ {
2637
+ "epoch": 0.044037156350670875,
2638
+ "grad_norm": 113.08384704589844,
2639
+ "learning_rate": 4.4e-06,
2640
+ "logits/chosen": 0.5231994986534119,
2641
+ "logits/rejected": 0.4796106517314911,
2642
+ "logps/chosen": -742.1634521484375,
2643
+ "logps/rejected": -823.2677612304688,
2644
+ "loss": 3.6957,
2645
+ "rewards/accuracies": 0.8125,
2646
+ "rewards/chosen": 0.830481231212616,
2647
+ "rewards/margins": 0.7274767160415649,
2648
+ "rewards/rejected": 0.10300463438034058,
2649
+ "step": 176
2650
+ },
2651
+ {
2652
+ "epoch": 0.04428736746629969,
2653
+ "grad_norm": 99.84241485595703,
2654
+ "learning_rate": 4.425e-06,
2655
+ "logits/chosen": 0.47837042808532715,
2656
+ "logits/rejected": 0.3609747290611267,
2657
+ "logps/chosen": -739.2203979492188,
2658
+ "logps/rejected": -757.5416259765625,
2659
+ "loss": 3.096,
2660
+ "rewards/accuracies": 0.6875,
2661
+ "rewards/chosen": 1.5671029090881348,
2662
+ "rewards/margins": 1.6435580253601074,
2663
+ "rewards/rejected": -0.07645538449287415,
2664
+ "step": 177
2665
+ },
2666
+ {
2667
+ "epoch": 0.0445375785819285,
2668
+ "grad_norm": 141.23953247070312,
2669
+ "learning_rate": 4.450000000000001e-06,
2670
+ "logits/chosen": 0.6331477761268616,
2671
+ "logits/rejected": 0.8105359077453613,
2672
+ "logps/chosen": -774.68017578125,
2673
+ "logps/rejected": -789.3485107421875,
2674
+ "loss": 4.5236,
2675
+ "rewards/accuracies": 0.6875,
2676
+ "rewards/chosen": 0.7711273431777954,
2677
+ "rewards/margins": 0.5388712286949158,
2678
+ "rewards/rejected": 0.23225615918636322,
2679
+ "step": 178
2680
+ },
2681
+ {
2682
+ "epoch": 0.04478778969755731,
2683
+ "grad_norm": 151.33343505859375,
2684
+ "learning_rate": 4.475e-06,
2685
+ "logits/chosen": 0.8690697550773621,
2686
+ "logits/rejected": 0.7319327592849731,
2687
+ "logps/chosen": -755.5984497070312,
2688
+ "logps/rejected": -873.7236938476562,
2689
+ "loss": 3.0347,
2690
+ "rewards/accuracies": 0.875,
2691
+ "rewards/chosen": 0.950842022895813,
2692
+ "rewards/margins": 1.3323326110839844,
2693
+ "rewards/rejected": -0.3814905285835266,
2694
+ "step": 179
2695
+ },
2696
+ {
2697
+ "epoch": 0.045038000813186124,
2698
+ "grad_norm": 122.43180084228516,
2699
+ "learning_rate": 4.5e-06,
2700
+ "logits/chosen": 0.41904574632644653,
2701
+ "logits/rejected": 0.33404892683029175,
2702
+ "logps/chosen": -989.1569213867188,
2703
+ "logps/rejected": -961.5740356445312,
2704
+ "loss": 2.3291,
2705
+ "rewards/accuracies": 0.9375,
2706
+ "rewards/chosen": 1.683314561843872,
2707
+ "rewards/margins": 1.5795354843139648,
2708
+ "rewards/rejected": 0.10377904027700424,
2709
+ "step": 180
2710
+ },
2711
+ {
2712
+ "epoch": 0.045288211928814937,
2713
+ "grad_norm": 120.82964324951172,
2714
+ "learning_rate": 4.525000000000001e-06,
2715
+ "logits/chosen": 0.17690542340278625,
2716
+ "logits/rejected": 0.21482543647289276,
2717
+ "logps/chosen": -854.7735595703125,
2718
+ "logps/rejected": -930.084228515625,
2719
+ "loss": 2.745,
2720
+ "rewards/accuracies": 0.875,
2721
+ "rewards/chosen": 1.029003620147705,
2722
+ "rewards/margins": 1.4290024042129517,
2723
+ "rewards/rejected": -0.39999890327453613,
2724
+ "step": 181
2725
+ },
2726
+ {
2727
+ "epoch": 0.04553842304444375,
2728
+ "grad_norm": 98.66107177734375,
2729
+ "learning_rate": 4.5500000000000005e-06,
2730
+ "logits/chosen": 0.4460292458534241,
2731
+ "logits/rejected": 0.509110689163208,
2732
+ "logps/chosen": -777.5671997070312,
2733
+ "logps/rejected": -822.9481811523438,
2734
+ "loss": 2.5145,
2735
+ "rewards/accuracies": 0.875,
2736
+ "rewards/chosen": 1.2376422882080078,
2737
+ "rewards/margins": 1.845048427581787,
2738
+ "rewards/rejected": -0.6074063181877136,
2739
+ "step": 182
2740
+ },
2741
+ {
2742
+ "epoch": 0.04578863416007256,
2743
+ "grad_norm": 200.034912109375,
2744
+ "learning_rate": 4.575e-06,
2745
+ "logits/chosen": 0.20393875241279602,
2746
+ "logits/rejected": 0.21906021237373352,
2747
+ "logps/chosen": -783.5596313476562,
2748
+ "logps/rejected": -711.1970825195312,
2749
+ "loss": 4.7052,
2750
+ "rewards/accuracies": 0.75,
2751
+ "rewards/chosen": 0.6918686032295227,
2752
+ "rewards/margins": 0.5280711054801941,
2753
+ "rewards/rejected": 0.1637975126504898,
2754
+ "step": 183
2755
+ },
2756
+ {
2757
+ "epoch": 0.04603884527570137,
2758
+ "grad_norm": 141.71707153320312,
2759
+ "learning_rate": 4.600000000000001e-06,
2760
+ "logits/chosen": 0.7268654704093933,
2761
+ "logits/rejected": 0.5893440246582031,
2762
+ "logps/chosen": -908.1588134765625,
2763
+ "logps/rejected": -839.3931884765625,
2764
+ "loss": 4.0667,
2765
+ "rewards/accuracies": 0.6875,
2766
+ "rewards/chosen": 1.4157216548919678,
2767
+ "rewards/margins": 0.8883066773414612,
2768
+ "rewards/rejected": 0.5274151563644409,
2769
+ "step": 184
2770
+ },
2771
+ {
2772
+ "epoch": 0.046289056391330186,
2773
+ "grad_norm": 97.34073638916016,
2774
+ "learning_rate": 4.625000000000001e-06,
2775
+ "logits/chosen": 0.6829476952552795,
2776
+ "logits/rejected": 0.8145304322242737,
2777
+ "logps/chosen": -716.8839111328125,
2778
+ "logps/rejected": -681.6070556640625,
2779
+ "loss": 2.9681,
2780
+ "rewards/accuracies": 0.9375,
2781
+ "rewards/chosen": 0.7450221180915833,
2782
+ "rewards/margins": 1.0474132299423218,
2783
+ "rewards/rejected": -0.30239105224609375,
2784
+ "step": 185
2785
+ },
2786
+ {
2787
+ "epoch": 0.046539267506959,
2788
+ "grad_norm": 193.9061279296875,
2789
+ "learning_rate": 4.65e-06,
2790
+ "logits/chosen": 0.6067227721214294,
2791
+ "logits/rejected": 0.533854603767395,
2792
+ "logps/chosen": -710.9017944335938,
2793
+ "logps/rejected": -903.33251953125,
2794
+ "loss": 5.8841,
2795
+ "rewards/accuracies": 0.5625,
2796
+ "rewards/chosen": 0.7944581508636475,
2797
+ "rewards/margins": 0.6155319213867188,
2798
+ "rewards/rejected": 0.17892615497112274,
2799
+ "step": 186
2800
+ },
2801
+ {
2802
+ "epoch": 0.04678947862258781,
2803
+ "grad_norm": 139.4214630126953,
2804
+ "learning_rate": 4.675000000000001e-06,
2805
+ "logits/chosen": 0.47161051630973816,
2806
+ "logits/rejected": 0.5310673713684082,
2807
+ "logps/chosen": -758.6500244140625,
2808
+ "logps/rejected": -694.9664306640625,
2809
+ "loss": 3.156,
2810
+ "rewards/accuracies": 0.75,
2811
+ "rewards/chosen": 1.4704225063323975,
2812
+ "rewards/margins": 2.380772113800049,
2813
+ "rewards/rejected": -0.9103498458862305,
2814
+ "step": 187
2815
+ },
2816
+ {
2817
+ "epoch": 0.04703968973821662,
2818
+ "grad_norm": 148.84498596191406,
2819
+ "learning_rate": 4.7e-06,
2820
+ "logits/chosen": 0.7463999390602112,
2821
+ "logits/rejected": 0.6421334743499756,
2822
+ "logps/chosen": -696.7813110351562,
2823
+ "logps/rejected": -788.3897094726562,
2824
+ "loss": 4.4071,
2825
+ "rewards/accuracies": 0.6875,
2826
+ "rewards/chosen": 0.5864818692207336,
2827
+ "rewards/margins": 0.9901420474052429,
2828
+ "rewards/rejected": -0.40365999937057495,
2829
+ "step": 188
2830
+ },
2831
+ {
2832
+ "epoch": 0.047289900853845435,
2833
+ "grad_norm": 96.11073303222656,
2834
+ "learning_rate": 4.7250000000000005e-06,
2835
+ "logits/chosen": 1.0857046842575073,
2836
+ "logits/rejected": 0.8691186308860779,
2837
+ "logps/chosen": -646.8231201171875,
2838
+ "logps/rejected": -615.858642578125,
2839
+ "loss": 2.8434,
2840
+ "rewards/accuracies": 0.875,
2841
+ "rewards/chosen": 0.9308801293373108,
2842
+ "rewards/margins": 1.6092588901519775,
2843
+ "rewards/rejected": -0.6783787608146667,
2844
+ "step": 189
2845
+ },
2846
+ {
2847
+ "epoch": 0.04754011196947425,
2848
+ "grad_norm": 171.08673095703125,
2849
+ "learning_rate": 4.75e-06,
2850
+ "logits/chosen": 0.7702385783195496,
2851
+ "logits/rejected": 0.643980860710144,
2852
+ "logps/chosen": -950.1005249023438,
2853
+ "logps/rejected": -999.3270874023438,
2854
+ "loss": 3.6033,
2855
+ "rewards/accuracies": 0.8125,
2856
+ "rewards/chosen": 1.6129151582717896,
2857
+ "rewards/margins": 1.880535364151001,
2858
+ "rewards/rejected": -0.26762041449546814,
2859
+ "step": 190
2860
+ },
2861
+ {
2862
+ "epoch": 0.04779032308510306,
2863
+ "grad_norm": 243.4348602294922,
2864
+ "learning_rate": 4.775e-06,
2865
+ "logits/chosen": 0.26067548990249634,
2866
+ "logits/rejected": 0.34642136096954346,
2867
+ "logps/chosen": -981.1875610351562,
2868
+ "logps/rejected": -782.4407958984375,
2869
+ "loss": 5.5213,
2870
+ "rewards/accuracies": 0.75,
2871
+ "rewards/chosen": 0.8981813788414001,
2872
+ "rewards/margins": 0.879020094871521,
2873
+ "rewards/rejected": 0.019161269068717957,
2874
+ "step": 191
2875
+ },
2876
+ {
2877
+ "epoch": 0.048040534200731864,
2878
+ "grad_norm": 173.99644470214844,
2879
+ "learning_rate": 4.800000000000001e-06,
2880
+ "logits/chosen": 0.672372579574585,
2881
+ "logits/rejected": 0.7054111957550049,
2882
+ "logps/chosen": -784.5167846679688,
2883
+ "logps/rejected": -712.2625732421875,
2884
+ "loss": 4.8373,
2885
+ "rewards/accuracies": 0.625,
2886
+ "rewards/chosen": 0.7209684252738953,
2887
+ "rewards/margins": 0.6813942193984985,
2888
+ "rewards/rejected": 0.03957414627075195,
2889
+ "step": 192
2890
+ },
2891
+ {
2892
+ "epoch": 0.04829074531636068,
2893
+ "grad_norm": 142.26370239257812,
2894
+ "learning_rate": 4.825e-06,
2895
+ "logits/chosen": 0.4435230791568756,
2896
+ "logits/rejected": 0.3302009105682373,
2897
+ "logps/chosen": -968.1591186523438,
2898
+ "logps/rejected": -1088.6114501953125,
2899
+ "loss": 3.1433,
2900
+ "rewards/accuracies": 0.875,
2901
+ "rewards/chosen": 1.4998180866241455,
2902
+ "rewards/margins": 1.830888032913208,
2903
+ "rewards/rejected": -0.3310699760913849,
2904
+ "step": 193
2905
+ },
2906
+ {
2907
+ "epoch": 0.04854095643198949,
2908
+ "grad_norm": 175.26205444335938,
2909
+ "learning_rate": 4.85e-06,
2910
+ "logits/chosen": 0.4360215365886688,
2911
+ "logits/rejected": 0.4668835401535034,
2912
+ "logps/chosen": -1311.623291015625,
2913
+ "logps/rejected": -1099.365478515625,
2914
+ "loss": 3.7134,
2915
+ "rewards/accuracies": 0.8125,
2916
+ "rewards/chosen": 1.3552098274230957,
2917
+ "rewards/margins": 1.2448309659957886,
2918
+ "rewards/rejected": 0.11037885397672653,
2919
+ "step": 194
2920
+ },
2921
+ {
2922
+ "epoch": 0.0487911675476183,
2923
+ "grad_norm": 162.4991455078125,
2924
+ "learning_rate": 4.875e-06,
2925
+ "logits/chosen": 0.43502160906791687,
2926
+ "logits/rejected": 0.3366948366165161,
2927
+ "logps/chosen": -917.325439453125,
2928
+ "logps/rejected": -1000.3974609375,
2929
+ "loss": 3.5885,
2930
+ "rewards/accuracies": 0.6875,
2931
+ "rewards/chosen": 1.0496270656585693,
2932
+ "rewards/margins": 1.293393611907959,
2933
+ "rewards/rejected": -0.24376648664474487,
2934
+ "step": 195
2935
+ },
2936
+ {
2937
+ "epoch": 0.049041378663247114,
2938
+ "grad_norm": 140.2455291748047,
2939
+ "learning_rate": 4.9000000000000005e-06,
2940
+ "logits/chosen": 0.7639466524124146,
2941
+ "logits/rejected": 0.8074101805686951,
2942
+ "logps/chosen": -978.490966796875,
2943
+ "logps/rejected": -858.6104736328125,
2944
+ "loss": 3.4015,
2945
+ "rewards/accuracies": 0.75,
2946
+ "rewards/chosen": 1.7361470460891724,
2947
+ "rewards/margins": 2.0762417316436768,
2948
+ "rewards/rejected": -0.3400946259498596,
2949
+ "step": 196
2950
+ },
2951
+ {
2952
+ "epoch": 0.049291589778875926,
2953
+ "grad_norm": 133.20802307128906,
2954
+ "learning_rate": 4.925e-06,
2955
+ "logits/chosen": 0.5749949812889099,
2956
+ "logits/rejected": 0.297177255153656,
2957
+ "logps/chosen": -712.5463256835938,
2958
+ "logps/rejected": -823.659912109375,
2959
+ "loss": 4.2319,
2960
+ "rewards/accuracies": 0.75,
2961
+ "rewards/chosen": 0.6333484649658203,
2962
+ "rewards/margins": 1.0953937768936157,
2963
+ "rewards/rejected": -0.462045282125473,
2964
+ "step": 197
2965
+ },
2966
+ {
2967
+ "epoch": 0.04954180089450474,
2968
+ "grad_norm": 85.10523986816406,
2969
+ "learning_rate": 4.95e-06,
2970
+ "logits/chosen": 0.45457780361175537,
2971
+ "logits/rejected": 0.7557879686355591,
2972
+ "logps/chosen": -828.2936401367188,
2973
+ "logps/rejected": -675.2957763671875,
2974
+ "loss": 2.1404,
2975
+ "rewards/accuracies": 0.9375,
2976
+ "rewards/chosen": 1.1196131706237793,
2977
+ "rewards/margins": 1.659280776977539,
2978
+ "rewards/rejected": -0.5396679043769836,
2979
+ "step": 198
2980
+ },
2981
+ {
2982
+ "epoch": 0.04979201201013355,
2983
+ "grad_norm": 96.11859130859375,
2984
+ "learning_rate": 4.975000000000001e-06,
2985
+ "logits/chosen": 0.8775396943092346,
2986
+ "logits/rejected": 0.8114781379699707,
2987
+ "logps/chosen": -720.8642578125,
2988
+ "logps/rejected": -787.377197265625,
2989
+ "loss": 1.9458,
2990
+ "rewards/accuracies": 0.875,
2991
+ "rewards/chosen": 1.2211010456085205,
2992
+ "rewards/margins": 1.7165383100509644,
2993
+ "rewards/rejected": -0.4954374134540558,
2994
+ "step": 199
2995
+ },
2996
+ {
2997
+ "epoch": 0.05004222312576236,
2998
+ "grad_norm": 161.368408203125,
2999
+ "learning_rate": 5e-06,
3000
+ "logits/chosen": 0.3539942502975464,
3001
+ "logits/rejected": 0.3524795472621918,
3002
+ "logps/chosen": -840.6842651367188,
3003
+ "logps/rejected": -821.8270874023438,
3004
+ "loss": 3.4471,
3005
+ "rewards/accuracies": 0.875,
3006
+ "rewards/chosen": 0.9537111520767212,
3007
+ "rewards/margins": 1.3869084119796753,
3008
+ "rewards/rejected": -0.4331972002983093,
3009
+ "step": 200
3010
  }
3011
  ],
3012
  "logging_steps": 1,
 
3021
  "should_evaluate": false,
3022
  "should_log": false,
3023
  "should_save": true,
3024
+ "should_training_stop": true
3025
  },
3026
  "attributes": {}
3027
  }