Upload checkpoint-1500

architecture.py

@@ -0,0 +1,238 @@
+# --- START OF FILE architectureV3.py ---
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from transformers import Phi3Config, Phi3ForCausalLM
+from transformers.modeling_outputs import CausalLMOutputWithPast
+from typing import Optional, Dict, Tuple
+from dataclasses import dataclass
+
+@dataclass
+class CausalLMOutputWithLTM(CausalLMOutputWithPast):
+    loss: Optional[torch.FloatTensor] = None
+    logits: torch.FloatTensor = None
+    past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None
+    hidden_states: Optional[Tuple[torch.FloatTensor]] = None
+    attentions: Optional[Tuple[torch.FloatTensor]] = None
+    ltm_state: Optional[torch.Tensor] = None # The returned LTM state
+
+# --- BUILDING BLOCK 1: Hierarchical VectorMemoryHead (Stateless) ---
+class VectorMemoryHead(nn.Module):
+    def __init__(self, hidden_dim: int, num_memory_slots: int, num_heads: int, ff_dim: int,
+                 num_long_term_memory_slots: int = 0,
+                 device=None, dtype=None):
+        super().__init__()
+        self.hidden_dim = hidden_dim
+        self.num_memory_slots = num_memory_slots
+        self.num_long_term_memory_slots = num_long_term_memory_slots
+        self.use_long_term_memory = self.num_long_term_memory_slots > 0
+
+        encoder_layer = nn.TransformerEncoderLayer(
+            d_model=hidden_dim, nhead=num_heads, dim_feedforward=ff_dim, dropout=0.1, batch_first=True,
+            device=device, dtype=dtype)
+        self.encoder = nn.TransformerEncoder(encoder_layer, num_layers=1)
+        self.memory_queries = nn.Parameter(torch.randn(1, num_memory_slots, hidden_dim, device=device, dtype=dtype))
+        self.memory_attention = nn.MultiheadAttention(
+            embed_dim=hidden_dim, num_heads=num_heads, dropout=0.1, batch_first=True, device=device, dtype=dtype)
+        self.memory_layernorm = nn.LayerNorm(hidden_dim, device=device, dtype=dtype)
+        self.decoder_attention = nn.MultiheadAttention(
+            embed_dim=hidden_dim, num_heads=num_heads, dropout=0.1, batch_first=True, device=device, dtype=dtype)
+        self.decoder_layernorm = nn.LayerNorm(hidden_dim, device=device, dtype=dtype)
+        self.decoder_ffn = nn.Sequential(
+            nn.Linear(hidden_dim, ff_dim, device=device, dtype=dtype), nn.ReLU(),
+            nn.Linear(ff_dim, hidden_dim, device=device, dtype=dtype))
+
+        if self.use_long_term_memory:
+            self.memory_update_gate = nn.Sequential(
+                nn.Linear(hidden_dim, hidden_dim, device=device, dtype=dtype), nn.Sigmoid())
+            self.ltm_retrieval_attention = nn.MultiheadAttention(
+                embed_dim=hidden_dim, num_heads=num_heads, dropout=0.1, batch_first=True, device=device, dtype=dtype)
+
+    def forward(self, memory_input_sequence: torch.Tensor,
+                long_term_memory: Optional[torch.Tensor] = None) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]:
+        batch_size = memory_input_sequence.shape[0]
+        new_ltm_state = long_term_memory
+        queries = self.memory_queries.expand(batch_size, -1, -1)
+        encoded_vectors = self.encoder(memory_input_sequence)
+        compressed_memory, _ = self.memory_attention(query=queries, key=encoded_vectors, value=encoded_vectors)
+        compressed_memory = self.memory_layernorm(compressed_memory + queries)
+        final_memory_context = compressed_memory
+
+        if self.use_long_term_memory and long_term_memory is not None:
+            retrieved_ltm, _ = self.ltm_retrieval_attention(
+                query=compressed_memory, key=long_term_memory, value=long_term_memory)
+            l1_summary = compressed_memory.mean(dim=1, keepdim=True)
+            update_gate = self.memory_update_gate(l1_summary)
+            new_ltm_state = (update_gate * l1_summary) + ((1 - update_gate) * long_term_memory)
+            final_memory_context = final_memory_context + retrieved_ltm
+
+        reconstructed, _ = self.decoder_attention(query=encoded_vectors, key=final_memory_context, value=final_memory_context)
+        reconstructed_vectors = self.decoder_layernorm(reconstructed + encoded_vectors)
+        reconstructed_vectors = self.decoder_ffn(reconstructed_vectors)
+        return compressed_memory, reconstructed_vectors, new_ltm_state
+
+# --- BUILDING BLOCK 2: ReflectiveMemoryLayer ---
+class ReflectiveMemoryLayer(nn.Module):
+    def __init__(self, original_layer: nn.Linear, global_input_dim: int,
+                 memory_dim: int, num_memory_slots: int, memory_num_heads: int,
+                 global_state_storage: Dict):
+        super().__init__()
+        self.input_dim, self.output_dim = original_layer.in_features, original_layer.out_features
+        self.memory_dim, self.global_state_storage = memory_dim, global_state_storage
+        self.linear = original_layer # Keep the original linear layer frozen
+        self.refinement_passes: int = 2
+        device, dtype = self.linear.weight.device, self.linear.weight.dtype
+
+        self.local_state_proj = nn.Linear(self.input_dim, memory_dim, device=device, dtype=dtype)
+        self.global_state_proj = nn.Linear(global_input_dim, memory_dim, device=device, dtype=dtype)
+        self.memory_head = VectorMemoryHead(
+            hidden_dim=memory_dim, num_memory_slots=num_memory_slots, num_heads=memory_num_heads,
+            ff_dim=memory_dim * 2, num_long_term_memory_slots=32, device=device, dtype=dtype)
+        self.thought_critique_attention = nn.MultiheadAttention(
+            embed_dim=memory_dim, num_heads=memory_num_heads, dropout=0.1, batch_first=True, device=device, dtype=dtype)
+        self.thought_layernorm = nn.LayerNorm(memory_dim, device=device, dtype=dtype)
+        self.correction_head = nn.Linear(memory_dim, 2 * self.output_dim, device=device, dtype=dtype)
+
+        self.last_corrected_activation, self.last_additive_correction = None, None
+        self.last_memory_input, self.last_reconstructed_from_memory = None, None
+
+    def forward(self, x: torch.Tensor):
+        base_output = self.linear(x)
+        if 'embeds' not in self.global_state_storage: 
+            return base_output
+
+        global_embeds = self.global_state_storage['embeds']
+        if global_embeds.shape[1] != x.shape[1]: 
+            global_embeds = global_embeds[:, -x.shape[1]:, :]
+        B, S, _ = x.shape
+        
+        # CRITICAL FIX: Always detach LTM state to prevent backward through previous graphs
+        ltm_state = self.global_state_storage.get('ltm', None)
+        if ltm_state is not None:
+            ltm_state = ltm_state.detach()
+        
+        proj_local = self.local_state_proj(x)
+        proj_global = self.global_state_proj(global_embeds)
+        memory_input = torch.stack([proj_global, proj_local], dim=2)
+        memory_input_flat = memory_input.view(B * S, 2, self.memory_dim)
+
+        # *** FIX: Expand LTM state to match the flattened token dimension (B*S) ***
+        ltm_state_expanded = None
+        if ltm_state is not None:
+            ltm_state_expanded = ltm_state.repeat_interleave(S, dim=0)
+
+        compressed_mem_flat, recon_flat, new_ltm_state_expanded = self.memory_head(memory_input_flat, ltm_state_expanded)
+        
+        # *** FIX: Condense updated LTM state back to batch dimension B ***
+        if new_ltm_state_expanded is not None:
+             num_ltm_slots = new_ltm_state_expanded.shape[1]
+             new_ltm_condensed = new_ltm_state_expanded.view(B, S, num_ltm_slots, self.memory_dim).mean(dim=1)
+             # CRITICAL FIX: Always detach when storing in global state
+             self.global_state_storage['ltm'] = new_ltm_condensed.detach()
+
+        initial_thought = compressed_mem_flat.mean(dim=1).view(B, S, self.memory_dim)
+        current_thought = initial_thought
+        if not self.training and self.refinement_passes > 0:
+            with torch.no_grad():
+                for _ in range(self.refinement_passes):
+                    current_thought_flat = current_thought.view(B * S, 1, self.memory_dim)
+                    internal_ref, _ = self.memory_head.decoder_attention(
+                        query=current_thought_flat, key=compressed_mem_flat, value=compressed_mem_flat)
+                    external_crit, _ = self.thought_critique_attention(
+                        query=current_thought_flat, key=memory_input_flat, value=memory_input_flat)
+                    refined_thought = current_thought + internal_ref.view(B,S,-1) + external_crit.view(B,S,-1)
+                    current_thought = self.thought_layernorm(refined_thought)
+        
+        thought_for_correction = current_thought if not self.training else initial_thought
+        raw_correction = self.correction_head(thought_for_correction)
+        gate, value = torch.chunk(raw_correction, 2, dim=-1)
+        final_activation = base_output * torch.sigmoid(gate.to(x.dtype)) + value.to(x.dtype)
+        
+        if self.training:
+            # CRITICAL FIX: Detach tensors stored for debugging/analysis
+            self.last_corrected_activation = final_activation.detach()
+            self.last_additive_correction = value.detach()
+            self.last_memory_input = memory_input.detach()
+            self.last_reconstructed_from_memory = recon_flat.view(B, S, 2, self.memory_dim).detach()
+        return final_activation
+
+# --- BUILDING BLOCK 3: The Full Custom Model with State Management ---
+class Phi3WithReflectiveMemoryForCausalLM(Phi3ForCausalLM):
+    def __init__(self, config):
+        super().__init__(config)
+        self.global_state_storage = {}
+        self.target_layer_path = "model.layers.15.mlp.gate_up_proj"
+        self.memory_dim, self.num_long_term_memory_slots = 256, 32
+
+        # CRITICAL FIX: Ensure embeddings are detached when stored
+        def embedding_hook(module, input, output):
+            self.global_state_storage['embeds'] = output.detach()
+        
+        self.model.embed_tokens.register_forward_hook(embedding_hook)
+
+        try:
+            original_layer = self.get_submodule(self.target_layer_path)
+            custom_layer = ReflectiveMemoryLayer(
+                original_layer=original_layer, global_input_dim=config.hidden_size,
+                memory_dim=self.memory_dim, num_memory_slots=32, memory_num_heads=16,
+                global_state_storage=self.global_state_storage)
+            parent_path = ".".join(self.target_layer_path.split('.')[:-1])
+            setattr(self.get_submodule(parent_path), self.target_layer_path.split('.')[-1], custom_layer)
+            print(f"Successfully replaced '{self.target_layer_path}' with ReflectiveMemoryLayer.")
+        except AttributeError:
+            print(f"Could not find target layer '{self.target_layer_path}'. Model remains unmodified.")
+
+    def _init_ltm_state(self, batch_size, device, dtype):
+        # *** FIX: Initialize LTM state per item in the batch (no hardcoded hack) ***
+        return torch.zeros(
+            batch_size, self.num_long_term_memory_slots, self.memory_dim, device=device, dtype=dtype)
+
+    def forward(self, input_ids: torch.LongTensor = None, attention_mask: Optional[torch.Tensor] = None,
+                position_ids: Optional[torch.LongTensor] = None, past_key_values: Optional[list[torch.FloatTensor]] = None,
+                inputs_embeds: Optional[torch.FloatTensor] = None, labels: Optional[torch.LongTensor] = None,
+                use_cache: Optional[bool] = None, output_attentions: Optional[bool] = None,
+                output_hidden_states: Optional[bool] = None, return_dict: Optional[bool] = None,
+                ltm_state: Optional[torch.Tensor] = None):
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        
+        # CRITICAL FIX: Don't clear global state storage completely, just reset embeds
+        # This prevents losing LTM state continuity
+        if 'embeds' in self.global_state_storage:
+            del self.global_state_storage['embeds']
+        
+        # *** FIX: Initialize LTM state if not provided, for both training and first step of inference ***
+        if ltm_state is None:
+             batch_size = input_ids.shape[0] if input_ids is not None else inputs_embeds.shape[0]
+             ltm_state = self._init_ltm_state(batch_size, self.device, self.dtype)
+        
+        # CRITICAL FIX: Ensure LTM state is detached when stored
+        self.global_state_storage['ltm'] = ltm_state.detach() if ltm_state is not None else None
+
+        outputs = self.model(
+            input_ids=input_ids, attention_mask=attention_mask, position_ids=position_ids,
+            past_key_values=past_key_values, inputs_embeds=inputs_embeds, use_cache=use_cache,
+            output_attentions=output_attentions, output_hidden_states=output_hidden_states, return_dict=return_dict)
+        
+        hidden_states = outputs[0]
+        logits = self.lm_head(hidden_states).float()
+
+        loss = None
+        if labels is not None:
+            loss_fct = nn.CrossEntropyLoss()
+            loss = loss_fct(logits[..., :-1, :].contiguous().view(-1, self.config.vocab_size),
+                            labels[..., 1:].contiguous().view(-1))
+            # Note: Auxiliary losses from main.py are calculated outside the model forward pass.
+
+        # CRITICAL FIX: Ensure returned LTM state is detached
+        new_ltm_state = self.global_state_storage.get('ltm', None)
+        if new_ltm_state is not None:
+            new_ltm_state = new_ltm_state.detach()
+        
+        if not return_dict:
+            output = (logits,) + outputs[1:] + (new_ltm_state,)
+            return (loss,) + output if loss is not None else output
+
+        return CausalLMOutputWithLTM(
+            loss=loss, logits=logits, past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states, attentions=outputs.attentions, ltm_state=new_ltm_state)

config.json

@@ -0,0 +1,37 @@
+{
+  "_name_or_path": "./phi3_mini_with_vectormemory_layer_v12_final",
+  "architectures": [
+    "Phi3WithReflectiveMemoryForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoModel": "architecture.Phi3WithReflectiveMemoryForCausalLM",
+    "AutoModelForCausalLM": "architecture.Phi3WithReflectiveMemoryForCausalLM"
+  },
+  "bos_token_id": 100257,
+  "embd_pdrop": 0.0,
+  "eos_token_id": 100257,
+  "hidden_act": "silu",
+  "hidden_size": 5120,
+  "initializer_range": 0.02,
+  "intermediate_size": 17920,
+  "max_position_embeddings": 16384,
+  "model_type": "phi3",
+  "num_attention_heads": 40,
+  "num_hidden_layers": 40,
+  "num_key_value_heads": 10,
+  "original_max_position_embeddings": 16384,
+  "pad_token_id": 100257,
+  "partial_rotary_factor": 1.0,
+  "resid_pdrop": 0.0,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 250000,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.49.0",
+  "use_cache": false,
+  "vocab_size": 100352
+}

generation_config.json

@@ -0,0 +1,9 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 100257,
+  "eos_token_id": [
+    100257,
+    100265
+  ],
+  "transformers_version": "4.49.0"
+}

model-00001-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e055a52dea9e514d5c3b1245ced6e6b0d04c9b316efd02946382844cbd0275b
+size 4933656472

model-00002-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ade161164aae03943fb17b68fba9390cbfe8c181c5bb8814a45006d6721b29ea
+size 4954690712

model-00003-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7e9f50b395c8d4362ddb88860eb9a3bd0d9d1944771527be41babcc5bde030e8
+size 4948172048

model-00004-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2f2b1c5f9fe700050099fb926413edc2d85b24ae9c317d7a1fc7a7ec2ce4491
+size 4771169120

model-00005-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:12b50575d59e89524f9ef42e17c0e80dbc6c89da906ccac5087c0ee0bba535a9
+size 4771169120

model-00006-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7bc116c49c0acd6edbf34ca52ab0f0a8a7d7fcfff62af2dde7cdcf3bb91fe2ba
+size 4954648144

model-00007-of-00007.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:289b2d9f9b099241b1268e2f95a4903e18439fd473149d38bde506612a377d78
+size 1394657832

model.safetensors.index.json

@@ -0,0 +1,297 @@
+{
+  "metadata": {
+    "total_size": 30728128000
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00007-of-00007.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.mlp.gate_up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.0.self_attn.qkv_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.mlp.gate_up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.1.self_attn.qkv_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.10.mlp.gate_up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.10.self_attn.qkv_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.11.mlp.gate_up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.11.self_attn.qkv_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.12.mlp.gate_up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.12.self_attn.qkv_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.mlp.gate_up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.13.self_attn.qkv_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.mlp.gate_up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.14.self_attn.qkv_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.correction_head.bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.correction_head.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.global_state_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.global_state_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.linear.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.local_state_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.local_state_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.decoder_attention.in_proj_bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.decoder_attention.in_proj_weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.decoder_attention.out_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.decoder_attention.out_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.decoder_ffn.0.bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.decoder_ffn.0.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.decoder_ffn.2.bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.decoder_ffn.2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.decoder_layernorm.bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.decoder_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.encoder.layers.0.linear1.bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.encoder.layers.0.linear1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.encoder.layers.0.linear2.bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.encoder.layers.0.linear2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.encoder.layers.0.norm1.bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.encoder.layers.0.norm1.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.encoder.layers.0.norm2.bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.encoder.layers.0.norm2.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.encoder.layers.0.self_attn.in_proj_bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.encoder.layers.0.self_attn.in_proj_weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.encoder.layers.0.self_attn.out_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.encoder.layers.0.self_attn.out_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.ltm_retrieval_attention.in_proj_bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.ltm_retrieval_attention.in_proj_weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.ltm_retrieval_attention.out_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.ltm_retrieval_attention.out_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.memory_attention.in_proj_bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.memory_attention.in_proj_weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.memory_attention.out_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.memory_attention.out_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.memory_layernorm.bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.memory_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.memory_queries": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.memory_update_gate.0.bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.memory_head.memory_update_gate.0.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.thought_critique_attention.in_proj_bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.thought_critique_attention.in_proj_weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.thought_critique_attention.out_proj.bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.thought_critique_attention.out_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.thought_layernorm.bias": "model-00003-of-00007.safetensors",
+    "model.layers.15.mlp.gate_up_proj.thought_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.15.self_attn.qkv_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.16.mlp.gate_up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.16.self_attn.qkv_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.17.mlp.gate_up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.17.self_attn.qkv_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.18.mlp.gate_up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.18.self_attn.qkv_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.19.mlp.gate_up_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.19.self_attn.qkv_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.mlp.gate_up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.2.self_attn.qkv_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.mlp.gate_up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.20.self_attn.qkv_proj.weight": "model-00003-of-00007.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.21.mlp.gate_up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.21.self_attn.qkv_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.22.mlp.gate_up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.22.self_attn.qkv_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.23.mlp.gate_up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.23.self_attn.qkv_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.24.mlp.gate_up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.24.self_attn.qkv_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.25.mlp.gate_up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.25.self_attn.qkv_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.26.mlp.gate_up_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.26.self_attn.qkv_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.27.mlp.gate_up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.27.self_attn.qkv_proj.weight": "model-00004-of-00007.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.28.mlp.gate_up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.28.self_attn.qkv_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.29.mlp.gate_up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.29.self_attn.qkv_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.mlp.gate_up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.3.self_attn.qkv_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.30.mlp.gate_up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.30.self_attn.qkv_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.31.mlp.gate_up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.31.self_attn.qkv_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.32.mlp.gate_up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.32.self_attn.qkv_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.33.mlp.gate_up_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.33.self_attn.qkv_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.34.mlp.gate_up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.34.self_attn.qkv_proj.weight": "model-00005-of-00007.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.35.mlp.gate_up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.35.self_attn.qkv_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.36.mlp.gate_up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.36.self_attn.qkv_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.37.mlp.gate_up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.37.self_attn.qkv_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.38.mlp.gate_up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.38.self_attn.qkv_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
+    "model.layers.39.mlp.gate_up_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.39.self_attn.qkv_proj.weight": "model-00006-of-00007.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.4.mlp.gate_up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.4.self_attn.qkv_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.mlp.gate_up_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.5.self_attn.qkv_proj.weight": "model-00001-of-00007.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.mlp.gate_up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.6.self_attn.qkv_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.mlp.gate_up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.7.self_attn.qkv_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.mlp.gate_up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.8.self_attn.qkv_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.mlp.gate_up_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
+    "model.layers.9.self_attn.qkv_proj.weight": "model-00002-of-00007.safetensors",
+    "model.norm.weight": "model-00007-of-00007.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": true,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": true,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": true,
+    "normalized": false,
+    "rstrip": true,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,784 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "100256": {
+      "content": "<|dummy_0|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100257": {
+      "content": "<|endoftext|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100258": {
+      "content": "<|fim_prefix|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100259": {
+      "content": "<|fim_middle|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100260": {
+      "content": "<|fim_suffix|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100261": {
+      "content": "<|dummy_1|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100262": {
+      "content": "<|dummy_2|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100263": {
+      "content": "<|dummy_3|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100264": {
+      "content": "<|im_start|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100265": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "100266": {
+      "content": "<|im_sep|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100267": {
+      "content": "<|dummy_4|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100268": {
+      "content": "<|dummy_5|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100269": {
+      "content": "<|dummy_6|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100270": {
+      "content": "<|dummy_7|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100271": {
+      "content": "<|dummy_8|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100272": {
+      "content": "<|dummy_9|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100273": {
+      "content": "<|dummy_10|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100274": {
+      "content": "<|dummy_11|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100275": {
+      "content": "<|dummy_12|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100276": {
+      "content": "<|endofprompt|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100277": {
+      "content": "<|dummy_13|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100278": {
+      "content": "<|dummy_14|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100279": {
+      "content": "<|dummy_15|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100280": {
+      "content": "<|dummy_16|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100281": {
+      "content": "<|dummy_17|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100282": {
+      "content": "<|dummy_18|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100283": {
+      "content": "<|dummy_19|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100284": {
+      "content": "<|dummy_20|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100285": {
+      "content": "<|dummy_21|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100286": {
+      "content": "<|dummy_22|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100287": {
+      "content": "<|dummy_23|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100288": {
+      "content": "<|dummy_24|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100289": {
+      "content": "<|dummy_25|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100290": {
+      "content": "<|dummy_26|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100291": {
+      "content": "<|dummy_27|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100292": {
+      "content": "<|dummy_28|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100293": {
+      "content": "<|dummy_29|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100294": {
+      "content": "<|dummy_30|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100295": {
+      "content": "<|dummy_31|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100296": {
+      "content": "<|dummy_32|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100297": {
+      "content": "<|dummy_33|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100298": {
+      "content": "<|dummy_34|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100299": {
+      "content": "<|dummy_35|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100300": {
+      "content": "<|dummy_36|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100301": {
+      "content": "<|dummy_37|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100302": {
+      "content": "<|dummy_38|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100303": {
+      "content": "<|dummy_39|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100304": {
+      "content": "<|dummy_40|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100305": {
+      "content": "<|dummy_41|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100306": {
+      "content": "<|dummy_42|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100307": {
+      "content": "<|dummy_43|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100308": {
+      "content": "<|dummy_44|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100309": {
+      "content": "<|dummy_45|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100310": {
+      "content": "<|dummy_46|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100311": {
+      "content": "<|dummy_47|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100312": {
+      "content": "<|dummy_48|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100313": {
+      "content": "<|dummy_49|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100314": {
+      "content": "<|dummy_50|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100315": {
+      "content": "<|dummy_51|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100316": {
+      "content": "<|dummy_52|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100317": {
+      "content": "<|dummy_53|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100318": {
+      "content": "<|dummy_54|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100319": {
+      "content": "<|dummy_55|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100320": {
+      "content": "<|dummy_56|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100321": {
+      "content": "<|dummy_57|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100322": {
+      "content": "<|dummy_58|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100323": {
+      "content": "<|dummy_59|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100324": {
+      "content": "<|dummy_60|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100325": {
+      "content": "<|dummy_61|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100326": {
+      "content": "<|dummy_62|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100327": {
+      "content": "<|dummy_63|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100328": {
+      "content": "<|dummy_64|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100329": {
+      "content": "<|dummy_65|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100330": {
+      "content": "<|dummy_66|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100331": {
+      "content": "<|dummy_67|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100332": {
+      "content": "<|dummy_68|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100333": {
+      "content": "<|dummy_69|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100334": {
+      "content": "<|dummy_70|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100335": {
+      "content": "<|dummy_71|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100336": {
+      "content": "<|dummy_72|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100337": {
+      "content": "<|dummy_73|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100338": {
+      "content": "<|dummy_74|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100339": {
+      "content": "<|dummy_75|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100340": {
+      "content": "<|dummy_76|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100341": {
+      "content": "<|dummy_77|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100342": {
+      "content": "<|dummy_78|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100343": {
+      "content": "<|dummy_79|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100344": {
+      "content": "<|dummy_80|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100345": {
+      "content": "<|dummy_81|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100346": {
+      "content": "<|dummy_82|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100347": {
+      "content": "<|dummy_83|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100348": {
+      "content": "<|dummy_84|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100349": {
+      "content": "<|dummy_85|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100350": {
+      "content": "<|dummy_86|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "100351": {
+      "content": "<|dummy_87|>",
+      "lstrip": true,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "chat_template": "{% for message in messages %}{% if (message['role'] == 'system') %}{{'<|im_start|>system<|im_sep|>' + message['content'] + '<|im_end|>'}}{% elif (message['role'] == 'user') %}{{'<|im_start|>user<|im_sep|>' + message['content'] + '<|im_end|><|im_start|>assistant<|im_sep|>'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|im_end|>'}}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": {},
+  "model_max_length": 16384,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

trainer_state.json

@@ -0,0 +1,1203 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 136.37209302325581,
+  "eval_steps": 100,
+  "global_step": 1500,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.9302325581395349,
+      "grad_norm": 0.11999286711215973,
+      "learning_rate": 1.0000000000000001e-11,
+      "loss": 0.7581,
+      "step": 10
+    },
+    {
+      "epoch": 1.8372093023255816,
+      "grad_norm": 0.11325732618570328,
+      "learning_rate": 2.0000000000000002e-11,
+      "loss": 0.7301,
+      "step": 20
+    },
+    {
+      "epoch": 2.744186046511628,
+      "grad_norm": 0.11053073406219482,
+      "learning_rate": 3.0000000000000006e-11,
+      "loss": 0.7401,
+      "step": 30
+    },
+    {
+      "epoch": 3.6511627906976747,
+      "grad_norm": 0.10902135074138641,
+      "learning_rate": 4.0000000000000004e-11,
+      "loss": 0.7335,
+      "step": 40
+    },
+    {
+      "epoch": 4.558139534883721,
+      "grad_norm": 0.12092321366071701,
+      "learning_rate": 5.000000000000001e-11,
+      "loss": 0.7457,
+      "step": 50
+    },
+    {
+      "epoch": 5.465116279069767,
+      "grad_norm": 0.11132938414812088,
+      "learning_rate": 6.000000000000001e-11,
+      "loss": 0.727,
+      "step": 60
+    },
+    {
+      "epoch": 6.372093023255814,
+      "grad_norm": 0.11651039868593216,
+      "learning_rate": 7e-11,
+      "loss": 0.743,
+      "step": 70
+    },
+    {
+      "epoch": 7.27906976744186,
+      "grad_norm": 0.11514942348003387,
+      "learning_rate": 8.000000000000001e-11,
+      "loss": 0.7364,
+      "step": 80
+    },
+    {
+      "epoch": 8.186046511627907,
+      "grad_norm": 0.10909898579120636,
+      "learning_rate": 9.000000000000001e-11,
+      "loss": 0.7424,
+      "step": 90
+    },
+    {
+      "epoch": 9.093023255813954,
+      "grad_norm": 0.11504866927862167,
+      "learning_rate": 1.0000000000000002e-10,
+      "loss": 0.7313,
+      "step": 100
+    },
+    {
+      "epoch": 9.093023255813954,
+      "eval_loss": 1.036345362663269,
+      "eval_runtime": 12.1723,
+      "eval_samples_per_second": 24.646,
+      "eval_steps_per_second": 1.561,
+      "step": 100
+    },
+    {
+      "epoch": 10.0,
+      "grad_norm": 0.08790237456560135,
+      "learning_rate": 1.1000000000000001e-10,
+      "loss": 0.7378,
+      "step": 110
+    },
+    {
+      "epoch": 10.930232558139535,
+      "grad_norm": 0.11877016723155975,
+      "learning_rate": 1.2000000000000003e-10,
+      "loss": 0.7512,
+      "step": 120
+    },
+    {
+      "epoch": 11.837209302325581,
+      "grad_norm": 0.11611445248126984,
+      "learning_rate": 1.3e-10,
+      "loss": 0.7507,
+      "step": 130
+    },
+    {
+      "epoch": 12.744186046511627,
+      "grad_norm": 0.113347128033638,
+      "learning_rate": 1.4e-10,
+      "loss": 0.7294,
+      "step": 140
+    },
+    {
+      "epoch": 13.651162790697674,
+      "grad_norm": 0.11206036806106567,
+      "learning_rate": 1.5000000000000002e-10,
+      "loss": 0.7424,
+      "step": 150
+    },
+    {
+      "epoch": 14.55813953488372,
+      "grad_norm": 0.1124008446931839,
+      "learning_rate": 1.6000000000000002e-10,
+      "loss": 0.7296,
+      "step": 160
+    },
+    {
+      "epoch": 15.465116279069768,
+      "grad_norm": 0.10790089517831802,
+      "learning_rate": 1.7e-10,
+      "loss": 0.7433,
+      "step": 170
+    },
+    {
+      "epoch": 16.372093023255815,
+      "grad_norm": 0.11213608831167221,
+      "learning_rate": 1.8000000000000002e-10,
+      "loss": 0.7361,
+      "step": 180
+    },
+    {
+      "epoch": 17.27906976744186,
+      "grad_norm": 0.11423593014478683,
+      "learning_rate": 1.9000000000000002e-10,
+      "loss": 0.7365,
+      "step": 190
+    },
+    {
+      "epoch": 18.186046511627907,
+      "grad_norm": 0.11061576008796692,
+      "learning_rate": 2.0000000000000003e-10,
+      "loss": 0.7323,
+      "step": 200
+    },
+    {
+      "epoch": 18.186046511627907,
+      "eval_loss": 1.0363353490829468,
+      "eval_runtime": 3.4847,
+      "eval_samples_per_second": 86.091,
+      "eval_steps_per_second": 5.452,
+      "step": 200
+    },
+    {
+      "epoch": 19.093023255813954,
+      "grad_norm": 0.1096525564789772,
+      "learning_rate": 2.1e-10,
+      "loss": 0.7416,
+      "step": 210
+    },
+    {
+      "epoch": 20.0,
+      "grad_norm": 0.09144695103168488,
+      "learning_rate": 2.2000000000000002e-10,
+      "loss": 0.7385,
+      "step": 220
+    },
+    {
+      "epoch": 20.930232558139537,
+      "grad_norm": 0.11446302384138107,
+      "learning_rate": 2.3e-10,
+      "loss": 0.7537,
+      "step": 230
+    },
+    {
+      "epoch": 21.837209302325583,
+      "grad_norm": 0.11259186267852783,
+      "learning_rate": 2.4000000000000005e-10,
+      "loss": 0.7392,
+      "step": 240
+    },
+    {
+      "epoch": 22.74418604651163,
+      "grad_norm": 0.11434454470872879,
+      "learning_rate": 2.5e-10,
+      "loss": 0.7313,
+      "step": 250
+    },
+    {
+      "epoch": 23.651162790697676,
+      "grad_norm": 0.11167125403881073,
+      "learning_rate": 2.6e-10,
+      "loss": 0.7353,
+      "step": 260
+    },
+    {
+      "epoch": 24.558139534883722,
+      "grad_norm": 0.11144951730966568,
+      "learning_rate": 2.7e-10,
+      "loss": 0.7399,
+      "step": 270
+    },
+    {
+      "epoch": 25.46511627906977,
+      "grad_norm": 0.1126742884516716,
+      "learning_rate": 2.8e-10,
+      "loss": 0.7399,
+      "step": 280
+    },
+    {
+      "epoch": 26.372093023255815,
+      "grad_norm": 0.1094028502702713,
+      "learning_rate": 2.9000000000000003e-10,
+      "loss": 0.7293,
+      "step": 290
+    },
+    {
+      "epoch": 27.27906976744186,
+      "grad_norm": 0.11483245342969894,
+      "learning_rate": 3.0000000000000005e-10,
+      "loss": 0.7399,
+      "step": 300
+    },
+    {
+      "epoch": 27.27906976744186,
+      "eval_loss": 1.0362988710403442,
+      "eval_runtime": 3.4771,
+      "eval_samples_per_second": 86.279,
+      "eval_steps_per_second": 5.464,
+      "step": 300
+    },
+    {
+      "epoch": 28.186046511627907,
+      "grad_norm": 0.11464320868253708,
+      "learning_rate": 3.1e-10,
+      "loss": 0.7331,
+      "step": 310
+    },
+    {
+      "epoch": 29.093023255813954,
+      "grad_norm": 0.11819777637720108,
+      "learning_rate": 3.2000000000000003e-10,
+      "loss": 0.7373,
+      "step": 320
+    },
+    {
+      "epoch": 30.0,
+      "grad_norm": 0.09258207678794861,
+      "learning_rate": 3.3000000000000005e-10,
+      "loss": 0.7354,
+      "step": 330
+    },
+    {
+      "epoch": 30.930232558139537,
+      "grad_norm": 0.1188226267695427,
+      "learning_rate": 3.4e-10,
+      "loss": 0.7555,
+      "step": 340
+    },
+    {
+      "epoch": 31.837209302325583,
+      "grad_norm": 0.11625378578901291,
+      "learning_rate": 3.5e-10,
+      "loss": 0.7378,
+      "step": 350
+    },
+    {
+      "epoch": 32.74418604651163,
+      "grad_norm": 0.11547369509935379,
+      "learning_rate": 3.6000000000000005e-10,
+      "loss": 0.743,
+      "step": 360
+    },
+    {
+      "epoch": 33.651162790697676,
+      "grad_norm": 0.11356023699045181,
+      "learning_rate": 3.7e-10,
+      "loss": 0.7284,
+      "step": 370
+    },
+    {
+      "epoch": 34.55813953488372,
+      "grad_norm": 0.11071494221687317,
+      "learning_rate": 3.8000000000000003e-10,
+      "loss": 0.7387,
+      "step": 380
+    },
+    {
+      "epoch": 35.46511627906977,
+      "grad_norm": 0.1178072988986969,
+      "learning_rate": 3.9000000000000005e-10,
+      "loss": 0.7358,
+      "step": 390
+    },
+    {
+      "epoch": 36.372093023255815,
+      "grad_norm": 0.11321323364973068,
+      "learning_rate": 4.0000000000000007e-10,
+      "loss": 0.7351,
+      "step": 400
+    },
+    {
+      "epoch": 36.372093023255815,
+      "eval_loss": 1.0363104343414307,
+      "eval_runtime": 3.476,
+      "eval_samples_per_second": 86.307,
+      "eval_steps_per_second": 5.466,
+      "step": 400
+    },
+    {
+      "epoch": 37.27906976744186,
+      "grad_norm": 0.11338995397090912,
+      "learning_rate": 4.1000000000000003e-10,
+      "loss": 0.7356,
+      "step": 410
+    },
+    {
+      "epoch": 38.18604651162791,
+      "grad_norm": 0.11327381432056427,
+      "learning_rate": 4.2e-10,
+      "loss": 0.7428,
+      "step": 420
+    },
+    {
+      "epoch": 39.093023255813954,
+      "grad_norm": 0.11791234463453293,
+      "learning_rate": 4.3000000000000007e-10,
+      "loss": 0.7357,
+      "step": 430
+    },
+    {
+      "epoch": 40.0,
+      "grad_norm": 0.0945475697517395,
+      "learning_rate": 4.4000000000000003e-10,
+      "loss": 0.7373,
+      "step": 440
+    },
+    {
+      "epoch": 40.93023255813954,
+      "grad_norm": 0.11064665764570236,
+      "learning_rate": 4.5000000000000005e-10,
+      "loss": 0.7571,
+      "step": 450
+    },
+    {
+      "epoch": 41.83720930232558,
+      "grad_norm": 0.11861127614974976,
+      "learning_rate": 4.6e-10,
+      "loss": 0.7367,
+      "step": 460
+    },
+    {
+      "epoch": 42.74418604651163,
+      "grad_norm": 0.11622505635023117,
+      "learning_rate": 4.7e-10,
+      "loss": 0.7379,
+      "step": 470
+    },
+    {
+      "epoch": 43.651162790697676,
+      "grad_norm": 0.11720574647188187,
+      "learning_rate": 4.800000000000001e-10,
+      "loss": 0.7405,
+      "step": 480
+    },
+    {
+      "epoch": 44.55813953488372,
+      "grad_norm": 0.1113167405128479,
+      "learning_rate": 4.900000000000001e-10,
+      "loss": 0.7325,
+      "step": 490
+    },
+    {
+      "epoch": 45.46511627906977,
+      "grad_norm": 0.1178051307797432,
+      "learning_rate": 5e-10,
+      "loss": 0.7357,
+      "step": 500
+    },
+    {
+      "epoch": 45.46511627906977,
+      "eval_loss": 1.0363179445266724,
+      "eval_runtime": 3.4794,
+      "eval_samples_per_second": 86.222,
+      "eval_steps_per_second": 5.461,
+      "step": 500
+    },
+    {
+      "epoch": 46.372093023255815,
+      "grad_norm": 0.11302001774311066,
+      "learning_rate": 5.1e-10,
+      "loss": 0.7368,
+      "step": 510
+    },
+    {
+      "epoch": 47.27906976744186,
+      "grad_norm": 0.11157575994729996,
+      "learning_rate": 5.2e-10,
+      "loss": 0.737,
+      "step": 520
+    },
+    {
+      "epoch": 48.18604651162791,
+      "grad_norm": 0.11483639478683472,
+      "learning_rate": 5.3e-10,
+      "loss": 0.738,
+      "step": 530
+    },
+    {
+      "epoch": 49.093023255813954,
+      "grad_norm": 0.11399766802787781,
+      "learning_rate": 5.4e-10,
+      "loss": 0.7383,
+      "step": 540
+    },
+    {
+      "epoch": 50.0,
+      "grad_norm": 0.09255016595125198,
+      "learning_rate": 5.500000000000001e-10,
+      "loss": 0.735,
+      "step": 550
+    },
+    {
+      "epoch": 50.93023255813954,
+      "grad_norm": 0.11325773596763611,
+      "learning_rate": 5.6e-10,
+      "loss": 0.7596,
+      "step": 560
+    },
+    {
+      "epoch": 51.83720930232558,
+      "grad_norm": 0.11773113161325455,
+      "learning_rate": 5.700000000000001e-10,
+      "loss": 0.7386,
+      "step": 570
+    },
+    {
+      "epoch": 52.74418604651163,
+      "grad_norm": 0.11714331805706024,
+      "learning_rate": 5.800000000000001e-10,
+      "loss": 0.732,
+      "step": 580
+    },
+    {
+      "epoch": 53.651162790697676,
+      "grad_norm": 0.11106540262699127,
+      "learning_rate": 5.9e-10,
+      "loss": 0.7379,
+      "step": 590
+    },
+    {
+      "epoch": 54.55813953488372,
+      "grad_norm": 0.1199754998087883,
+      "learning_rate": 6.000000000000001e-10,
+      "loss": 0.7337,
+      "step": 600
+    },
+    {
+      "epoch": 54.55813953488372,
+      "eval_loss": 1.0363249778747559,
+      "eval_runtime": 3.4958,
+      "eval_samples_per_second": 85.818,
+      "eval_steps_per_second": 5.435,
+      "step": 600
+    },
+    {
+      "epoch": 55.46511627906977,
+      "grad_norm": 0.11072533577680588,
+      "learning_rate": 6.100000000000001e-10,
+      "loss": 0.7349,
+      "step": 610
+    },
+    {
+      "epoch": 56.372093023255815,
+      "grad_norm": 0.11371786892414093,
+      "learning_rate": 6.2e-10,
+      "loss": 0.7356,
+      "step": 620
+    },
+    {
+      "epoch": 57.27906976744186,
+      "grad_norm": 0.11891023069620132,
+      "learning_rate": 6.300000000000001e-10,
+      "loss": 0.7408,
+      "step": 630
+    },
+    {
+      "epoch": 58.18604651162791,
+      "grad_norm": 0.11063282191753387,
+      "learning_rate": 6.400000000000001e-10,
+      "loss": 0.7358,
+      "step": 640
+    },
+    {
+      "epoch": 59.093023255813954,
+      "grad_norm": 0.1132306382060051,
+      "learning_rate": 6.5e-10,
+      "loss": 0.741,
+      "step": 650
+    },
+    {
+      "epoch": 60.0,
+      "grad_norm": 0.09035759419202805,
+      "learning_rate": 6.600000000000001e-10,
+      "loss": 0.7358,
+      "step": 660
+    },
+    {
+      "epoch": 60.93023255813954,
+      "grad_norm": 0.11425936967134476,
+      "learning_rate": 6.700000000000001e-10,
+      "loss": 0.7549,
+      "step": 670
+    },
+    {
+      "epoch": 61.83720930232558,
+      "grad_norm": 0.10887646675109863,
+      "learning_rate": 6.8e-10,
+      "loss": 0.738,
+      "step": 680
+    },
+    {
+      "epoch": 62.74418604651163,
+      "grad_norm": 0.11473378539085388,
+      "learning_rate": 6.9e-10,
+      "loss": 0.7357,
+      "step": 690
+    },
+    {
+      "epoch": 63.651162790697676,
+      "grad_norm": 0.11401313543319702,
+      "learning_rate": 7e-10,
+      "loss": 0.7405,
+      "step": 700
+    },
+    {
+      "epoch": 63.651162790697676,
+      "eval_loss": 1.036296010017395,
+      "eval_runtime": 3.4935,
+      "eval_samples_per_second": 85.874,
+      "eval_steps_per_second": 5.439,
+      "step": 700
+    },
+    {
+      "epoch": 64.55813953488372,
+      "grad_norm": 0.11050975322723389,
+      "learning_rate": 7.100000000000001e-10,
+      "loss": 0.7423,
+      "step": 710
+    },
+    {
+      "epoch": 65.46511627906976,
+      "grad_norm": 0.11604534834623337,
+      "learning_rate": 7.200000000000001e-10,
+      "loss": 0.7356,
+      "step": 720
+    },
+    {
+      "epoch": 66.37209302325581,
+      "grad_norm": 0.11239884048700333,
+      "learning_rate": 7.300000000000001e-10,
+      "loss": 0.7332,
+      "step": 730
+    },
+    {
+      "epoch": 67.27906976744185,
+      "grad_norm": 0.11227668821811676,
+      "learning_rate": 7.4e-10,
+      "loss": 0.7402,
+      "step": 740
+    },
+    {
+      "epoch": 68.18604651162791,
+      "grad_norm": 0.11599355936050415,
+      "learning_rate": 7.5e-10,
+      "loss": 0.7305,
+      "step": 750
+    },
+    {
+      "epoch": 69.09302325581395,
+      "grad_norm": 0.11684999614953995,
+      "learning_rate": 7.600000000000001e-10,
+      "loss": 0.7435,
+      "step": 760
+    },
+    {
+      "epoch": 70.0,
+      "grad_norm": 0.09078619629144669,
+      "learning_rate": 7.7e-10,
+      "loss": 0.7354,
+      "step": 770
+    },
+    {
+      "epoch": 70.93023255813954,
+      "grad_norm": 0.11326087266206741,
+      "learning_rate": 7.800000000000001e-10,
+      "loss": 0.7546,
+      "step": 780
+    },
+    {
+      "epoch": 71.83720930232558,
+      "grad_norm": 0.1127663403749466,
+      "learning_rate": 7.900000000000001e-10,
+      "loss": 0.7374,
+      "step": 790
+    },
+    {
+      "epoch": 72.74418604651163,
+      "grad_norm": 0.11612005531787872,
+      "learning_rate": 8.000000000000001e-10,
+      "loss": 0.7267,
+      "step": 800
+    },
+    {
+      "epoch": 72.74418604651163,
+      "eval_loss": 1.036307692527771,
+      "eval_runtime": 3.489,
+      "eval_samples_per_second": 85.984,
+      "eval_steps_per_second": 5.446,
+      "step": 800
+    },
+    {
+      "epoch": 73.65116279069767,
+      "grad_norm": 0.10922073572874069,
+      "learning_rate": 8.100000000000001e-10,
+      "loss": 0.7435,
+      "step": 810
+    },
+    {
+      "epoch": 74.55813953488372,
+      "grad_norm": 0.11204863339662552,
+      "learning_rate": 8.200000000000001e-10,
+      "loss": 0.7351,
+      "step": 820
+    },
+    {
+      "epoch": 75.46511627906976,
+      "grad_norm": 0.11315888166427612,
+      "learning_rate": 8.3e-10,
+      "loss": 0.7346,
+      "step": 830
+    },
+    {
+      "epoch": 76.37209302325581,
+      "grad_norm": 0.11475896835327148,
+      "learning_rate": 8.4e-10,
+      "loss": 0.7368,
+      "step": 840
+    },
+    {
+      "epoch": 77.27906976744185,
+      "grad_norm": 0.11209738999605179,
+      "learning_rate": 8.500000000000002e-10,
+      "loss": 0.7345,
+      "step": 850
+    },
+    {
+      "epoch": 78.18604651162791,
+      "grad_norm": 0.10578745603561401,
+      "learning_rate": 8.600000000000001e-10,
+      "loss": 0.7347,
+      "step": 860
+    },
+    {
+      "epoch": 79.09302325581395,
+      "grad_norm": 0.11142127960920334,
+      "learning_rate": 8.700000000000001e-10,
+      "loss": 0.7382,
+      "step": 870
+    },
+    {
+      "epoch": 80.0,
+      "grad_norm": 0.08924738317728043,
+      "learning_rate": 8.800000000000001e-10,
+      "loss": 0.7369,
+      "step": 880
+    },
+    {
+      "epoch": 80.93023255813954,
+      "grad_norm": 0.1128871962428093,
+      "learning_rate": 8.9e-10,
+      "loss": 0.7531,
+      "step": 890
+    },
+    {
+      "epoch": 81.83720930232558,
+      "grad_norm": 0.11317210644483566,
+      "learning_rate": 9.000000000000001e-10,
+      "loss": 0.7381,
+      "step": 900
+    },
+    {
+      "epoch": 81.83720930232558,
+      "eval_loss": 1.0362827777862549,
+      "eval_runtime": 3.4931,
+      "eval_samples_per_second": 85.883,
+      "eval_steps_per_second": 5.439,
+      "step": 900
+    },
+    {
+      "epoch": 82.74418604651163,
+      "grad_norm": 0.11075228452682495,
+      "learning_rate": 9.100000000000001e-10,
+      "loss": 0.7398,
+      "step": 910
+    },
+    {
+      "epoch": 83.65116279069767,
+      "grad_norm": 0.11806778609752655,
+      "learning_rate": 9.2e-10,
+      "loss": 0.7383,
+      "step": 920
+    },
+    {
+      "epoch": 84.55813953488372,
+      "grad_norm": 0.10795407742261887,
+      "learning_rate": 9.300000000000001e-10,
+      "loss": 0.7358,
+      "step": 930
+    },
+    {
+      "epoch": 85.46511627906976,
+      "grad_norm": 0.11355073004961014,
+      "learning_rate": 9.4e-10,
+      "loss": 0.7318,
+      "step": 940
+    },
+    {
+      "epoch": 86.37209302325581,
+      "grad_norm": 0.11858271807432175,
+      "learning_rate": 9.5e-10,
+      "loss": 0.7382,
+      "step": 950
+    },
+    {
+      "epoch": 87.27906976744185,
+      "grad_norm": 0.10782402008771896,
+      "learning_rate": 9.600000000000002e-10,
+      "loss": 0.7339,
+      "step": 960
+    },
+    {
+      "epoch": 88.18604651162791,
+      "grad_norm": 0.10977963358163834,
+      "learning_rate": 9.700000000000002e-10,
+      "loss": 0.7422,
+      "step": 970
+    },
+    {
+      "epoch": 89.09302325581395,
+      "grad_norm": 0.1131366416811943,
+      "learning_rate": 9.800000000000001e-10,
+      "loss": 0.7395,
+      "step": 980
+    },
+    {
+      "epoch": 90.0,
+      "grad_norm": 0.08679576963186264,
+      "learning_rate": 9.9e-10,
+      "loss": 0.7349,
+      "step": 990
+    },
+    {
+      "epoch": 90.93023255813954,
+      "grad_norm": 0.1211756095290184,
+      "learning_rate": 1e-09,
+      "loss": 0.7559,
+      "step": 1000
+    },
+    {
+      "epoch": 90.93023255813954,
+      "eval_loss": 1.0363088846206665,
+      "eval_runtime": 3.483,
+      "eval_samples_per_second": 86.132,
+      "eval_steps_per_second": 5.455,
+      "step": 1000
+    },
+    {
+      "epoch": 91.83720930232558,
+      "grad_norm": 0.11302389204502106,
+      "learning_rate": 1.01e-09,
+      "loss": 0.7363,
+      "step": 1010
+    },
+    {
+      "epoch": 92.74418604651163,
+      "grad_norm": 0.10905308276414871,
+      "learning_rate": 1.02e-09,
+      "loss": 0.7377,
+      "step": 1020
+    },
+    {
+      "epoch": 93.65116279069767,
+      "grad_norm": 0.11702612787485123,
+      "learning_rate": 1.03e-09,
+      "loss": 0.7262,
+      "step": 1030
+    },
+    {
+      "epoch": 94.55813953488372,
+      "grad_norm": 0.11045394837856293,
+      "learning_rate": 1.04e-09,
+      "loss": 0.7451,
+      "step": 1040
+    },
+    {
+      "epoch": 95.46511627906976,
+      "grad_norm": 0.11365518718957901,
+      "learning_rate": 1.05e-09,
+      "loss": 0.7371,
+      "step": 1050
+    },
+    {
+      "epoch": 96.37209302325581,
+      "grad_norm": 0.11288804560899734,
+      "learning_rate": 1.06e-09,
+      "loss": 0.74,
+      "step": 1060
+    },
+    {
+      "epoch": 97.27906976744185,
+      "grad_norm": 0.11331238597631454,
+      "learning_rate": 1.07e-09,
+      "loss": 0.732,
+      "step": 1070
+    },
+    {
+      "epoch": 98.18604651162791,
+      "grad_norm": 0.11176671087741852,
+      "learning_rate": 1.08e-09,
+      "loss": 0.742,
+      "step": 1080
+    },
+    {
+      "epoch": 99.09302325581395,
+      "grad_norm": 0.10659754276275635,
+      "learning_rate": 1.0900000000000002e-09,
+      "loss": 0.7335,
+      "step": 1090
+    },
+    {
+      "epoch": 100.0,
+      "grad_norm": 0.08784844726324081,
+      "learning_rate": 1.1000000000000001e-09,
+      "loss": 0.7371,
+      "step": 1100
+    },
+    {
+      "epoch": 100.0,
+      "eval_loss": 1.0363367795944214,
+      "eval_runtime": 3.5094,
+      "eval_samples_per_second": 85.485,
+      "eval_steps_per_second": 5.414,
+      "step": 1100
+    },
+    {
+      "epoch": 100.93023255813954,
+      "grad_norm": 0.11300911009311676,
+      "learning_rate": 1.11e-09,
+      "loss": 0.7575,
+      "step": 1110
+    },
+    {
+      "epoch": 101.83720930232558,
+      "grad_norm": 0.11774452775716782,
+      "learning_rate": 1.12e-09,
+      "loss": 0.7373,
+      "step": 1120
+    },
+    {
+      "epoch": 102.74418604651163,
+      "grad_norm": 0.10687077045440674,
+      "learning_rate": 1.13e-09,
+      "loss": 0.7418,
+      "step": 1130
+    },
+    {
+      "epoch": 103.65116279069767,
+      "grad_norm": 0.11354215443134308,
+      "learning_rate": 1.1400000000000002e-09,
+      "loss": 0.7316,
+      "step": 1140
+    },
+    {
+      "epoch": 104.55813953488372,
+      "grad_norm": 0.11316734552383423,
+      "learning_rate": 1.1500000000000002e-09,
+      "loss": 0.746,
+      "step": 1150
+    },
+    {
+      "epoch": 105.46511627906976,
+      "grad_norm": 0.11537086218595505,
+      "learning_rate": 1.1600000000000001e-09,
+      "loss": 0.7324,
+      "step": 1160
+    },
+    {
+      "epoch": 106.37209302325581,
+      "grad_norm": 0.11474544554948807,
+      "learning_rate": 1.17e-09,
+      "loss": 0.7341,
+      "step": 1170
+    },
+    {
+      "epoch": 107.27906976744185,
+      "grad_norm": 0.11136472970247269,
+      "learning_rate": 1.18e-09,
+      "loss": 0.7334,
+      "step": 1180
+    },
+    {
+      "epoch": 108.18604651162791,
+      "grad_norm": 0.10674024373292923,
+      "learning_rate": 1.1900000000000002e-09,
+      "loss": 0.7345,
+      "step": 1190
+    },
+    {
+      "epoch": 109.09302325581395,
+      "grad_norm": 0.10979607701301575,
+      "learning_rate": 1.2000000000000002e-09,
+      "loss": 0.7396,
+      "step": 1200
+    },
+    {
+      "epoch": 109.09302325581395,
+      "eval_loss": 1.036333441734314,
+      "eval_runtime": 3.4932,
+      "eval_samples_per_second": 85.881,
+      "eval_steps_per_second": 5.439,
+      "step": 1200
+    },
+    {
+      "epoch": 110.0,
+      "grad_norm": 0.09423299878835678,
+      "learning_rate": 1.2100000000000002e-09,
+      "loss": 0.7386,
+      "step": 1210
+    },
+    {
+      "epoch": 110.93023255813954,
+      "grad_norm": 0.11316266655921936,
+      "learning_rate": 1.2200000000000001e-09,
+      "loss": 0.7584,
+      "step": 1220
+    },
+    {
+      "epoch": 111.83720930232558,
+      "grad_norm": 0.12226978689432144,
+      "learning_rate": 1.23e-09,
+      "loss": 0.737,
+      "step": 1230
+    },
+    {
+      "epoch": 112.74418604651163,
+      "grad_norm": 0.11621031910181046,
+      "learning_rate": 1.24e-09,
+      "loss": 0.7368,
+      "step": 1240
+    },
+    {
+      "epoch": 113.65116279069767,
+      "grad_norm": 0.11429636180400848,
+      "learning_rate": 1.25e-09,
+      "loss": 0.731,
+      "step": 1250
+    },
+    {
+      "epoch": 114.55813953488372,
+      "grad_norm": 0.11287425458431244,
+      "learning_rate": 1.2600000000000002e-09,
+      "loss": 0.7398,
+      "step": 1260
+    },
+    {
+      "epoch": 115.46511627906976,
+      "grad_norm": 0.10956660658121109,
+      "learning_rate": 1.2700000000000002e-09,
+      "loss": 0.7341,
+      "step": 1270
+    },
+    {
+      "epoch": 116.37209302325581,
+      "grad_norm": 0.10884570330381393,
+      "learning_rate": 1.2800000000000001e-09,
+      "loss": 0.7364,
+      "step": 1280
+    },
+    {
+      "epoch": 117.27906976744185,
+      "grad_norm": 0.11374054849147797,
+      "learning_rate": 1.29e-09,
+      "loss": 0.7431,
+      "step": 1290
+    },
+    {
+      "epoch": 118.18604651162791,
+      "grad_norm": 0.11244415491819382,
+      "learning_rate": 1.3e-09,
+      "loss": 0.7411,
+      "step": 1300
+    },
+    {
+      "epoch": 118.18604651162791,
+      "eval_loss": 1.0363229513168335,
+      "eval_runtime": 3.504,
+      "eval_samples_per_second": 85.616,
+      "eval_steps_per_second": 5.422,
+      "step": 1300
+    },
+    {
+      "epoch": 119.09302325581395,
+      "grad_norm": 0.11712588369846344,
+      "learning_rate": 1.3100000000000002e-09,
+      "loss": 0.7345,
+      "step": 1310
+    },
+    {
+      "epoch": 120.0,
+      "grad_norm": 0.0905037373304367,
+      "learning_rate": 1.3200000000000002e-09,
+      "loss": 0.7356,
+      "step": 1320
+    },
+    {
+      "epoch": 120.93023255813954,
+      "grad_norm": 0.11537771672010422,
+      "learning_rate": 1.3300000000000002e-09,
+      "loss": 0.7568,
+      "step": 1330
+    },
+    {
+      "epoch": 121.83720930232558,
+      "grad_norm": 0.1106189489364624,
+      "learning_rate": 1.3400000000000001e-09,
+      "loss": 0.7353,
+      "step": 1340
+    },
+    {
+      "epoch": 122.74418604651163,
+      "grad_norm": 0.11432438343763351,
+      "learning_rate": 1.35e-09,
+      "loss": 0.7348,
+      "step": 1350
+    },
+    {
+      "epoch": 123.65116279069767,
+      "grad_norm": 0.11351707577705383,
+      "learning_rate": 1.36e-09,
+      "loss": 0.7478,
+      "step": 1360
+    },
+    {
+      "epoch": 124.55813953488372,
+      "grad_norm": 0.11303924024105072,
+      "learning_rate": 1.37e-09,
+      "loss": 0.7317,
+      "step": 1370
+    },
+    {
+      "epoch": 125.46511627906976,
+      "grad_norm": 0.11493802070617676,
+      "learning_rate": 1.38e-09,
+      "loss": 0.7358,
+      "step": 1380
+    },
+    {
+      "epoch": 126.37209302325581,
+      "grad_norm": 0.10831434279680252,
+      "learning_rate": 1.39e-09,
+      "loss": 0.7397,
+      "step": 1390
+    },
+    {
+      "epoch": 127.27906976744185,
+      "grad_norm": 0.10965836048126221,
+      "learning_rate": 1.4e-09,
+      "loss": 0.7304,
+      "step": 1400
+    },
+    {
+      "epoch": 127.27906976744185,
+      "eval_loss": 1.0363410711288452,
+      "eval_runtime": 3.4972,
+      "eval_samples_per_second": 85.782,
+      "eval_steps_per_second": 5.433,
+      "step": 1400
+    },
+    {
+      "epoch": 128.1860465116279,
+      "grad_norm": 0.11228539049625397,
+      "learning_rate": 1.4100000000000003e-09,
+      "loss": 0.7395,
+      "step": 1410
+    },
+    {
+      "epoch": 129.09302325581396,
+      "grad_norm": 0.11310495436191559,
+      "learning_rate": 1.4200000000000003e-09,
+      "loss": 0.7389,
+      "step": 1420
+    },
+    {
+      "epoch": 130.0,
+      "grad_norm": 0.08860716968774796,
+      "learning_rate": 1.4300000000000002e-09,
+      "loss": 0.7369,
+      "step": 1430
+    },
+    {
+      "epoch": 130.93023255813952,
+      "grad_norm": 0.10922189801931381,
+      "learning_rate": 1.4400000000000002e-09,
+      "loss": 0.7562,
+      "step": 1440
+    },
+    {
+      "epoch": 131.8372093023256,
+      "grad_norm": 0.11167009174823761,
+      "learning_rate": 1.4500000000000002e-09,
+      "loss": 0.7369,
+      "step": 1450
+    },
+    {
+      "epoch": 132.74418604651163,
+      "grad_norm": 0.1166851818561554,
+      "learning_rate": 1.4600000000000001e-09,
+      "loss": 0.7347,
+      "step": 1460
+    },
+    {
+      "epoch": 133.65116279069767,
+      "grad_norm": 0.11318089067935944,
+      "learning_rate": 1.47e-09,
+      "loss": 0.7338,
+      "step": 1470
+    },
+    {
+      "epoch": 134.5581395348837,
+      "grad_norm": 0.11904707551002502,
+      "learning_rate": 1.48e-09,
+      "loss": 0.7341,
+      "step": 1480
+    },
+    {
+      "epoch": 135.46511627906978,
+      "grad_norm": 0.11629388481378555,
+      "learning_rate": 1.49e-09,
+      "loss": 0.7504,
+      "step": 1490
+    },
+    {
+      "epoch": 136.37209302325581,
+      "grad_norm": 0.12060245871543884,
+      "learning_rate": 1.5e-09,
+      "loss": 0.7234,
+      "step": 1500
+    },
+    {
+      "epoch": 136.37209302325581,
+      "eval_loss": 1.0363295078277588,
+      "eval_runtime": 3.4888,
+      "eval_samples_per_second": 85.989,
+      "eval_steps_per_second": 5.446,
+      "step": 1500
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 100000000,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10000000,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.3019504646600786e+19,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:48f8699711f9e3ca0e1fbf79e7da011ba8aaf36be3a9c643ba4df2a4b6a46c25
+size 5752