Upload model

README.md

@@ -0,0 +1,164 @@
+---
+library_name: transformers
+license: apache-2.0
+base_model: mistralai/Mistral-7B-vp0.3
+language: en
+datasets:
+- Word2Li/MiddOptimized
+tags:
+- llama-factory
+- full
+pipeline_tag: text-generation
+model-index:
+- name: Mistral-7B-v0.3-Middo-Alpaca-4o-mini
+  results:
+    - task:
+        type: text-generation
+      dataset:
+        name: MMLU
+        type: MMLU
+      metrics:
+        - name: weighted accuracy
+          type: weighted accuracy
+          value: 43.26
+          verified: true
+    - task:
+        type: text-generation
+      dataset:
+        name: IFEval
+        type: IFEval
+      metrics:
+        - name: overall accuracy
+          type: overall accuracy
+          value: 49.80
+          verified: true
+    - task:
+        type: text-generation
+      dataset:
+        name: GSM8K
+        type: GSM8K
+      metrics:
+        - name: accuracy
+          type: accuracy
+          value: 41.09
+          verified: true
+    - task:
+        type: text-generation
+      dataset:
+        name: MATH
+        type: MATH
+      metrics:
+        - name: accuracy
+          type: accuracy
+          value: 10.02
+          verified: true
+    - task:
+        type: text-generation
+      dataset:
+        name: HumanEval
+        type: HumanEval
+      metrics:
+        - name: humaneval_pass@1
+          type: humaneval_pass@1
+          value: 41.46
+          verified: true
+    - task:
+        type: text-generation
+      dataset:
+        name: MBPP
+        type: MBPP
+      metrics:
+        - name: score
+          type: score
+          value: 34.60
+          verified: true
+    - task:
+        type: text-generation
+      dataset:
+        name: Hellaswag
+        type: Hellaswag
+      metrics:
+        - name: accuracy
+          type: accuracy
+          value: 66.02
+          verified: true
+    - task:
+        type: text-generation
+      dataset:
+        name: GPQA
+        type: GPQA
+      metrics:
+        - name: accuracy
+          type: accuracy
+          value: 22.22
+          verified: true
+metrics:
+- accuracy
+---
+
+# Mistral-7B-v0.3-Middo-WizardLM
+
+Paper: [Middo: Model-Informed Dynamic Data Optimization for Enhanced LLM Fine-Tuning via Closed-Loop Learning](https://arxiv.org/abs/2508.21589)
+
+Code: https://github.com/Word2VecT/Middo
+
+## Model description
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.3](https://huggingface.co/mistralai/Mistral-7B-v0.3) on the [MiddOptimzed/mistral_wizard](https://huggingface.co/datasets/Word2Li/MiddOptimized/viewer/default/mistral_wizard) dataset.
+
+## Training and evaluation data
+
+### Training data
+
+Middo optimized [WizardLMTeam/WizardLM_evol_instruct_70k](https://huggingface.co/datasets/WizardLMTeam/WizardLM_evol_instruct_70k) on [mistralai/Mistral-7B-v0.3](https://huggingface.co/mistralai/Mistral-7B-v0.3).
+
+### Evaluation data
+
+- General
+  - MMLU
+  - IFEval
+- Math
+  - GSM8K
+  - MATH
+- Code
+  - HumanEval
+  - MBPP
+- Reasoning
+  - Hellaswag
+  - GPQA
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+
+- learning_rate: 1e-05
+- train_batch_size: 4
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 256
+- total_eval_batch_size: 64
+- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.03
+- num_epochs: 1.0
+
+### Training results
+
+- epoch: 1.0
+- total_flos: 4.871785990877872e+18
+- train_loss: 0.6260631282554998
+- train_runtime: 6928.3413
+- train_samples_per_second: 12.871
+- train_steps_per_second: 0.05
+
+### Framework versions
+
+- Transformers 4.55.0
+- Pytorch 2.6.0+cu124
+- Datasets 3.6.0
+- Tokenizers 0.21.1

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "total_flos": 4.871785990877872e+18,
+    "train_loss": 0.6260631282554998,
+    "train_runtime": 6928.3413,
+    "train_samples_per_second": 12.871,
+    "train_steps_per_second": 0.05
+}

chat_template.jinja

@@ -0,0 +1,4 @@
+{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ 'System: ' + system_message + '</s>' + '
+' }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ 'Human: ' + content + '</s>' + '
+Assistant:' }}{% elif message['role'] == 'assistant' %}{{ content + '</s>' + '
+' }}{% endif %}{% endfor %}

config.json

@@ -0,0 +1,26 @@
+{
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "head_dim": null,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.55.0",
+  "use_cache": false,
+  "vocab_size": 32768
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.55.0"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:32bd83189edfae6c02e34773dd429747f5c7acc5f305bc0a7e53e06c91864727
+size 4949453792

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51821c5b0119c1cd86f4624ecc1dc4ebda99d747ff78fa144ec52b14d8384f05
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c847514a586c038163c39cb8f76e62111ce0f1b5873e592d388d147c76d1c919
+size 4546807800

model.safetensors.index.json

@@ -0,0 +1,299 @@
+{
+  "metadata": {
+    "total_parameters": 7248023552,
+    "total_size": 14496047104
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:37f00374dea48658ee8f5d0f21895b9bc55cb0103939607c8185bfd1c6ca1f89
+size 587404

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "total_flos": 4.871785990877872e+18,
+    "train_loss": 0.6260631282554998,
+    "train_runtime": 6928.3413,
+    "train_samples_per_second": 12.871,
+    "train_steps_per_second": 0.05
+}

trainer_log.jsonl

@@ -0,0 +1,70 @@
+{"current_steps": 5, "total_steps": 349, "loss": 0.7414, "lr": 3.6363636363636366e-06, "epoch": 0.014352350197344816, "percentage": 1.43, "elapsed_time": "0:01:46", "remaining_time": "2:02:12"}
+{"current_steps": 10, "total_steps": 349, "loss": 0.7038, "lr": 8.181818181818183e-06, "epoch": 0.02870470039468963, "percentage": 2.87, "elapsed_time": "0:03:29", "remaining_time": "1:58:14"}
+{"current_steps": 15, "total_steps": 349, "loss": 0.7009, "lr": 9.998056338091415e-06, "epoch": 0.04305705059203445, "percentage": 4.3, "elapsed_time": "0:05:06", "remaining_time": "1:53:40"}
+{"current_steps": 20, "total_steps": 349, "loss": 0.6751, "lr": 9.986183876164412e-06, "epoch": 0.05740940078937926, "percentage": 5.73, "elapsed_time": "0:06:44", "remaining_time": "1:50:46"}
+{"current_steps": 25, "total_steps": 349, "loss": 0.6547, "lr": 9.96354437049027e-06, "epoch": 0.07176175098672408, "percentage": 7.16, "elapsed_time": "0:08:23", "remaining_time": "1:48:39"}
+{"current_steps": 30, "total_steps": 349, "loss": 0.6616, "lr": 9.930186708264902e-06, "epoch": 0.0861141011840689, "percentage": 8.6, "elapsed_time": "0:10:06", "remaining_time": "1:47:34"}
+{"current_steps": 35, "total_steps": 349, "loss": 0.6828, "lr": 9.88618292120984e-06, "epoch": 0.1004664513814137, "percentage": 10.03, "elapsed_time": "0:11:42", "remaining_time": "1:45:05"}
+{"current_steps": 40, "total_steps": 349, "loss": 0.6486, "lr": 9.831628030028698e-06, "epoch": 0.11481880157875853, "percentage": 11.46, "elapsed_time": "0:13:23", "remaining_time": "1:43:25"}
+{"current_steps": 45, "total_steps": 349, "loss": 0.6592, "lr": 9.76663983922178e-06, "epoch": 0.12917115177610333, "percentage": 12.89, "elapsed_time": "0:15:02", "remaining_time": "1:41:33"}
+{"current_steps": 50, "total_steps": 349, "loss": 0.6616, "lr": 9.691358682701927e-06, "epoch": 0.14352350197344815, "percentage": 14.33, "elapsed_time": "0:16:42", "remaining_time": "1:39:55"}
+{"current_steps": 55, "total_steps": 349, "loss": 0.6479, "lr": 9.605947120760878e-06, "epoch": 0.15787585217079297, "percentage": 15.76, "elapsed_time": "0:18:21", "remaining_time": "1:38:10"}
+{"current_steps": 60, "total_steps": 349, "loss": 0.6677, "lr": 9.510589589040554e-06, "epoch": 0.1722282023681378, "percentage": 17.19, "elapsed_time": "0:19:58", "remaining_time": "1:36:14"}
+{"current_steps": 65, "total_steps": 349, "loss": 0.6507, "lr": 9.405492000267228e-06, "epoch": 0.1865805525654826, "percentage": 18.62, "elapsed_time": "0:21:39", "remaining_time": "1:34:37"}
+{"current_steps": 70, "total_steps": 349, "loss": 0.6615, "lr": 9.29088129960862e-06, "epoch": 0.2009329027628274, "percentage": 20.06, "elapsed_time": "0:23:14", "remaining_time": "1:32:37"}
+{"current_steps": 75, "total_steps": 349, "loss": 0.6522, "lr": 9.16700497461403e-06, "epoch": 0.21528525296017223, "percentage": 21.49, "elapsed_time": "0:24:52", "remaining_time": "1:30:53"}
+{"current_steps": 80, "total_steps": 349, "loss": 0.6521, "lr": 9.034130520795774e-06, "epoch": 0.22963760315751705, "percentage": 22.92, "elapsed_time": "0:26:30", "remaining_time": "1:29:07"}
+{"current_steps": 85, "total_steps": 349, "loss": 0.6496, "lr": 8.892544864005899e-06, "epoch": 0.24398995335486187, "percentage": 24.36, "elapsed_time": "0:28:07", "remaining_time": "1:27:20"}
+{"current_steps": 90, "total_steps": 349, "loss": 0.6378, "lr": 8.742553740855507e-06, "epoch": 0.25834230355220666, "percentage": 25.79, "elapsed_time": "0:29:47", "remaining_time": "1:25:44"}
+{"current_steps": 95, "total_steps": 349, "loss": 0.6422, "lr": 8.584481038514573e-06, "epoch": 0.2726946537495515, "percentage": 27.22, "elapsed_time": "0:31:25", "remaining_time": "1:24:00"}
+{"current_steps": 100, "total_steps": 349, "loss": 0.6441, "lr": 8.418668095317912e-06, "epoch": 0.2870470039468963, "percentage": 28.65, "elapsed_time": "0:33:07", "remaining_time": "1:22:28"}
+{"current_steps": 105, "total_steps": 349, "loss": 0.6333, "lr": 8.245472963687484e-06, "epoch": 0.3013993541442411, "percentage": 30.09, "elapsed_time": "0:34:45", "remaining_time": "1:20:45"}
+{"current_steps": 110, "total_steps": 349, "loss": 0.6393, "lr": 8.065269636962765e-06, "epoch": 0.31575170434158595, "percentage": 31.52, "elapsed_time": "0:36:27", "remaining_time": "1:19:12"}
+{"current_steps": 115, "total_steps": 349, "loss": 0.647, "lr": 7.878447241808634e-06, "epoch": 0.33010405453893077, "percentage": 32.95, "elapsed_time": "0:38:08", "remaining_time": "1:17:36"}
+{"current_steps": 120, "total_steps": 349, "loss": 0.6226, "lr": 7.685409197944768e-06, "epoch": 0.3444564047362756, "percentage": 34.38, "elapsed_time": "0:39:47", "remaining_time": "1:15:55"}
+{"current_steps": 125, "total_steps": 349, "loss": 0.6442, "lr": 7.486572347010937e-06, "epoch": 0.35880875493362036, "percentage": 35.82, "elapsed_time": "0:41:23", "remaining_time": "1:14:11"}
+{"current_steps": 130, "total_steps": 349, "loss": 0.6434, "lr": 7.282366052449351e-06, "epoch": 0.3731611051309652, "percentage": 37.25, "elapsed_time": "0:43:01", "remaining_time": "1:12:28"}
+{"current_steps": 135, "total_steps": 349, "loss": 0.628, "lr": 7.073231272347714e-06, "epoch": 0.38751345532831, "percentage": 38.68, "elapsed_time": "0:44:40", "remaining_time": "1:10:49"}
+{"current_steps": 140, "total_steps": 349, "loss": 0.6349, "lr": 6.859619607245102e-06, "epoch": 0.4018658055256548, "percentage": 40.11, "elapsed_time": "0:46:20", "remaining_time": "1:09:10"}
+{"current_steps": 145, "total_steps": 349, "loss": 0.6159, "lr": 6.641992324956776e-06, "epoch": 0.41621815572299964, "percentage": 41.55, "elapsed_time": "0:47:54", "remaining_time": "1:07:24"}
+{"current_steps": 150, "total_steps": 349, "loss": 0.6397, "lr": 6.4208193645237314e-06, "epoch": 0.43057050592034446, "percentage": 42.98, "elapsed_time": "0:49:33", "remaining_time": "1:05:44"}
+{"current_steps": 155, "total_steps": 349, "loss": 0.6268, "lr": 6.1965783214377895e-06, "epoch": 0.4449228561176893, "percentage": 44.41, "elapsed_time": "0:51:12", "remaining_time": "1:04:05"}
+{"current_steps": 160, "total_steps": 349, "loss": 0.6302, "lr": 5.9697534163335645e-06, "epoch": 0.4592752063150341, "percentage": 45.85, "elapsed_time": "0:52:50", "remaining_time": "1:02:25"}
+{"current_steps": 165, "total_steps": 349, "loss": 0.6163, "lr": 5.740834449374237e-06, "epoch": 0.4736275565123789, "percentage": 47.28, "elapsed_time": "0:54:35", "remaining_time": "1:00:52"}
+{"current_steps": 170, "total_steps": 349, "loss": 0.6243, "lr": 5.510315742589042e-06, "epoch": 0.48797990670972374, "percentage": 48.71, "elapsed_time": "0:56:18", "remaining_time": "0:59:17"}
+{"current_steps": 175, "total_steps": 349, "loss": 0.6245, "lr": 5.278695072446342e-06, "epoch": 0.5023322569070685, "percentage": 50.14, "elapsed_time": "0:57:57", "remaining_time": "0:57:37"}
+{"current_steps": 180, "total_steps": 349, "loss": 0.607, "lr": 5.046472594967279e-06, "epoch": 0.5166846071044133, "percentage": 51.58, "elapsed_time": "0:59:33", "remaining_time": "0:55:55"}
+{"current_steps": 185, "total_steps": 349, "loss": 0.609, "lr": 4.814149765701059e-06, "epoch": 0.5310369573017582, "percentage": 53.01, "elapsed_time": "1:01:14", "remaining_time": "0:54:17"}
+{"current_steps": 190, "total_steps": 349, "loss": 0.6119, "lr": 4.582228256894093e-06, "epoch": 0.545389307499103, "percentage": 54.44, "elapsed_time": "1:02:56", "remaining_time": "0:52:39"}
+{"current_steps": 195, "total_steps": 349, "loss": 0.6291, "lr": 4.351208874191192e-06, "epoch": 0.5597416576964478, "percentage": 55.87, "elapsed_time": "1:04:40", "remaining_time": "0:51:04"}
+{"current_steps": 200, "total_steps": 349, "loss": 0.6255, "lr": 4.121590475208071e-06, "epoch": 0.5740940078937926, "percentage": 57.31, "elapsed_time": "1:06:17", "remaining_time": "0:49:23"}
+{"current_steps": 205, "total_steps": 349, "loss": 0.6269, "lr": 3.8938688923104015e-06, "epoch": 0.5884463580911374, "percentage": 58.74, "elapsed_time": "1:07:55", "remaining_time": "0:47:43"}
+{"current_steps": 210, "total_steps": 349, "loss": 0.6278, "lr": 3.668535861925509e-06, "epoch": 0.6027987082884823, "percentage": 60.17, "elapsed_time": "1:09:29", "remaining_time": "0:46:00"}
+{"current_steps": 215, "total_steps": 349, "loss": 0.6, "lr": 3.4460779626987186e-06, "epoch": 0.6171510584858271, "percentage": 61.6, "elapsed_time": "1:11:03", "remaining_time": "0:44:17"}
+{"current_steps": 220, "total_steps": 349, "loss": 0.5925, "lr": 3.226975564787322e-06, "epoch": 0.6315034086831719, "percentage": 63.04, "elapsed_time": "1:12:43", "remaining_time": "0:42:38"}
+{"current_steps": 225, "total_steps": 349, "loss": 0.6035, "lr": 3.0117017925609802e-06, "epoch": 0.6458557588805167, "percentage": 64.47, "elapsed_time": "1:14:18", "remaining_time": "0:40:56"}
+{"current_steps": 230, "total_steps": 349, "loss": 0.6022, "lr": 2.800721502948506e-06, "epoch": 0.6602081090778615, "percentage": 65.9, "elapsed_time": "1:15:58", "remaining_time": "0:39:18"}
+{"current_steps": 235, "total_steps": 349, "loss": 0.5966, "lr": 2.5944902816371573e-06, "epoch": 0.6745604592752064, "percentage": 67.34, "elapsed_time": "1:17:39", "remaining_time": "0:37:40"}
+{"current_steps": 240, "total_steps": 349, "loss": 0.5839, "lr": 2.3934534592920416e-06, "epoch": 0.6889128094725512, "percentage": 68.77, "elapsed_time": "1:19:16", "remaining_time": "0:36:00"}
+{"current_steps": 245, "total_steps": 349, "loss": 0.6063, "lr": 2.1980451499199262e-06, "epoch": 0.703265159669896, "percentage": 70.2, "elapsed_time": "1:20:54", "remaining_time": "0:34:20"}
+{"current_steps": 250, "total_steps": 349, "loss": 0.6068, "lr": 2.0086873134540626e-06, "epoch": 0.7176175098672407, "percentage": 71.63, "elapsed_time": "1:22:28", "remaining_time": "0:32:39"}
+{"current_steps": 255, "total_steps": 349, "loss": 0.5948, "lr": 1.8257888445842026e-06, "epoch": 0.7319698600645855, "percentage": 73.07, "elapsed_time": "1:24:10", "remaining_time": "0:31:01"}
+{"current_steps": 260, "total_steps": 349, "loss": 0.5932, "lr": 1.6497446897993885e-06, "epoch": 0.7463222102619304, "percentage": 74.5, "elapsed_time": "1:25:50", "remaining_time": "0:29:23"}
+{"current_steps": 265, "total_steps": 349, "loss": 0.5998, "lr": 1.4809349945501422e-06, "epoch": 0.7606745604592752, "percentage": 75.93, "elapsed_time": "1:27:27", "remaining_time": "0:27:43"}
+{"current_steps": 270, "total_steps": 349, "loss": 0.6135, "lr": 1.319724282371664e-06, "epoch": 0.77502691065662, "percentage": 77.36, "elapsed_time": "1:29:06", "remaining_time": "0:26:04"}
+{"current_steps": 275, "total_steps": 349, "loss": 0.6069, "lr": 1.1664606677406025e-06, "epoch": 0.7893792608539648, "percentage": 78.8, "elapsed_time": "1:30:48", "remaining_time": "0:24:26"}
+{"current_steps": 280, "total_steps": 349, "loss": 0.5974, "lr": 1.0214751043651582e-06, "epoch": 0.8037316110513096, "percentage": 80.23, "elapsed_time": "1:32:25", "remaining_time": "0:22:46"}
+{"current_steps": 285, "total_steps": 349, "loss": 0.5973, "lr": 8.850806705317183e-07, "epoch": 0.8180839612486545, "percentage": 81.66, "elapsed_time": "1:33:59", "remaining_time": "0:21:06"}
+{"current_steps": 290, "total_steps": 349, "loss": 0.5867, "lr": 7.575718930512516e-07, "epoch": 0.8324363114459993, "percentage": 83.09, "elapsed_time": "1:35:36", "remaining_time": "0:19:27"}
+{"current_steps": 295, "total_steps": 349, "loss": 0.61, "lr": 6.392241112653031e-07, "epoch": 0.8467886616433441, "percentage": 84.53, "elapsed_time": "1:37:13", "remaining_time": "0:17:47"}
+{"current_steps": 300, "total_steps": 349, "loss": 0.6084, "lr": 5.302928824849335e-07, "epoch": 0.8611410118406889, "percentage": 85.96, "elapsed_time": "1:38:48", "remaining_time": "0:16:08"}
+{"current_steps": 305, "total_steps": 349, "loss": 0.599, "lr": 4.3101343014651356e-07, "epoch": 0.8754933620380337, "percentage": 87.39, "elapsed_time": "1:40:28", "remaining_time": "0:14:29"}
+{"current_steps": 310, "total_steps": 349, "loss": 0.5911, "lr": 3.416001358759635e-07, "epoch": 0.8898457122353786, "percentage": 88.83, "elapsed_time": "1:42:05", "remaining_time": "0:12:50"}
+{"current_steps": 315, "total_steps": 349, "loss": 0.5886, "lr": 2.6224607655831236e-07, "epoch": 0.9041980624327234, "percentage": 90.26, "elapsed_time": "1:43:45", "remaining_time": "0:11:11"}
+{"current_steps": 320, "total_steps": 349, "loss": 0.5917, "lr": 1.9312260741218114e-07, "epoch": 0.9185504126300682, "percentage": 91.69, "elapsed_time": "1:45:26", "remaining_time": "0:09:33"}
+{"current_steps": 325, "total_steps": 349, "loss": 0.5799, "lr": 1.3437899196950765e-07, "epoch": 0.932902762827413, "percentage": 93.12, "elapsed_time": "1:47:14", "remaining_time": "0:07:55"}
+{"current_steps": 330, "total_steps": 349, "loss": 0.6015, "lr": 8.614207975952083e-08, "epoch": 0.9472551130247578, "percentage": 94.56, "elapsed_time": "1:48:53", "remaining_time": "0:06:16"}
+{"current_steps": 335, "total_steps": 349, "loss": 0.5848, "lr": 4.851603239296065e-08, "epoch": 0.9616074632221027, "percentage": 95.99, "elapsed_time": "1:50:31", "remaining_time": "0:04:37"}
+{"current_steps": 340, "total_steps": 349, "loss": 0.5879, "lr": 2.158209863804217e-08, "epoch": 0.9759598134194475, "percentage": 97.42, "elapsed_time": "1:52:08", "remaining_time": "0:02:58"}
+{"current_steps": 345, "total_steps": 349, "loss": 0.5836, "lr": 5.398438973845954e-09, "epoch": 0.9903121636167922, "percentage": 98.85, "elapsed_time": "1:53:47", "remaining_time": "0:01:19"}
+{"current_steps": 349, "total_steps": 349, "epoch": 1.0, "percentage": 100.0, "elapsed_time": "1:55:28", "remaining_time": "0:00:00"}

trainer_state.json

@@ -0,0 +1,517 @@
+{
+  "best_global_step": null,
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 349,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.014352350197344816,
+      "grad_norm": 11.071812629699707,
+      "learning_rate": 3.6363636363636366e-06,
+      "loss": 0.7414,
+      "step": 5
+    },
+    {
+      "epoch": 0.02870470039468963,
+      "grad_norm": 7.1317901611328125,
+      "learning_rate": 8.181818181818183e-06,
+      "loss": 0.7038,
+      "step": 10
+    },
+    {
+      "epoch": 0.04305705059203445,
+      "grad_norm": 4.095168113708496,
+      "learning_rate": 9.998056338091415e-06,
+      "loss": 0.7009,
+      "step": 15
+    },
+    {
+      "epoch": 0.05740940078937926,
+      "grad_norm": 2.938896894454956,
+      "learning_rate": 9.986183876164412e-06,
+      "loss": 0.6751,
+      "step": 20
+    },
+    {
+      "epoch": 0.07176175098672408,
+      "grad_norm": 3.5328586101531982,
+      "learning_rate": 9.96354437049027e-06,
+      "loss": 0.6547,
+      "step": 25
+    },
+    {
+      "epoch": 0.0861141011840689,
+      "grad_norm": 2.831934690475464,
+      "learning_rate": 9.930186708264902e-06,
+      "loss": 0.6616,
+      "step": 30
+    },
+    {
+      "epoch": 0.1004664513814137,
+      "grad_norm": 2.707149028778076,
+      "learning_rate": 9.88618292120984e-06,
+      "loss": 0.6828,
+      "step": 35
+    },
+    {
+      "epoch": 0.11481880157875853,
+      "grad_norm": 2.4383013248443604,
+      "learning_rate": 9.831628030028698e-06,
+      "loss": 0.6486,
+      "step": 40
+    },
+    {
+      "epoch": 0.12917115177610333,
+      "grad_norm": 2.7486937046051025,
+      "learning_rate": 9.76663983922178e-06,
+      "loss": 0.6592,
+      "step": 45
+    },
+    {
+      "epoch": 0.14352350197344815,
+      "grad_norm": 2.4958226680755615,
+      "learning_rate": 9.691358682701927e-06,
+      "loss": 0.6616,
+      "step": 50
+    },
+    {
+      "epoch": 0.15787585217079297,
+      "grad_norm": 2.9137589931488037,
+      "learning_rate": 9.605947120760878e-06,
+      "loss": 0.6479,
+      "step": 55
+    },
+    {
+      "epoch": 0.1722282023681378,
+      "grad_norm": 2.6844582557678223,
+      "learning_rate": 9.510589589040554e-06,
+      "loss": 0.6677,
+      "step": 60
+    },
+    {
+      "epoch": 0.1865805525654826,
+      "grad_norm": 2.127028703689575,
+      "learning_rate": 9.405492000267228e-06,
+      "loss": 0.6507,
+      "step": 65
+    },
+    {
+      "epoch": 0.2009329027628274,
+      "grad_norm": 2.487703800201416,
+      "learning_rate": 9.29088129960862e-06,
+      "loss": 0.6615,
+      "step": 70
+    },
+    {
+      "epoch": 0.21528525296017223,
+      "grad_norm": 2.5370140075683594,
+      "learning_rate": 9.16700497461403e-06,
+      "loss": 0.6522,
+      "step": 75
+    },
+    {
+      "epoch": 0.22963760315751705,
+      "grad_norm": 2.203188896179199,
+      "learning_rate": 9.034130520795774e-06,
+      "loss": 0.6521,
+      "step": 80
+    },
+    {
+      "epoch": 0.24398995335486187,
+      "grad_norm": 2.402724504470825,
+      "learning_rate": 8.892544864005899e-06,
+      "loss": 0.6496,
+      "step": 85
+    },
+    {
+      "epoch": 0.25834230355220666,
+      "grad_norm": 2.3624093532562256,
+      "learning_rate": 8.742553740855507e-06,
+      "loss": 0.6378,
+      "step": 90
+    },
+    {
+      "epoch": 0.2726946537495515,
+      "grad_norm": 2.36360239982605,
+      "learning_rate": 8.584481038514573e-06,
+      "loss": 0.6422,
+      "step": 95
+    },
+    {
+      "epoch": 0.2870470039468963,
+      "grad_norm": 2.0742623805999756,
+      "learning_rate": 8.418668095317912e-06,
+      "loss": 0.6441,
+      "step": 100
+    },
+    {
+      "epoch": 0.3013993541442411,
+      "grad_norm": 2.300394058227539,
+      "learning_rate": 8.245472963687484e-06,
+      "loss": 0.6333,
+      "step": 105
+    },
+    {
+      "epoch": 0.31575170434158595,
+      "grad_norm": 2.1633546352386475,
+      "learning_rate": 8.065269636962765e-06,
+      "loss": 0.6393,
+      "step": 110
+    },
+    {
+      "epoch": 0.33010405453893077,
+      "grad_norm": 2.291600227355957,
+      "learning_rate": 7.878447241808634e-06,
+      "loss": 0.647,
+      "step": 115
+    },
+    {
+      "epoch": 0.3444564047362756,
+      "grad_norm": 2.0962560176849365,
+      "learning_rate": 7.685409197944768e-06,
+      "loss": 0.6226,
+      "step": 120
+    },
+    {
+      "epoch": 0.35880875493362036,
+      "grad_norm": 2.4263839721679688,
+      "learning_rate": 7.486572347010937e-06,
+      "loss": 0.6442,
+      "step": 125
+    },
+    {
+      "epoch": 0.3731611051309652,
+      "grad_norm": 2.1769137382507324,
+      "learning_rate": 7.282366052449351e-06,
+      "loss": 0.6434,
+      "step": 130
+    },
+    {
+      "epoch": 0.38751345532831,
+      "grad_norm": 2.181265115737915,
+      "learning_rate": 7.073231272347714e-06,
+      "loss": 0.628,
+      "step": 135
+    },
+    {
+      "epoch": 0.4018658055256548,
+      "grad_norm": 2.1604084968566895,
+      "learning_rate": 6.859619607245102e-06,
+      "loss": 0.6349,
+      "step": 140
+    },
+    {
+      "epoch": 0.41621815572299964,
+      "grad_norm": 2.0021893978118896,
+      "learning_rate": 6.641992324956776e-06,
+      "loss": 0.6159,
+      "step": 145
+    },
+    {
+      "epoch": 0.43057050592034446,
+      "grad_norm": 2.3016304969787598,
+      "learning_rate": 6.4208193645237314e-06,
+      "loss": 0.6397,
+      "step": 150
+    },
+    {
+      "epoch": 0.4449228561176893,
+      "grad_norm": 2.1002519130706787,
+      "learning_rate": 6.1965783214377895e-06,
+      "loss": 0.6268,
+      "step": 155
+    },
+    {
+      "epoch": 0.4592752063150341,
+      "grad_norm": 4.018091201782227,
+      "learning_rate": 5.9697534163335645e-06,
+      "loss": 0.6302,
+      "step": 160
+    },
+    {
+      "epoch": 0.4736275565123789,
+      "grad_norm": 1.9025262594223022,
+      "learning_rate": 5.740834449374237e-06,
+      "loss": 0.6163,
+      "step": 165
+    },
+    {
+      "epoch": 0.48797990670972374,
+      "grad_norm": 1.9070066213607788,
+      "learning_rate": 5.510315742589042e-06,
+      "loss": 0.6243,
+      "step": 170
+    },
+    {
+      "epoch": 0.5023322569070685,
+      "grad_norm": 2.002209186553955,
+      "learning_rate": 5.278695072446342e-06,
+      "loss": 0.6245,
+      "step": 175
+    },
+    {
+      "epoch": 0.5166846071044133,
+      "grad_norm": 1.9884896278381348,
+      "learning_rate": 5.046472594967279e-06,
+      "loss": 0.607,
+      "step": 180
+    },
+    {
+      "epoch": 0.5310369573017582,
+      "grad_norm": 2.062976837158203,
+      "learning_rate": 4.814149765701059e-06,
+      "loss": 0.609,
+      "step": 185
+    },
+    {
+      "epoch": 0.545389307499103,
+      "grad_norm": 1.9803415536880493,
+      "learning_rate": 4.582228256894093e-06,
+      "loss": 0.6119,
+      "step": 190
+    },
+    {
+      "epoch": 0.5597416576964478,
+      "grad_norm": 1.8966432809829712,
+      "learning_rate": 4.351208874191192e-06,
+      "loss": 0.6291,
+      "step": 195
+    },
+    {
+      "epoch": 0.5740940078937926,
+      "grad_norm": 2.038236618041992,
+      "learning_rate": 4.121590475208071e-06,
+      "loss": 0.6255,
+      "step": 200
+    },
+    {
+      "epoch": 0.5884463580911374,
+      "grad_norm": 2.072604179382324,
+      "learning_rate": 3.8938688923104015e-06,
+      "loss": 0.6269,
+      "step": 205
+    },
+    {
+      "epoch": 0.6027987082884823,
+      "grad_norm": 2.125389814376831,
+      "learning_rate": 3.668535861925509e-06,
+      "loss": 0.6278,
+      "step": 210
+    },
+    {
+      "epoch": 0.6171510584858271,
+      "grad_norm": 2.0525448322296143,
+      "learning_rate": 3.4460779626987186e-06,
+      "loss": 0.6,
+      "step": 215
+    },
+    {
+      "epoch": 0.6315034086831719,
+      "grad_norm": 1.934063196182251,
+      "learning_rate": 3.226975564787322e-06,
+      "loss": 0.5925,
+      "step": 220
+    },
+    {
+      "epoch": 0.6458557588805167,
+      "grad_norm": 1.8446972370147705,
+      "learning_rate": 3.0117017925609802e-06,
+      "loss": 0.6035,
+      "step": 225
+    },
+    {
+      "epoch": 0.6602081090778615,
+      "grad_norm": 2.0047950744628906,
+      "learning_rate": 2.800721502948506e-06,
+      "loss": 0.6022,
+      "step": 230
+    },
+    {
+      "epoch": 0.6745604592752064,
+      "grad_norm": 1.9434928894042969,
+      "learning_rate": 2.5944902816371573e-06,
+      "loss": 0.5966,
+      "step": 235
+    },
+    {
+      "epoch": 0.6889128094725512,
+      "grad_norm": 1.9444347620010376,
+      "learning_rate": 2.3934534592920416e-06,
+      "loss": 0.5839,
+      "step": 240
+    },
+    {
+      "epoch": 0.703265159669896,
+      "grad_norm": 1.9846431016921997,
+      "learning_rate": 2.1980451499199262e-06,
+      "loss": 0.6063,
+      "step": 245
+    },
+    {
+      "epoch": 0.7176175098672407,
+      "grad_norm": 1.8537969589233398,
+      "learning_rate": 2.0086873134540626e-06,
+      "loss": 0.6068,
+      "step": 250
+    },
+    {
+      "epoch": 0.7319698600645855,
+      "grad_norm": 1.9193341732025146,
+      "learning_rate": 1.8257888445842026e-06,
+      "loss": 0.5948,
+      "step": 255
+    },
+    {
+      "epoch": 0.7463222102619304,
+      "grad_norm": 1.9606044292449951,
+      "learning_rate": 1.6497446897993885e-06,
+      "loss": 0.5932,
+      "step": 260
+    },
+    {
+      "epoch": 0.7606745604592752,
+      "grad_norm": 1.9930768013000488,
+      "learning_rate": 1.4809349945501422e-06,
+      "loss": 0.5998,
+      "step": 265
+    },
+    {
+      "epoch": 0.77502691065662,
+      "grad_norm": 1.9653593301773071,
+      "learning_rate": 1.319724282371664e-06,
+      "loss": 0.6135,
+      "step": 270
+    },
+    {
+      "epoch": 0.7893792608539648,
+      "grad_norm": 1.7707090377807617,
+      "learning_rate": 1.1664606677406025e-06,
+      "loss": 0.6069,
+      "step": 275
+    },
+    {
+      "epoch": 0.8037316110513096,
+      "grad_norm": 1.9296361207962036,
+      "learning_rate": 1.0214751043651582e-06,
+      "loss": 0.5974,
+      "step": 280
+    },
+    {
+      "epoch": 0.8180839612486545,
+      "grad_norm": 1.89194917678833,
+      "learning_rate": 8.850806705317183e-07,
+      "loss": 0.5973,
+      "step": 285
+    },
+    {
+      "epoch": 0.8324363114459993,
+      "grad_norm": 1.991350769996643,
+      "learning_rate": 7.575718930512516e-07,
+      "loss": 0.5867,
+      "step": 290
+    },
+    {
+      "epoch": 0.8467886616433441,
+      "grad_norm": 1.789543867111206,
+      "learning_rate": 6.392241112653031e-07,
+      "loss": 0.61,
+      "step": 295
+    },
+    {
+      "epoch": 0.8611410118406889,
+      "grad_norm": 1.8926385641098022,
+      "learning_rate": 5.302928824849335e-07,
+      "loss": 0.6084,
+      "step": 300
+    },
+    {
+      "epoch": 0.8754933620380337,
+      "grad_norm": 1.8285890817642212,
+      "learning_rate": 4.3101343014651356e-07,
+      "loss": 0.599,
+      "step": 305
+    },
+    {
+      "epoch": 0.8898457122353786,
+      "grad_norm": 1.9200646877288818,
+      "learning_rate": 3.416001358759635e-07,
+      "loss": 0.5911,
+      "step": 310
+    },
+    {
+      "epoch": 0.9041980624327234,
+      "grad_norm": 1.8193855285644531,
+      "learning_rate": 2.6224607655831236e-07,
+      "loss": 0.5886,
+      "step": 315
+    },
+    {
+      "epoch": 0.9185504126300682,
+      "grad_norm": 1.7819342613220215,
+      "learning_rate": 1.9312260741218114e-07,
+      "loss": 0.5917,
+      "step": 320
+    },
+    {
+      "epoch": 0.932902762827413,
+      "grad_norm": 1.8590822219848633,
+      "learning_rate": 1.3437899196950765e-07,
+      "loss": 0.5799,
+      "step": 325
+    },
+    {
+      "epoch": 0.9472551130247578,
+      "grad_norm": 1.7508701086044312,
+      "learning_rate": 8.614207975952083e-08,
+      "loss": 0.6015,
+      "step": 330
+    },
+    {
+      "epoch": 0.9616074632221027,
+      "grad_norm": 1.8058841228485107,
+      "learning_rate": 4.851603239296065e-08,
+      "loss": 0.5848,
+      "step": 335
+    },
+    {
+      "epoch": 0.9759598134194475,
+      "grad_norm": 1.788930892944336,
+      "learning_rate": 2.158209863804217e-08,
+      "loss": 0.5879,
+      "step": 340
+    },
+    {
+      "epoch": 0.9903121636167922,
+      "grad_norm": 1.9363088607788086,
+      "learning_rate": 5.398438973845954e-09,
+      "loss": 0.5836,
+      "step": 345
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 349,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.871785990877872e+18,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}