add config files for training

README.md

@@ -24,15 +24,41 @@ More information needed
 **Training procedure**
 
 **Training hyperparameters**
+
 The following hyperparameters were used during training:
 
 
-* learning_rate: 5e-05
-* train_batch_size: 1
-* eval_batch_size: 2
 * seed: 42
-* optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+* learning_rate: 5e-06
+* train_batch_size: 32
+* eval_batch_size: 8
+* optimizer: Adam with betas : {'lr': 5e-06, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0.0}
 * lr_scheduler_type: linear
 * training_steps: 5000
+* fp16
+* warmup_steps 5
+
+* Num examples = 53k
+
+**Training results**
+
+
+```
+{
+    "epoch": 1.0,
+    "train_loss": 0.8031303182039198,
+    "train_runtime": 6338.6403,
+    "train_samples": 53455,
+    "train_samples_per_second": 8.433,
+    "train_steps_per_second": 0.264
+}
+```
+
+
+
+**Framework versions**
 
-**Training results**
+* transformers 4.24.0
+* torch 1.10.0+cu111
+* datasets 2.10.0
+* tokenizers 0.12.1

all_results.json

@@ -0,0 +1,14 @@
+{
+    "epoch": 1.0,
+    "eval_loss": 0.449462890625,
+    "eval_runtime": 8.3379,
+    "eval_samples": 540,
+    "eval_samples_per_second": 64.765,
+    "eval_steps_per_second": 8.156,
+    "perplexity": 1.567470056490351,
+    "train_loss": 0.8031303182039198,
+    "train_runtime": 6338.6403,
+    "train_samples": 53455,
+    "train_samples_per_second": 8.433,
+    "train_steps_per_second": 0.264
+}

config.json

@@ -0,0 +1,24 @@
+{
+  "architectures": [
+    "GPTNeoXForCausalLM"
+  ],
+  "bos_token_id": 0,
+  "eos_token_id": 0,
+  "hidden_act": "gelu",
+  "hidden_size": 2048,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "layer_norm_eps": 1e-05,
+  "max_position_embeddings": 2048,
+  "model_type": "gpt_neox",
+  "num_attention_heads": 16,
+  "num_hidden_layers": 24,
+  "rotary_emb_base": 10000,
+  "rotary_pct": 0.25,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.24.0",
+  "use_cache": false,
+  "use_parallel_residual": true,
+  "vocab_size": 50277
+}

eval_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "eval_loss": 0.449462890625,
+    "eval_runtime": 8.3379,
+    "eval_samples": 540,
+    "eval_samples_per_second": 64.765,
+    "eval_steps_per_second": 8.156,
+    "perplexity": 1.567470056490351
+}

special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
+}

tokenizer_config.json

@@ -0,0 +1,9 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "name_or_path": "EleutherAI/gpt-neox-20b",
+  "special_tokens_map_file": "/fsx/home-hailey/.cache/huggingface/hub/models--EleutherAI--gpt-neox-20b/snapshots/3523781c8df75f7741687a4284f6f70e1afa12f4/special_tokens_map.json",
+  "tokenizer_class": "GPTNeoXTokenizer",
+  "unk_token": "<|endoftext|>"
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.8031303182039198,
+    "train_runtime": 6338.6403,
+    "train_samples": 53455,
+    "train_samples_per_second": 8.433,
+    "train_steps_per_second": 0.264
+}

trainer_state.json

@@ -0,0 +1,107 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "global_step": 1671,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.12,
+      "eval_loss": 0.49560546875,
+      "eval_runtime": 8.4829,
+      "eval_samples_per_second": 63.657,
+      "eval_steps_per_second": 8.016,
+      "step": 200
+    },
+    {
+      "epoch": 0.24,
+      "eval_loss": 0.479248046875,
+      "eval_runtime": 8.8248,
+      "eval_samples_per_second": 61.191,
+      "eval_steps_per_second": 7.706,
+      "step": 400
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 5e-06,
+      "loss": 1.6261,
+      "step": 500
+    },
+    {
+      "epoch": 0.36,
+      "eval_loss": 0.47021484375,
+      "eval_runtime": 7.5494,
+      "eval_samples_per_second": 71.529,
+      "eval_steps_per_second": 9.007,
+      "step": 600
+    },
+    {
+      "epoch": 0.48,
+      "eval_loss": 0.465087890625,
+      "eval_runtime": 8.7604,
+      "eval_samples_per_second": 61.641,
+      "eval_steps_per_second": 7.762,
+      "step": 800
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 5e-06,
+      "loss": 0.4577,
+      "step": 1000
+    },
+    {
+      "epoch": 0.6,
+      "eval_loss": 0.46044921875,
+      "eval_runtime": 7.9849,
+      "eval_samples_per_second": 67.627,
+      "eval_steps_per_second": 8.516,
+      "step": 1000
+    },
+    {
+      "epoch": 0.72,
+      "eval_loss": 0.455322265625,
+      "eval_runtime": 8.5578,
+      "eval_samples_per_second": 63.1,
+      "eval_steps_per_second": 7.946,
+      "step": 1200
+    },
+    {
+      "epoch": 0.84,
+      "eval_loss": 0.45263671875,
+      "eval_runtime": 7.841,
+      "eval_samples_per_second": 68.868,
+      "eval_steps_per_second": 8.672,
+      "step": 1400
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 5e-06,
+      "loss": 0.4486,
+      "step": 1500
+    },
+    {
+      "epoch": 0.96,
+      "eval_loss": 0.450439453125,
+      "eval_runtime": 10.8958,
+      "eval_samples_per_second": 49.56,
+      "eval_steps_per_second": 6.241,
+      "step": 1600
+    },
+    {
+      "epoch": 1.0,
+      "step": 1671,
+      "total_flos": 1.1778474718632346e+17,
+      "train_loss": 0.8031303182039198,
+      "train_runtime": 6338.6403,
+      "train_samples_per_second": 8.433,
+      "train_steps_per_second": 0.264
+    }
+  ],
+  "max_steps": 1671,
+  "num_train_epochs": 1,
+  "total_flos": 1.1778474718632346e+17,
+  "trial_name": null,
+  "trial_params": null
+}