MHGanainy/gpt2-xl-lora-multi-shared-512-top

README.md

@@ -1,7 +1,7 @@
 ---
-library_name: peft
+library_name: transformers
 license: mit
-base_model: openai-community/gpt2-xl
+base_model: openai-community/gpt2
 tags:
 - generated_from_trainer
 model-index:
@@ -14,9 +14,14 @@ should probably proofread and complete it, then remove this comment. -->
 
 # gpt2-xl-lora-multi-shared-512-top
 
-This model is a fine-tuned version of [openai-community/gpt2-xl](https://huggingface.co/openai-community/gpt2-xl) on an unknown dataset.
+This model is a fine-tuned version of [openai-community/gpt2](https://huggingface.co/openai-community/gpt2) on an unknown dataset.
 It achieves the following results on the evaluation set:
-- Loss: 2.1270
+- eval_loss: 2.9468
+- eval_model_preparation_time: 0.0024
+- eval_runtime: 1258.8862
+- eval_samples_per_second: 150.453
+- eval_steps_per_second: 75.227
+- step: 0
 
 ## Model description
 
@@ -39,25 +44,15 @@ The following hyperparameters were used during training:
 - train_batch_size: 2
 - eval_batch_size: 2
 - seed: 42
-- distributed_type: multi-GPU
-- num_devices: 8
-- total_train_batch_size: 16
-- total_eval_batch_size: 16
-- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- optimizer: Use adamw_torch with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
-- lr_scheduler_warmup_steps: 69884
 - num_epochs: 1
 - mixed_precision_training: Native AMP
 
-### Training results
-
-
-
 ### Framework versions
 
-- PEFT 0.13.2
-- Transformers 4.45.2
+- Transformers 4.47.1
 - Pytorch 2.1.0a0+32f93b1
-- Datasets 3.0.1
-- Tokenizers 0.20.1
+- Datasets 3.2.0
+- Tokenizers 0.21.0

all_results.json

@@ -1,13 +1,8 @@
 {
-    "epoch": 1.0,
-    "eval_loss": 2.127037763595581,
-    "eval_runtime": 2671.6931,
-    "eval_samples_per_second": 70.892,
-    "eval_steps_per_second": 4.431,
-    "perplexity": 8.389976869779035,
-    "total_flos": 1.2729340395184456e+19,
-    "train_loss": 2.3255439768606525,
-    "train_runtime": 52634.826,
-    "train_samples_per_second": 26.555,
-    "train_steps_per_second": 1.66
+    "eval_loss": 2.9467811584472656,
+    "eval_model_preparation_time": 0.0024,
+    "eval_runtime": 1258.8862,
+    "eval_samples_per_second": 150.453,
+    "eval_steps_per_second": 75.227,
+    "perplexity": 19.044553562183122
 }

config.json

@@ -0,0 +1,39 @@
+{
+  "_name_or_path": "openai-community/gpt2",
+  "activation_function": "gelu_new",
+  "architectures": [
+    "GPT2LMHeadModel"
+  ],
+  "attn_pdrop": 0.1,
+  "bos_token_id": 50256,
+  "embd_pdrop": 0.1,
+  "eos_token_id": 50256,
+  "initializer_range": 0.02,
+  "layer_norm_epsilon": 1e-05,
+  "model_type": "gpt2",
+  "n_ctx": 1024,
+  "n_embd": 768,
+  "n_head": 12,
+  "n_inner": null,
+  "n_layer": 12,
+  "n_positions": 1024,
+  "reorder_and_upcast_attn": false,
+  "resid_pdrop": 0.1,
+  "scale_attn_by_inverse_layer_idx": false,
+  "scale_attn_weights": true,
+  "summary_activation": null,
+  "summary_first_dropout": 0.1,
+  "summary_proj_to_labels": true,
+  "summary_type": "cls_index",
+  "summary_use_proj": true,
+  "task_specific_params": {
+    "text-generation": {
+      "do_sample": true,
+      "max_length": 50
+    }
+  },
+  "torch_dtype": "float16",
+  "transformers_version": "4.47.1",
+  "use_cache": true,
+  "vocab_size": 50257
+}

eval_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 1.0,
-    "eval_loss": 2.127037763595581,
-    "eval_runtime": 2671.6931,
-    "eval_samples_per_second": 70.892,
-    "eval_steps_per_second": 4.431,
-    "perplexity": 8.389976869779035
+    "eval_loss": 2.9467811584472656,
+    "eval_model_preparation_time": 0.0024,
+    "eval_runtime": 1258.8862,
+    "eval_samples_per_second": 150.453,
+    "eval_steps_per_second": 75.227,
+    "perplexity": 19.044553562183122
 }

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "transformers_version": "4.47.1"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9a9affa1d9f5548ccc1e92cbe5f38b42cd237e1c2b1cdc6fd5db60813bfe1ab9
+size 248894512

runs/Dec20_15-21-58_0a5d2fff92ba/events.out.tfevents.1734709378.0a5d2fff92ba.5181.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1e9c2b3c85a504e4dc847a2d0d8aba119e0d808d510fb925b4a54d9114b23c0
+size 312

tokenizer_config.json

@@ -13,6 +13,7 @@
   "bos_token": "<|endoftext|>",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
+  "extra_special_tokens": {},
   "model_max_length": 1024,
   "pad_token": "<|endoftext|>",
   "tokenizer_class": "GPT2Tokenizer",

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6376f21d9ddc0e35ba852326e2c1f4c51a3b967fd83385e601e5286452290c9c
-size 5304
+oid sha256:045e2803cf84fbb192b66f7ef78d1756619f42c4e3f6ae66f870656e7c4f94d1
+size 5432