Training in progress, step 2000, checkpoint

checkpoint-2000/README.md

@@ -0,0 +1,34 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+### Framework versions
+
+- PEFT 0.5.0
+
+- PEFT 0.5.0

checkpoint-2000/adapter_config.json

@@ -0,0 +1,20 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "EleutherAI/polyglot-ko-1.3b",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 32,
+  "lora_dropout": 0.05,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 16,
+  "revision": null,
+  "target_modules": [
+    "query_key_value"
+  ],
+  "task_type": "CAUSAL_LM"
+}

checkpoint-2000/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:edaa4885d15c6a606b3b1f662cf494343eb1e84cf7e80abac93240caaa9dbf56
+size 12600958

checkpoint-2000/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cc68efdf01fbc075b828e7fdf2959d297a585aea25fea0df09517a9aec6eae1
+size 25206586

checkpoint-2000/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3728e2f439637f446ae37ad9d652d0c841fefd1f8e3d6c878ea421b8b8d3ebca
+size 14308

checkpoint-2000/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e9c205cbf93770e99005c10540588ee0d3f3d16b7f841603e62ea2b9056806fb
+size 1064

checkpoint-2000/special_tokens_map.json

@@ -0,0 +1,11 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|sep|>",
+    "<|acc|>",
+    "<|tel|>",
+    "<|rrn|>"
+  ],
+  "eos_token": "<|endoftext|>",
+  "pad_token": "<|endoftext|>"
+}

checkpoint-2000/tokenizer_config.json

@@ -0,0 +1,72 @@
+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|unused0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|unused1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<|sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30000": {
+      "content": "<|acc|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30001": {
+      "content": "<|tel|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "30002": {
+      "content": "<|rrn|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "<|sep|>",
+    "<|acc|>",
+    "<|tel|>",
+    "<|rrn|>"
+  ],
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

checkpoint-2000/trainer_state.json

@@ -0,0 +1,71 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.861003861003861,
+  "eval_steps": 1000,
+  "global_step": 2000,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.58,
+      "learning_rate": 7.239382239382239e-07,
+      "loss": 2.3335,
+      "step": 300
+    },
+    {
+      "epoch": 1.16,
+      "learning_rate": 1.4478764478764478e-06,
+      "loss": 2.322,
+      "step": 600
+    },
+    {
+      "epoch": 1.74,
+      "learning_rate": 2.171814671814672e-06,
+      "loss": 2.268,
+      "step": 900
+    },
+    {
+      "epoch": 1.93,
+      "eval_loss": 2.2363617420196533,
+      "eval_runtime": 56.2328,
+      "eval_samples_per_second": 51.98,
+      "eval_steps_per_second": 3.254,
+      "step": 1000
+    },
+    {
+      "epoch": 2.32,
+      "learning_rate": 2.8957528957528956e-06,
+      "loss": 2.2427,
+      "step": 1200
+    },
+    {
+      "epoch": 2.9,
+      "learning_rate": 3.61969111969112e-06,
+      "loss": 2.2085,
+      "step": 1500
+    },
+    {
+      "epoch": 3.47,
+      "learning_rate": 4.343629343629344e-06,
+      "loss": 2.1902,
+      "step": 1800
+    },
+    {
+      "epoch": 3.86,
+      "eval_loss": 2.1604719161987305,
+      "eval_runtime": 56.4281,
+      "eval_samples_per_second": 51.8,
+      "eval_steps_per_second": 3.243,
+      "step": 2000
+    }
+  ],
+  "logging_steps": 300,
+  "max_steps": 8288,
+  "num_train_epochs": 16,
+  "save_steps": 1000,
+  "total_flos": 1.733060194257961e+17,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-2000/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3f853383e8dce9f00867dad3f3b4917bf63575060d38b299cdecb6532211b41
+size 4536