[INFO|tokenization_utils_base.py:2082] 2024-11-08 03:33:16,595 >> loading file tokenizer.model
[INFO|tokenization_utils_base.py:2082] 2024-11-08 03:33:16,595 >> loading file added_tokens.json
[INFO|tokenization_utils_base.py:2082] 2024-11-08 03:33:16,595 >> loading file special_tokens_map.json
[INFO|tokenization_utils_base.py:2082] 2024-11-08 03:33:16,595 >> loading file tokenizer_config.json
[INFO|tokenization_utils_base.py:2082] 2024-11-08 03:33:16,595 >> loading file tokenizer.json
tokenizer special_tokens_map: {'bos_token': '<s>', 'eos_token': '</s>', 'unk_token': '<unk>', 'pad_token': '<unk>'}
tokenizer special_tokens_map: {'bos_token': '<s>', 'eos_token': '</s>', 'unk_token': '<unk>', 'pad_token': '<unk>'}

  0%|          | 0/495 [00:00<?, ?ex/s]
  0%|          | 0/495 [00:00<?, ?ex/s]
  2%|▏         | 8/495 [00:00<00:06, 75.52ex/s]
  2%|▏         | 9/495 [00:00<00:06, 80.49ex/s]
  4%|▎         | 18/495 [00:00<00:05, 84.99ex/s]
  4%|▎         | 18/495 [00:00<00:05, 85.72ex/s]
  5%|▌         | 27/495 [00:00<00:05, 79.64ex/s]
  5%|▌         | 27/495 [00:00<00:05, 80.11ex/s]
  7%|▋         | 36/495 [00:00<00:05, 82.03ex/s]
  7%|▋         | 36/495 [00:00<00:05, 82.68ex/s]
  9%|▉         | 46/495 [00:00<00:05, 85.11ex/s]
  9%|▉         | 45/495 [00:00<00:05, 84.91ex/s]
 11%|█         | 55/495 [00:00<00:05, 84.52ex/s]
 11%|█         | 54/495 [00:00<00:05, 84.16ex/s]
 13%|█▎        | 65/495 [00:00<00:05, 85.48ex/s]
 13%|█▎        | 64/495 [00:00<00:04, 86.32ex/s]
 15%|█▍        | 74/495 [00:00<00:04, 84.73ex/s]
 15%|█▍        | 73/495 [00:00<00:04, 84.47ex/s]
 17%|█▋        | 83/495 [00:01<00:05, 81.72ex/s]
 17%|█▋        | 82/495 [00:00<00:05, 80.74ex/s]
 19%|█▊        | 92/495 [00:01<00:04, 81.31ex/s]
 18%|█▊        | 91/495 [00:01<00:04, 80.85ex/s]
 20%|██        | 101/495 [00:01<00:04, 81.21ex/s]
 20%|██        | 100/495 [00:01<00:05, 78.34ex/s]
 22%|██▏       | 110/495 [00:01<00:05, 76.94ex/s]
 22%|██▏       | 108/495 [00:01<00:05, 76.58ex/s]
 24%|██▍       | 119/495 [00:01<00:04, 79.74ex/s]
 24%|██▍       | 118/495 [00:01<00:04, 81.00ex/s]
 26%|██▌       | 128/495 [00:01<00:04, 80.79ex/s]
 26%|██▌       | 127/495 [00:01<00:04, 80.92ex/s]
 28%|██▊       | 137/495 [00:01<00:04, 81.64ex/s]
 27%|██▋       | 136/495 [00:01<00:04, 82.97ex/s]
 30%|██▉       | 147/495 [00:01<00:04, 84.18ex/s]
 29%|██▉       | 146/495 [00:01<00:04, 85.60ex/s]
 32%|███▏      | 156/495 [00:01<00:03, 84.85ex/s]
 31%|███▏      | 155/495 [00:01<00:04, 84.42ex/s]
 33%|███▎      | 165/495 [00:02<00:04, 77.61ex/s]
 33%|███▎      | 164/495 [00:02<00:04, 77.74ex/s]
 35%|███▌      | 175/495 [00:02<00:03, 83.01ex/s]
 35%|███▌      | 174/495 [00:02<00:03, 82.78ex/s]
 37%|███▋      | 185/495 [00:02<00:03, 83.97ex/s]
 37%|███▋      | 184/495 [00:02<00:03, 85.33ex/s]
 39%|███▉      | 195/495 [00:02<00:03, 88.10ex/s]
 39%|███▉      | 193/495 [00:02<00:03, 86.25ex/s]
 41%|████      | 204/495 [00:02<00:03, 87.55ex/s]
 41%|████      | 202/495 [00:02<00:03, 86.28ex/s]
 43%|████▎     | 213/495 [00:02<00:03, 86.39ex/s]
 43%|████▎     | 211/495 [00:02<00:03, 86.27ex/s]
 45%|████▍     | 222/495 [00:02<00:03, 85.12ex/s]
 44%|████▍     | 220/495 [00:02<00:03, 85.17ex/s]
 47%|████▋     | 231/495 [00:02<00:03, 85.11ex/s]
 46%|████▋     | 229/495 [00:02<00:03, 82.55ex/s]
 48%|████▊     | 240/495 [00:02<00:02, 85.58ex/s]
 48%|████▊     | 238/495 [00:02<00:03, 83.28ex/s]
 50%|█████     | 249/495 [00:03<00:03, 72.85ex/s]
 50%|████▉     | 247/495 [00:03<00:03, 74.26ex/s]
 52%|█████▏    | 257/495 [00:03<00:03, 62.96ex/s]
 52%|█████▏    | 255/495 [00:03<00:03, 63.16ex/s]
 53%|█████▎    | 264/495 [00:03<00:04, 56.63ex/s]
 53%|█████▎    | 262/495 [00:03<00:04, 57.98ex/s]
 55%|█████▍    | 271/495 [00:03<00:04, 53.13ex/s]
 54%|█████▍    | 269/495 [00:03<00:04, 53.07ex/s]
 56%|█████▌    | 277/495 [00:03<00:04, 51.58ex/s]
 56%|█████▌    | 275/495 [00:03<00:04, 51.49ex/s]
 57%|█████▋    | 283/495 [00:03<00:04, 51.09ex/s]
 57%|█████▋    | 281/495 [00:03<00:04, 50.43ex/s]
 58%|█████▊    | 289/495 [00:03<00:04, 50.38ex/s]
 58%|█████▊    | 287/495 [00:03<00:04, 49.36ex/s]
 60%|█████▉    | 295/495 [00:04<00:04, 46.62ex/s]
 59%|█████▉    | 293/495 [00:04<00:04, 47.06ex/s]
 61%|██████    | 300/495 [00:04<00:04, 46.73ex/s]
 60%|██████    | 298/495 [00:04<00:04, 46.31ex/s]
 62%|██████▏   | 305/495 [00:04<00:04, 46.23ex/s]
 61%|██████    | 303/495 [00:04<00:04, 44.87ex/s]
 63%|██████▎   | 310/495 [00:04<00:03, 46.34ex/s]
 62%|██████▏   | 308/495 [00:04<00:04, 45.40ex/s]
 64%|██████▎   | 315/495 [00:04<00:03, 46.95ex/s]
 63%|██████▎   | 314/495 [00:04<00:03, 47.72ex/s]
 65%|██████▍   | 321/495 [00:04<00:03, 49.09ex/s]
 65%|██████▍   | 320/495 [00:04<00:03, 48.57ex/s]
 66%|██████▌   | 326/495 [00:04<00:03, 47.02ex/s]
 66%|██████▌   | 325/495 [00:04<00:03, 46.90ex/s]
 67%|██████▋   | 331/495 [00:04<00:03, 43.99ex/s]
 67%|██████▋   | 330/495 [00:04<00:03, 43.84ex/s]
 68%|██████▊   | 336/495 [00:05<00:04, 39.17ex/s]
 68%|██████▊   | 335/495 [00:05<00:04, 38.92ex/s]
 69%|██████▉   | 341/495 [00:05<00:04, 38.21ex/s]
 69%|██████▊   | 340/495 [00:05<00:03, 40.93ex/s]
 70%|██████▉   | 346/495 [00:05<00:03, 39.70ex/s]
 70%|██████▉   | 345/495 [00:05<00:03, 40.00ex/s]
 71%|███████   | 351/495 [00:05<00:03, 40.78ex/s]
 71%|███████   | 350/495 [00:05<00:03, 39.99ex/s]
 72%|███████▏  | 356/495 [00:05<00:03, 41.52ex/s]
 72%|███████▏  | 355/495 [00:05<00:03, 40.89ex/s]
 73%|███████▎  | 361/495 [00:05<00:03, 38.55ex/s]
 73%|███████▎  | 360/495 [00:05<00:03, 37.61ex/s]
 74%|███████▍  | 366/495 [00:05<00:03, 40.84ex/s]
 74%|███████▎  | 365/495 [00:05<00:03, 39.98ex/s]
 75%|███████▍  | 371/495 [00:05<00:02, 41.59ex/s]
 75%|███████▍  | 370/495 [00:05<00:03, 40.84ex/s]
 76%|███████▌  | 376/495 [00:05<00:02, 41.04ex/s]
 76%|███████▌  | 375/495 [00:06<00:02, 40.75ex/s]
 77%|███████▋  | 381/495 [00:06<00:02, 40.77ex/s]
 77%|███████▋  | 380/495 [00:06<00:02, 40.62ex/s]
 78%|███████▊  | 386/495 [00:06<00:02, 37.01ex/s]
 78%|███████▊  | 385/495 [00:06<00:03, 36.55ex/s]
 79%|███████▉  | 391/495 [00:06<00:02, 38.64ex/s]
 79%|███████▊  | 389/495 [00:06<00:02, 36.43ex/s]
 80%|████████  | 396/495 [00:06<00:02, 38.63ex/s]
 79%|███████▉  | 393/495 [00:06<00:02, 36.93ex/s]
 81%|████████  | 400/495 [00:06<00:02, 38.57ex/s]
 80%|████████  | 397/495 [00:06<00:02, 37.03ex/s]
 82%|████████▏ | 405/495 [00:06<00:02, 41.28ex/s]
 81%|████████▏ | 403/495 [00:06<00:02, 41.88ex/s]
 83%|████████▎ | 410/495 [00:06<00:02, 42.44ex/s]
 82%|████████▏ | 408/495 [00:06<00:02, 43.00ex/s]
 84%|████████▍ | 415/495 [00:06<00:01, 42.81ex/s]
 83%|████████▎ | 413/495 [00:06<00:01, 44.44ex/s]
 85%|████████▍ | 420/495 [00:07<00:01, 42.78ex/s]
 84%|████████▍ | 418/495 [00:07<00:01, 44.93ex/s]
 86%|████████▌ | 426/495 [00:07<00:01, 46.20ex/s]
 86%|████████▌ | 424/495 [00:07<00:01, 47.08ex/s]
 87%|████████▋ | 431/495 [00:07<00:01, 46.50ex/s]
 87%|████████▋ | 430/495 [00:07<00:01, 48.20ex/s]
 88%|████████▊ | 437/495 [00:07<00:01, 47.65ex/s]
 88%|████████▊ | 435/495 [00:07<00:01, 47.85ex/s]
 89%|████████▉ | 443/495 [00:07<00:01, 48.71ex/s]
 89%|████████▉ | 440/495 [00:07<00:01, 48.36ex/s]
 91%|█████████ | 448/495 [00:07<00:00, 48.23ex/s]
 90%|████████▉ | 445/495 [00:07<00:01, 48.49ex/s]
 92%|█████████▏| 454/495 [00:07<00:00, 49.79ex/s]
 91%|█████████ | 451/495 [00:07<00:00, 50.02ex/s]
 93%|█████████▎| 459/495 [00:07<00:00, 48.62ex/s]
 92%|█████████▏| 456/495 [00:07<00:00, 48.95ex/s]
 94%|█████████▎| 464/495 [00:07<00:00, 46.50ex/s]
 93%|█████████▎| 462/495 [00:07<00:00, 49.78ex/s]
 95%|█████████▌| 471/495 [00:08<00:00, 49.93ex/s]
 94%|█████████▍| 467/495 [00:08<00:00, 48.26ex/s]
 96%|█████████▌| 476/495 [00:08<00:00, 48.93ex/s]
 96%|█████████▌| 473/495 [00:08<00:00, 49.38ex/s]
 97%|█████████▋| 482/495 [00:08<00:00, 50.74ex/s]
 97%|█████████▋| 479/495 [00:08<00:00, 50.27ex/s]
 99%|█████████▊| 488/495 [00:08<00:00, 49.46ex/s]
 98%|█████████▊| 485/495 [00:08<00:00, 50.91ex/s]
100%|█████████▉| 493/495 [00:08<00:00, 48.98ex/s]
 99%|█████████▉| 491/495 [00:08<00:00, 48.36ex/s]
100%|██████████| 495/495 [00:08<00:00, 57.62ex/s]

100%|██████████| 495/495 [00:08<00:00, 57.44ex/s]
[INFO|configuration_utils.py:724] 2024-11-08 03:33:26,280 >> loading configuration file model/config.json
[INFO|configuration_utils.py:724] 2024-11-08 03:33:26,284 >> loading configuration file model/config.json
[INFO|configuration_utils.py:789] 2024-11-08 03:33:26,286 >> Model config IndexConfig {
  "_name_or_path": "model",
  "architectures": [
    "IndexForCausalLM"
  ],
  "attention_bias": false,
  "attention_dropout": 0.0,
  "auto_map": {
    "AutoConfig": "configuration_index.IndexConfig",
    "AutoModelForCausalLM": "modeling_index.IndexForCausalLM",
    "AutoModelForSequenceClassification": "modeling_index.IndexForSequenceClassification"
  },
  "bos_token_id": 1,
  "eos_token_id": 2,
  "hidden_act": "silu",
  "hidden_size": 2048,
  "initializer_range": 0.01,
  "intermediate_size": 5888,
  "max_length": 32768,
  "max_position_embeddings": 32768,
  "model_type": "index",
  "norm_head": 1,
  "num_attention_heads": 16,
  "num_hidden_layers": 36,
  "num_key_value_heads": 16,
  "pad_token_id": 0,
  "pretraining_tp": 1,
  "rms_norm_eps": 1e-06,
  "rope_ratio": 32,
  "rope_scaling": null,
  "rope_theta": 10000.0,
  "tie_word_embeddings": false,
  "torch_dtype": "bfloat16",
  "transformers_version": "4.39.2",
  "use_cache": true,
  "vocab_size": 65029
}

[INFO|modeling_utils.py:3280] 2024-11-08 03:33:26,333 >> loading weights file model/pytorch_model.bin
[INFO|modeling_utils.py:1417] 2024-11-08 03:33:26,403 >> Instantiating IndexForCausalLM model under default dtype torch.bfloat16.
[INFO|configuration_utils.py:928] 2024-11-08 03:33:26,404 >> Generate config GenerationConfig {
  "bos_token_id": 1,
  "eos_token_id": 2,
  "max_length": 32768,
  "pad_token_id": 0,
  "use_cache": false
}

[INFO|modeling_utils.py:4024] 2024-11-08 03:33:31,504 >> All model checkpoint weights were used when initializing IndexForCausalLM.

[INFO|modeling_utils.py:4032] 2024-11-08 03:33:31,504 >> All the weights of IndexForCausalLM were initialized from the model checkpoint at model.
If your task is similar to the task the model of the checkpoint was trained on, you can already use IndexForCausalLM for predictions without further training.
[INFO|configuration_utils.py:881] 2024-11-08 03:33:31,511 >> loading configuration file model/generation_config.json
/opt/conda/lib/python3.10/site-packages/transformers/generation/configuration_utils.py:492: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.9` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`. This was detected when initializing the generation config instance, which means the corresponding file may hold incorrect parameterization and should be fixed.
  warnings.warn(
/opt/conda/lib/python3.10/site-packages/transformers/generation/configuration_utils.py:509: UserWarning: `do_sample` is set to `False`. However, `top_k` is set to `1` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_k`. This was detected when initializing the generation config instance, which means the corresponding file may hold incorrect parameterization and should be fixed.
  warnings.warn(
/opt/conda/lib/python3.10/site-packages/transformers/generation/configuration_utils.py:492: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.9` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.
  warnings.warn(
/opt/conda/lib/python3.10/site-packages/transformers/generation/configuration_utils.py:509: UserWarning: `do_sample` is set to `False`. However, `top_k` is set to `1` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_k`.
  warnings.warn(
[INFO|configuration_utils.py:928] 2024-11-08 03:33:31,512 >> Generate config GenerationConfig {
  "bos_token_id": 1,
  "eos_token_id": 2,
  "pad_token_id": 0,
  "repetition_penalty": 1.1,
  "temperature": 0.9,
  "top_k": 1
}

/opt/conda/lib/python3.10/site-packages/transformers/generation/configuration_utils.py:492: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.9` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`. This was detected when initializing the generation config instance, which means the corresponding file may hold incorrect parameterization and should be fixed.
  warnings.warn(
/opt/conda/lib/python3.10/site-packages/transformers/generation/configuration_utils.py:509: UserWarning: `do_sample` is set to `False`. However, `top_k` is set to `1` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_k`. This was detected when initializing the generation config instance, which means the corresponding file may hold incorrect parameterization and should be fixed.
  warnings.warn(
/opt/conda/lib/python3.10/site-packages/transformers/generation/configuration_utils.py:492: UserWarning: `do_sample` is set to `False`. However, `temperature` is set to `0.9` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `temperature`.
  warnings.warn(
/opt/conda/lib/python3.10/site-packages/transformers/generation/configuration_utils.py:509: UserWarning: `do_sample` is set to `False`. However, `top_k` is set to `1` -- this flag is only used in sample-based generation modes. You should set `do_sample=True` or unset `top_k`.
  warnings.warn(
trainable params: 46,301,184 || all params: 2,219,120,640 || trainable%: 2.0864653847751153
trainable params: 46,301,184 || all params: 2,219,120,640 || trainable%: 2.0864653847751153
[INFO|trainer.py:607] 2024-11-08 03:33:34,099 >> Using auto half precision backend
[WARNING|modeling_utils.py:2127] 2024-11-08 03:33:34,107 >> You are using an old version of the checkpointing format that is deprecated (We will also silently ignore `gradient_checkpointing_kwargs` in case you passed it).Please update to the new format on your modeling file. To use the new format, you need to completely remove the definition of the method `_set_gradient_checkpointing` in your model.
[WARNING|modeling_utils.py:2127] 2024-11-08 03:33:34,323 >> You are using an old version of the checkpointing format that is deprecated (We will also silently ignore `gradient_checkpointing_kwargs` in case you passed it).Please update to the new format on your modeling file. To use the new format, you need to completely remove the definition of the method `_set_gradient_checkpointing` in your model.
[INFO|trainer.py:1969] 2024-11-08 03:33:35,208 >> ***** Running training *****
[INFO|trainer.py:1970] 2024-11-08 03:33:35,208 >>   Num examples = 495
[INFO|trainer.py:1971] 2024-11-08 03:33:35,208 >>   Num Epochs = 6
[INFO|trainer.py:1972] 2024-11-08 03:33:35,208 >>   Instantaneous batch size per device = 4
[INFO|trainer.py:1975] 2024-11-08 03:33:35,208 >>   Total train batch size (w. parallel, distributed & accumulation) = 32
[INFO|trainer.py:1976] 2024-11-08 03:33:35,208 >>   Gradient Accumulation steps = 4
[INFO|trainer.py:1977] 2024-11-08 03:33:35,208 >>   Total optimization steps = 90
[INFO|trainer.py:1978] 2024-11-08 03:33:35,218 >>   Number of trainable parameters = 46,301,184

  0%|          | 0/90 [00:00<?, ?it/s]/opt/conda/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py:600: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.
  return fn(*args, **kwargs)
/opt/conda/lib/python3.10/site-packages/torch/_dynamo/eval_frame.py:600: UserWarning: torch.utils.checkpoint: the use_reentrant parameter should be passed explicitly. In version 2.4 we will raise an exception if use_reentrant is not passed. use_reentrant=False is recommended, but if you need to preserve the current default behavior, you can pass use_reentrant=True. Refer to docs for more details on the differences between the two variants.
  return fn(*args, **kwargs)
/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:295: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
  with torch.enable_grad(), device_autocast_ctx, torch.cpu.amp.autocast(**ctx.cpu_autocast_kwargs):  # type: ignore[attr-defined]
/opt/conda/lib/python3.10/site-packages/torch/utils/checkpoint.py:295: FutureWarning: `torch.cpu.amp.autocast(args...)` is deprecated. Please use `torch.amp.autocast('cpu', args...)` instead.
  with torch.enable_grad(), device_autocast_ctx, torch.cpu.amp.autocast(**ctx.cpu_autocast_kwargs):  # type: ignore[attr-defined]

  1%|          | 1/90 [05:04<7:32:22, 304.97s/it]
  2%|▏         | 2/90 [10:20<7:36:06, 310.99s/it]
  3%|▎         | 3/90 [15:37<7:34:58, 313.77s/it]
  4%|▍         | 4/90 [20:55<7:32:21, 315.59s/it]
  6%|▌         | 5/90 [26:12<7:27:52, 316.14s/it]
  7%|▋         | 6/90 [31:28<7:22:15, 315.90s/it]
  8%|▊         | 7/90 [36:43<7:16:41, 315.68s/it]
  9%|▉         | 8/90 [42:02<7:13:00, 316.84s/it]
 10%|█         | 9/90 [47:20<7:08:11, 317.17s/it]
 11%|█         | 10/90 [52:36<7:02:20, 316.76s/it]
                                                  
{'loss': 1.3217, 'grad_norm': 0.439453125, 'learning_rate': 0.0001469619730210873, 'epoch': 0.65}

 11%|█         | 10/90 [52:36<7:02:20, 316.76s/it]
 12%|█▏        | 11/90 [57:51<6:56:28, 316.31s/it]
 13%|█▎        | 12/90 [1:03:10<6:52:01, 316.94s/it]
 14%|█▍        | 13/90 [1:08:26<6:46:36, 316.83s/it]
 16%|█▌        | 14/90 [1:13:45<6:42:08, 317.48s/it]
 17%|█▋        | 15/90 [1:19:04<6:37:18, 317.85s/it]
 18%|█▊        | 16/90 [1:24:20<6:31:14, 317.23s/it]
 19%|█▉        | 17/90 [1:29:35<6:25:24, 316.77s/it]
 20%|██        | 18/90 [1:34:54<6:20:40, 317.22s/it]
 21%|██        | 19/90 [1:40:08<6:14:25, 316.42s/it]
 22%|██▏       | 20/90 [1:45:26<6:09:46, 316.95s/it]
                                                    
{'loss': 1.069, 'grad_norm': 0.2490234375, 'learning_rate': 0.00013504059306932703, 'epoch': 1.29}

 22%|██▏       | 20/90 [1:45:26<6:09:46, 316.95s/it]
 23%|██▎       | 21/90 [1:50:47<6:05:49, 318.11s/it]
 24%|██▍       | 22/90 [1:56:05<6:00:19, 317.93s/it]
 26%|██▌       | 23/90 [2:01:22<5:54:44, 317.69s/it]
 27%|██▋       | 24/90 [2:06:37<5:48:43, 317.03s/it]
 28%|██▊       | 25/90 [2:11:55<5:43:46, 317.34s/it]
 29%|██▉       | 26/90 [2:17:12<5:38:13, 317.09s/it]
 30%|███       | 27/90 [2:22:32<5:33:56, 318.03s/it]
 31%|███       | 28/90 [2:27:48<5:27:56, 317.36s/it]
 32%|███▏      | 29/90 [2:33:02<5:21:37, 316.36s/it]
 33%|███▎      | 30/90 [2:38:18<5:16:07, 316.13s/it]
                                                    
{'loss': 1.1914, 'grad_norm': 0.259765625, 'learning_rate': 0.00011554806130916982, 'epoch': 1.94}

 33%|███▎      | 30/90 [2:38:18<5:16:07, 316.13s/it]
 34%|███▍      | 31/90 [2:43:35<5:11:15, 316.54s/it]
 36%|███▌      | 32/90 [2:48:52<5:06:00, 316.56s/it]
 37%|███▋      | 33/90 [2:54:12<5:01:53, 317.79s/it]
 38%|███▊      | 34/90 [2:59:27<4:55:48, 316.94s/it]
 39%|███▉      | 35/90 [3:04:44<4:50:26, 316.85s/it]
 40%|████      | 36/90 [3:09:58<4:44:26, 316.05s/it]
 41%|████      | 37/90 [3:15:17<4:39:51, 316.82s/it]
 42%|████▏     | 38/90 [3:20:38<4:35:43, 318.15s/it]
 43%|████▎     | 39/90 [3:25:55<4:30:15, 317.95s/it]
 44%|████▍     | 40/90 [3:31:12<4:24:40, 317.61s/it]
                                                    
{'loss': 1.0584, 'grad_norm': 0.33203125, 'learning_rate': 9.094239671647324e-05, 'epoch': 2.58}

 44%|████▍     | 40/90 [3:31:12<4:24:40, 317.61s/it]
 46%|████▌     | 41/90 [3:36:31<4:19:35, 317.87s/it]
 47%|████▋     | 42/90 [3:41:49<4:14:23, 317.99s/it]
 48%|████▊     | 43/90 [3:47:07<4:09:03, 317.94s/it]
 49%|████▉     | 44/90 [3:52:26<4:04:06, 318.41s/it]
 50%|█████     | 45/90 [3:57:41<3:57:57, 317.27s/it]
 51%|█████     | 46/90 [4:02:55<3:51:50, 316.15s/it]
 52%|█████▏    | 47/90 [4:08:11<3:46:35, 316.17s/it]
 53%|█████▎    | 48/90 [4:13:28<3:41:33, 316.52s/it]
 54%|█████▍    | 49/90 [4:18:45<3:36:28, 316.78s/it]
 56%|█████▌    | 50/90 [4:24:00<3:30:39, 315.98s/it]
                                                    
{'loss': 1.1262, 'grad_norm': 0.3359375, 'learning_rate': 6.432638712950362e-05, 'epoch': 3.23}

 56%|█████▌    | 50/90 [4:24:00<3:30:39, 315.98s/it]
 57%|█████▋    | 51/90 [4:29:20<3:26:16, 317.34s/it]
 58%|█████▊    | 52/90 [4:34:39<3:21:17, 317.82s/it]
 59%|█████▉    | 53/90 [4:39:57<3:16:01, 317.87s/it]
 60%|██████    | 54/90 [4:45:13<3:10:24, 317.34s/it]
 61%|██████    | 55/90 [4:50:31<3:05:13, 317.52s/it]
 62%|██████▏   | 56/90 [4:55:49<3:00:04, 317.78s/it]
 63%|██████▎   | 57/90 [5:01:08<2:54:56, 318.07s/it]
 64%|██████▍   | 58/90 [5:06:25<2:49:21, 317.56s/it]
 66%|██████▌   | 59/90 [5:11:43<2:44:14, 317.87s/it]
 67%|██████▋   | 60/90 [5:16:58<2:38:25, 316.86s/it]
                                                    
{'loss': 0.7994, 'grad_norm': 0.34765625, 'learning_rate': 3.905632599599576e-05, 'epoch': 3.87}

 67%|██████▋   | 60/90 [5:16:58<2:38:25, 316.86s/it]
 68%|██████▊   | 61/90 [5:22:17<2:33:26, 317.47s/it]
 69%|██████▉   | 62/90 [5:27:34<2:28:12, 317.58s/it]
 70%|███████   | 63/90 [5:32:51<2:22:45, 317.25s/it]
 71%|███████   | 64/90 [5:38:11<2:17:50, 318.10s/it]
 72%|███████▏  | 65/90 [5:43:31<2:12:48, 318.73s/it]
 73%|███████▎  | 66/90 [5:48:47<2:07:06, 317.75s/it]
 74%|███████▍  | 67/90 [5:54:05<2:01:49, 317.82s/it]
 76%|███████▌  | 68/90 [5:59:23<1:56:37, 318.07s/it]
 77%|███████▋  | 69/90 [6:04:42<1:51:25, 318.34s/it]
 78%|███████▊  | 70/90 [6:10:01<1:46:10, 318.53s/it]
                                                    
{'loss': 0.7715, 'grad_norm': 0.333984375, 'learning_rate': 1.8318781923430636e-05, 'epoch': 4.52}

 78%|███████▊  | 70/90 [6:10:01<1:46:10, 318.53s/it]
 79%|███████▉  | 71/90 [6:15:19<1:40:48, 318.33s/it]
 80%|████████  | 72/90 [6:20:34<1:35:09, 317.19s/it]
 81%|████████  | 73/90 [6:25:50<1:29:47, 316.92s/it]
 82%|████████▏ | 74/90 [6:31:03<1:24:11, 315.70s/it]
 83%|████████▎ | 75/90 [6:36:16<1:18:42, 314.87s/it]
 84%|████████▍ | 76/90 [6:41:31<1:13:30, 315.05s/it]
 86%|████████▌ | 77/90 [6:46:49<1:08:25, 315.80s/it]
 87%|████████▋ | 78/90 [6:52:07<1:03:19, 316.59s/it]
 88%|████████▊ | 79/90 [6:57:26<58:09, 317.18s/it]  
 89%|████████▉ | 80/90 [7:02:42<52:47, 316.78s/it]
                                                  
{'loss': 1.0044, 'grad_norm': 0.291015625, 'learning_rate': 4.72877062501788e-06, 'epoch': 5.16}

 89%|████████▉ | 80/90 [7:02:42<52:47, 316.78s/it]
 90%|█████████ | 81/90 [7:07:56<47:25, 316.12s/it]
 91%|█████████ | 82/90 [7:13:10<42:03, 315.42s/it]
 92%|█████████▏| 83/90 [7:18:23<36:43, 314.82s/it]
 93%|█████████▎| 84/90 [7:23:42<31:36, 316.08s/it]
 94%|█████████▍| 85/90 [7:28:57<26:18, 315.73s/it]
 96%|█████████▌| 86/90 [7:34:16<21:06, 316.58s/it]
 97%|█████████▋| 87/90 [7:39:34<15:51, 317.06s/it]
 98%|█████████▊| 88/90 [7:44:54<10:35, 317.84s/it]
 99%|█████████▉| 89/90 [7:50:10<05:17, 317.32s/it]
100%|██████████| 90/90 [7:55:28<00:00, 317.68s/it]
                                                  
{'loss': 0.96, 'grad_norm': 0.287109375, 'learning_rate': 0.0, 'epoch': 5.81}

100%|██████████| 90/90 [7:55:28<00:00, 317.68s/it][INFO|trainer.py:2231] 2024-11-08 11:29:04,434 >> 

Training completed. Do not forget to share your model on huggingface.co/models =)



                                                  
{'train_runtime': 28529.2155, 'train_samples_per_second': 0.104, 'train_steps_per_second': 0.003, 'train_loss': 1.0335554016960993, 'epoch': 5.81}

100%|██████████| 90/90 [7:55:28<00:00, 317.68s/it]
100%|██████████| 90/90 [7:55:28<00:00, 316.99s/it]
[INFO|tokenization_utils_base.py:2502] 2024-11-08 11:29:04,871 >> tokenizer config file saved in /kaggle/working/adapter/tokenizer_config.json
[INFO|tokenization_utils_base.py:2511] 2024-11-08 11:29:04,872 >> Special tokens file saved in /kaggle/working/adapter/special_tokens_map.json
ModelTrainingArguments(
_n_gpu=1,
accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True},
adafactor=False,
adam_beta1=0.9,
adam_beta2=0.95,
adam_epsilon=1e-08,
auto_find_batch_size=False,
bf16=True,
bf16_full_eval=False,
data_seed=None,
dataloader_drop_last=False,
dataloader_num_workers=0,
dataloader_persistent_workers=False,
dataloader_pin_memory=True,
dataloader_prefetch_factor=None,
ddp_backend=None,
ddp_broadcast_buffers=None,
ddp_bucket_cap_mb=None,
ddp_find_unused_parameters=False,
ddp_timeout=1800,
debug=[],
deepspeed=None,
disable_tqdm=False,
dispatch_batches=None,
do_eval=False,
do_predict=False,
do_train=False,
eval_accumulation_steps=None,
eval_delay=0,
eval_steps=None,
evaluation_strategy=no,
fp16=False,
fp16_backend=auto,
fp16_full_eval=False,
fp16_opt_level=O1,
fsdp=[],
fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
fsdp_min_num_params=0,
fsdp_transformer_layer_cls_to_wrap=None,
full_determinism=False,
gradient_accumulation_steps=4,
gradient_checkpointing=False,
gradient_checkpointing_kwargs=None,
greater_is_better=None,
group_by_length=False,
half_precision_backend=auto,
hub_always_push=False,
hub_model_id=None,
hub_private_repo=False,
hub_strategy=every_save,
hub_token=<HUB_TOKEN>,
ignore_data_skip=False,
include_inputs_for_metrics=False,
include_num_input_tokens_seen=False,
include_tokens_per_second=False,
jit_mode_eval=False,
label_names=None,
label_smoothing_factor=0.0,
learning_rate=0.00015,
length_column_name=length,
load_best_model_at_end=False,
local_rank=0,
log_level=passive,
log_level_replica=warning,
log_on_each_node=True,
logging_dir=/kaggle/working/adapter/runs/Nov08_03-33-16_803be201245f,
logging_first_step=False,
logging_nan_inf_filter=True,
logging_steps=10,
logging_strategy=steps,
lr_scheduler_kwargs={},
lr_scheduler_type=cosine,
max_grad_norm=0.3,
max_steps=-1,
metric_for_best_model=None,
mp_parameters=,
neftune_noise_alpha=None,
no_cuda=False,
num_train_epochs=6.0,
optim=adamw_torch,
optim_args=None,
optim_target_modules=None,
output_dir=/kaggle/working/adapter,
overwrite_output_dir=False,
past_index=-1,
per_device_eval_batch_size=8,
per_device_train_batch_size=4,
prediction_loss_only=False,
push_to_hub=False,
push_to_hub_model_id=None,
push_to_hub_organization=None,
push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
ray_scope=last,
remove_unused_columns=True,
report_to=[],
resume_from_checkpoint=None,
run_name=/kaggle/working/adapter,
save_on_each_node=False,
save_only_model=False,
save_safetensors=False,
save_steps=250,
save_strategy=steps,
save_total_limit=None,
seed=1234,
skip_memory_metrics=True,
split_batches=None,
tf32=None,
torch_compile=False,
torch_compile_backend=None,
torch_compile_mode=None,
torchdynamo=None,
tpu_metrics_debug=False,
tpu_num_cores=None,
use_cpu=False,
use_ipex=False,
use_legacy_prediction_loop=False,
use_mps_device=False,
warmup_ratio=0.02,
warmup_steps=0,
weight_decay=0.01,
)ModelTrainingArguments(
_n_gpu=1,
accelerator_config={'split_batches': False, 'dispatch_batches': None, 'even_batches': True, 'use_seedable_sampler': True},
adafactor=False,
adam_beta1=0.9,
adam_beta2=0.95,
adam_epsilon=1e-08,
auto_find_batch_size=False,
bf16=True,
bf16_full_eval=False,
data_seed=None,
dataloader_drop_last=False,
dataloader_num_workers=0,
dataloader_persistent_workers=False,
dataloader_pin_memory=True,
dataloader_prefetch_factor=None,
ddp_backend=None,
ddp_broadcast_buffers=None,
ddp_bucket_cap_mb=None,
ddp_find_unused_parameters=False,
ddp_timeout=1800,
debug=[],
deepspeed=None,
disable_tqdm=False,
dispatch_batches=None,
do_eval=False,
do_predict=False,
do_train=False,
eval_accumulation_steps=None,
eval_delay=0,
eval_steps=None,
evaluation_strategy=no,
fp16=False,
fp16_backend=auto,
fp16_full_eval=False,
fp16_opt_level=O1,
fsdp=[],
fsdp_config={'min_num_params': 0, 'xla': False, 'xla_fsdp_v2': False, 'xla_fsdp_grad_ckpt': False},
fsdp_min_num_params=0,
fsdp_transformer_layer_cls_to_wrap=None,
full_determinism=False,
gradient_accumulation_steps=4,
gradient_checkpointing=False,
gradient_checkpointing_kwargs=None,
greater_is_better=None,
group_by_length=False,
half_precision_backend=auto,
hub_always_push=False,
hub_model_id=None,
hub_private_repo=False,
hub_strategy=every_save,
hub_token=<HUB_TOKEN>,
ignore_data_skip=False,
include_inputs_for_metrics=False,
include_num_input_tokens_seen=False,
include_tokens_per_second=False,
jit_mode_eval=False,
label_names=None,
label_smoothing_factor=0.0,
learning_rate=0.00015,
length_column_name=length,
load_best_model_at_end=False,
local_rank=1,
log_level=passive,
log_level_replica=warning,
log_on_each_node=True,
logging_dir=/kaggle/working/adapter/runs/Nov08_03-33-16_803be201245f,
logging_first_step=False,
logging_nan_inf_filter=True,
logging_steps=10,
logging_strategy=steps,
lr_scheduler_kwargs={},
lr_scheduler_type=cosine,
max_grad_norm=0.3,
max_steps=-1,
metric_for_best_model=None,
mp_parameters=,
neftune_noise_alpha=None,
no_cuda=False,
num_train_epochs=6.0,
optim=adamw_torch,
optim_args=None,
optim_target_modules=None,
output_dir=/kaggle/working/adapter,
overwrite_output_dir=False,
past_index=-1,
per_device_eval_batch_size=8,
per_device_train_batch_size=4,
prediction_loss_only=False,
push_to_hub=False,
push_to_hub_model_id=None,
push_to_hub_organization=None,
push_to_hub_token=<PUSH_TO_HUB_TOKEN>,
ray_scope=last,
remove_unused_columns=True,
report_to=[],
resume_from_checkpoint=None,
run_name=/kaggle/working/adapter,
save_on_each_node=False,
save_only_model=False,
save_safetensors=False,
save_steps=250,
save_strategy=steps,
save_total_limit=None,
seed=1234,
skip_memory_metrics=True,
split_batches=None,
tf32=None,
torch_compile=False,
torch_compile_backend=None,
torch_compile_mode=None,
torchdynamo=None,
tpu_metrics_debug=False,
tpu_num_cores=None,
use_cpu=False,
use_ipex=False,
use_legacy_prediction_loop=False,
use_mps_device=False,
warmup_ratio=0.02,
warmup_steps=0,
weight_decay=0.01,
)

[rank0]:[W1108 11:29:05.205724944 ProcessGroupNCCL.cpp:1168] Warning: WARNING: process group has NOT been destroyed before we destruct ProcessGroupNCCL. On normal program exit, the application should call destroy_process_group to ensure that any pending NCCL operations have finished in this process. In rare cases this process can exit before this point and block the progress of another member of the process group. This constraint has always been present,  but this warning has only been added since PyTorch 2.4 (function operator())