Save model at global step 5

buffer_dataloader/data.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b98e2753af533b5e60bfe96bf6ce98b21362e4b5a19d9d16bb2a53e0d382fb47
+size 1108

config.yaml

@@ -0,0 +1,178 @@
+data:
+  tokenizer: null
+  train_files: ace-numina_ds_train_sample.parquet
+  val_files: matheval.parquet
+  prompt_key: prompt
+  reward_fn_key: data_source
+  max_prompt_length: 768
+  max_response_length: 13312
+  train_batch_size: 1024
+  val_batch_size: 640
+  return_raw_input_ids: false
+  return_raw_chat: false
+  shuffle: true
+  filter_overlong_prompts: false
+  filter_overlong_prompts_workers: 64
+  filter_overlong_responses: true
+  truncation: left
+  image_key: images
+  custom_cls:
+    path: null
+    name: null
+actor_rollout_ref:
+  hybrid_engine: true
+  model:
+    path: deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
+    external_lib: null
+    override_config: {}
+    enable_gradient_checkpointing: true
+    use_remove_padding: false
+    use_liger: false
+    save_hf_repo_id: RyanYr/brm-ace-numina-ace-numina-r1qwen1.5B-base-lr2.5e-6-bfsz1024-mbs2048-beta0.002
+    tokenizer_chat_template: deepseek-r1-think
+  actor:
+    brm:
+      norm_factor: value
+      value_constant: null
+    buffer:
+      buffer_type: offline_dataset_buffer
+      offline_dataset_buffer:
+        train_files: ace-numina_ds_offline_train_sample.parquet
+        response_key: response
+        response_truncation: right
+        shuffle: true
+        update_size: 1024
+    strategy: fsdp
+    ppo_mini_batch_size: 2048
+    ppo_micro_batch_size: null
+    ppo_micro_batch_size_per_gpu: 2
+    use_dynamic_bsz: false
+    ppo_max_token_len_per_gpu: 16384
+    grad_clip: 1.0
+    use_torch_compile: true
+    ppo_epochs: 1
+    shuffle: false
+    ulysses_sequence_parallel_size: 1
+    checkpoint:
+      contents:
+      - model
+      - optimizer
+      - extra
+    optim:
+      lr: 2.5e-06
+      lr_warmup_steps: -1
+      lr_warmup_steps_ratio: 0
+      min_lr_ratio: null
+      warmup_style: constant
+      total_training_steps: 100
+      weight_decay: 0.01
+    fsdp_config:
+      wrap_policy:
+        min_num_params: 0
+      param_offload: false
+      optimizer_offload: false
+      fsdp_size: -1
+    report_entropy: false
+  ref:
+    ref_model_path: deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
+    strategy: fsdp
+    fsdp_config:
+      param_offload: false
+      wrap_policy:
+        min_num_params: 0
+    log_prob_micro_batch_size: null
+    log_prob_micro_batch_size_per_gpu: 4
+    log_prob_use_dynamic_bsz: false
+    log_prob_max_token_len_per_gpu: 16384
+    ulysses_sequence_parallel_size: 1
+  rollout:
+    name: vllm
+    temperature: 1.0
+    top_k: -1
+    top_p: 1
+    use_fire_sampling: false
+    prompt_length: 768
+    response_length: 13312
+    dtype: bfloat16
+    gpu_memory_utilization: 0.8
+    ignore_eos: false
+    enforce_eager: false
+    free_cache_engine: false
+    load_format: dummy_dtensor
+    tensor_model_parallel_size: 4
+    max_num_batched_tokens: 14080
+    max_model_len: null
+    max_num_seqs: 1024
+    log_prob_micro_batch_size: null
+    log_prob_micro_batch_size_per_gpu: 2
+    log_prob_use_dynamic_bsz: false
+    log_prob_max_token_len_per_gpu: 16384
+    disable_log_stats: true
+    enable_chunked_prefill: true
+    do_sample: true
+    'n': 1
+    engine_kwargs:
+      swap_space: null
+    val_kwargs:
+      top_k: -1
+      top_p: 1.0
+      temperature: 0.6
+      'n': 32
+      do_sample: true
+reward_model:
+  enable: false
+  strategy: fsdp
+  model:
+    input_tokenizer: deepseek-ai/DeepSeek-R1-Distill-Qwen-1.5B
+    path: ~/models/FsfairX-LLaMA3-RM-v0.1
+    external_lib: null
+    use_remove_padding: false
+    fsdp_config:
+      wrap_policy:
+        min_num_params: 0
+      param_offload: false
+      fsdp_size: -1
+  micro_batch_size: null
+  micro_batch_size_per_gpu: null
+  max_length: null
+  ulysses_sequence_parallel_size: 1
+  use_dynamic_bsz: false
+  forward_max_token_len_per_gpu: 16384
+  reward_manager: prime
+  reward_kwargs:
+    format_reward: 0.0
+    format_type: null
+custom_reward_function:
+  path: null
+  name: compute_score
+trainer:
+  balance_batch: true
+  total_epochs: 100
+  total_training_steps: 100
+  project_name: value-LLM
+  experiment_name: brm-ace-numina-ace-numina-r1qwen1.5B-base_lr2.5e-6-bfsz1024-mbs2048-beta0.002
+  logger:
+  - console
+  - wandb
+  log_val_generations: 0
+  nnodes: 1
+  n_gpus_per_node: 8
+  save_freq: 5
+  resume_mode: auto
+  resume_from_path: null
+  val_before_train: false
+  test_freq: -1
+  default_hdfs_dir: null
+  del_local_ckpt_after_load: false
+  default_local_dir: ./BRM
+  max_actor_ckpt_to_keep: 1
+  ray_wait_register_center_timeout: 300
+  hf_token: null
+  resume_from_hf:
+    enable: false
+    hf_repo_id: null
+    hf_token: null
+    revision: main
+algorithm:
+  kl_ctrl:
+    kl_coef: 0.002

data.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afc4062d66c054b19727ba23a501f7adbdff67ceaff6342f2c199a6ee147863d
+size 1492

extra_state_world_size_8_rank_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2ba5748c2912062a3d03558edf4e18d19f7cdbe792c8157e87cbf0c4733bdcdc
+size 14632

extra_state_world_size_8_rank_1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c45b43345c7bfb5f859ec8c657bd63c4488c85e1153b6a755e0fa49873011e7
+size 14632

extra_state_world_size_8_rank_2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38b8a507cbc0aad8f79aabaad028adba3e91ed06590e2521c679a73609b3cbfe
+size 14632

extra_state_world_size_8_rank_3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:60acb24f0a5933736d62ac943458714910a4e2f8114ebf9464b03146b1a04621
+size 14632

extra_state_world_size_8_rank_4.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b2925a2b769fce6505d3902718c9351851287f8dd8b719318da2654a0571a4b
+size 14632

extra_state_world_size_8_rank_5.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f12794f2a8e8e04e493bd334a98bd451dcc5b355623af4c47209880046765fc
+size 14632

extra_state_world_size_8_rank_6.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0e11319d98ccb1476b0cdedab864a6d8fdd6af32dede866c8d0386f35d8edd55
+size 14632

extra_state_world_size_8_rank_7.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f34da93ba77dd57f3133991fc7b78c8605fa74124807f34171d12710e2eea99f
+size 14632

model_world_size_8_rank_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:143ddd57ab3b108cf920a9e81886ec5b82fab17bbe8e962458befb26622d9987
+size 888732218

model_world_size_8_rank_1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:193d22d86adef818d780e639dbd37d10a16cb806ad35b6b82dffc7155270959b
+size 888732218

model_world_size_8_rank_2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b3318e60b6314c664c910919c25887b5bb9237ec074a2f0343e8355cd9c350a3
+size 888732218

model_world_size_8_rank_3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccf9a44f1663fd2e500cb6e87a669c41c6bbb20423dd3af7eb1abff058c38684
+size 888732218

model_world_size_8_rank_4.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f888f9c74cad6b2ba64f0fea235e5259fc8fc9ab8e058ccd93e832a11b4286c2
+size 888732218

model_world_size_8_rank_5.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:317a2c8d732e1be38cb8c7b619bb888afbef0b699eeab6aadb7ea3d6b05bd2c1
+size 888732218

model_world_size_8_rank_6.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5c7efb60ebae7babb556b8872d920209217038bff3e63ed72775d030cdb40b00
+size 888732218

model_world_size_8_rank_7.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7d24eb767a1d911131b00feac5460460384735f4fb040fb11a05ac9dacab83d
+size 888732218

optim_world_size_8_rank_0.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eea5cfd8cec5954fcca23744d704a7d99f035272271a6347635267b848d9edd5
+size 1777114167

optim_world_size_8_rank_1.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:859077b316be0aeffab055c17873021a9701726675e36c57032f67a211c356df
+size 1777114167

optim_world_size_8_rank_2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7cc67fd372ca1e7473690ac21dad1dcf60b8abe52c4d594b44e02efea10ab301
+size 1777114167

optim_world_size_8_rank_3.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f6ad79f9a40e3a28a880ece89b63aa07f136855d1560624a1c33e15ad50b97f5
+size 1777114167

optim_world_size_8_rank_4.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a24e4685f7530dc9e6342b3fc755d78a3f86845d5e8146d157c52f4fe072346e
+size 1777114167

optim_world_size_8_rank_5.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:13aa87f94ac5d72caddf3f0016b8df39642e6faf5aceedbcb99ee35a460a1539
+size 1777114167

optim_world_size_8_rank_6.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1bc35da8c9199bdc39edd4f9d0dbc59df74621c19ee79fb09ed535faae2ef4dd
+size 1777114167

optim_world_size_8_rank_7.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:578682a0b1c46a43010488c7d78d3fdeefa6c82dfce981780eccf04189f71266
+size 1777114167