.hydra/config.yaml

finetune:
  model:
    name: gemma3_1B
    model_id: google/gemma-3-1b-it
    end_of_turn_token: <end_of_turn>
    attn_implementation: eager
  organism:
    name: cond2
    dataset:
      name: science-of-finetuning/tulu-3-sft-olmo-2-mixture-generated-${finetune.model.name}-cond2
      column: messages
      num_train: 10000
      num_test: 10000
      target_words:
      - life
      - fascinating
    training:
      lora_r: 16
      lora_alpha: 32
      lora_dropout: 0.05
      lora_bias: none
      lora_task_type: CAUSAL_LM
      lora_target_modules:
      - down_proj
  training:
    name: standard
    learning_rate: 5.0e-05
    batch_size: 8
    gradient_accumulation_steps: 2
    num_epochs: 1
    warmup_steps: 50
    weight_decay: 0.01
    max_grad_norm: 1.0
    optimizer:
      name: adamw
      betas:
      - 0.9
      - 0.999
      eps: 1.0e-08
    scheduler:
      name: linear
      warmup_ratio: 0.1
    training_config:
      evaluation_strategy: steps
      eval_steps: 100
      save_strategy: steps
      save_steps: 100
      logging_steps: 100
      save_total_limit: 3
      load_best_model_at_end: true
      metric_for_best_model: eval_loss
      greater_is_better: false
      early_stopping_patience: 2
    dataloader:
      num_workers: 4
      pin_memory: true
      drop_last: false
infrastructure:
  name: runpod
  storage:
    base_dir: /workspace/model-organisms/
    checkpoint_dir: ${infrastructure.storage.base_dir}/checkpoints
    logs_dir: ./logs
output_dir: ${infrastructure.storage.base_dir}/${finetune.model.name}_${finetune.organism.name}
run_name: ${finetune.model.name}_${finetune.organism.name}_${now:%Y%m%d_%H%M%S}
wandb:
  project: model-organisms-finetune
  entity: jkminder
  group: ${finetune.model.name}_${finetune.organism.name}
seed: 42
debug: false
upload_to_hf: true
hf_repo_name: science-of-finetuning/${finetune.model.name}_model_organism_${finetune.organism.name}