Upload folder using huggingface_hub

config.json

@@ -0,0 +1,77 @@
+{
+  "architectures": [
+    "Qwen3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoModelForCausalLM": "modeling_speculative_qwen3.SpeculativeQwen3ForCausalLM"
+  },
+  "bos_token_id": 151643,
+  "draft_layers": 1,
+  "eos_token_id": 151645,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 2560,
+  "initializer_range": 0.02,
+  "intermediate_size": 9728,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 262144,
+  "max_window_layers": 36,
+  "model_type": "qwen3",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 0,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 5000000,
+  "skip_input_layernorm": true,
+  "skip_output_norm": true,
+  "sliding_window": null,
+  "speculative_decoding_algorithm": "EagleV2",
+  "speculative_decoding_base_model_path": "Qwen/Qwen3-4B-Instruct-2507",
+  "speculative_decoding_draft_model": "Qwen3MoeDrafter",
+  "tie_word_embeddings": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.54.1",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

draft_model/config.json

@@ -0,0 +1,38 @@
+{
+  "architectures": [
+    "Qwen3MoeForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 151643,
+  "decoder_sparse_step": 1,
+  "eos_token_id": 151645,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 2560,
+  "initializer_range": 0.02,
+  "intermediate_size": 6144,
+  "max_position_embeddings": 262144,
+  "max_window_layers": 48,
+  "mlp_only_layers": [],
+  "model_type": "qwen3_moe",
+  "moe_intermediate_size": 1216,
+  "norm_topk_prob": true,
+  "num_attention_heads": 32,
+  "num_experts": 128,
+  "num_experts_per_tok": 8,
+  "num_hidden_layers": 1,
+  "num_key_value_heads": 8,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 5000000,
+  "router_aux_loss_coef": 0.001,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.54.1",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 151936
+}

draft_model/states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ae9e2e4bc8b39094938df8bebfa7afd0b39f93ddad29843846ddf6863d51b3a0
+size 4940286931

modeling_speculative_qwen3.py

@@ -0,0 +1,45 @@
+import torch
+from transformers.models.qwen3.modeling_qwen3 import *
+from specforge_het.specforge_lm import SpecForgeLM
+
+
+class Qwen3Drafter(Qwen3Model):
+    def __init__(self, draft_config, base_model):
+        draft_config.num_hidden_layers = base_model.config.draft_layers
+        draft_config.hidden_size = base_model.get_hidden_size()
+        super().__init__(draft_config)
+
+        if base_model.config.skip_input_layernorm:
+            for layer in self.layers:
+                delattr(layer, 'input_layernorm')
+                layer.input_layernorm = torch.nn.Identity()
+
+        if base_model.config.skip_output_norm:
+            delattr(self, 'norm')
+            self.norm = torch.nn.Identity()
+
+        delattr(self, 'embed_tokens')
+
+    def get_hidden_size(self):
+        return self.config.hidden_size
+
+
+class SpeculativeQwen3ForCausalLM(SpecForgeLM, Qwen3ForCausalLM):
+    @property
+    def base_model(self):
+        return self.model
+
+    def get_hidden_size(self):
+        return self.config.hidden_size
+
+    def get_base_layers(self):
+        return self.base_model.layers
+
+    def get_token_embedding(self, input_ids):
+        return self.base_model.embed_tokens(input_ids)
+
+    def get_token_logits(self, hidden_states):
+        return self.lm_head(hidden_states)
+
+    def save_pretrained(self, path, **kwargs):
+        return self.save_speculative_model(path, **kwargs)

specforge_het.json

@@ -0,0 +1,120 @@
+{
+  "dataset.debug": false,
+  "dataset.eval_path": null,
+  "dataset.git_diff": "",
+  "dataset.git_sha1": "unknown",
+  "dataset.manual_sample_ids": [],
+  "dataset.max_read_items": null,
+  "dataset.output_dir": "output",
+  "dataset.path": "output/datasets/ds_Qwen3-4B-Instruct-2507",
+  "dataset.read_eagle_format": false,
+  "dataset.run_name": "temp_run",
+  "dataset.seed": 0,
+  "dataset_generation.batch_size": 1,
+  "dataset_generation.debug": false,
+  "dataset_generation.debug_target": null,
+  "dataset_generation.ds_prefix": "ds_",
+  "dataset_generation.git_diff": "",
+  "dataset_generation.git_sha1": "unknown",
+  "dataset_generation.max_length": 2048,
+  "dataset_generation.output_dir": "output",
+  "dataset_generation.run_name": "temp_run",
+  "dataset_generation.save_every": 1000,
+  "dataset_generation.seed": 0,
+  "dataset_generation.sharegpt_path": "Aeala/ShareGPT_Vicuna_unfiltered",
+  "device_names": [
+    "NVIDIA H100 80GB HBM3"
+  ],
+  "inference.alpha_stats": false,
+  "inference.debug": false,
+  "inference.draft_growing": false,
+  "inference.draft_tree_shape": "mc_sim_7b_64",
+  "inference.dynamic_draft": true,
+  "inference.dynamic_draft_all_top_k": 59,
+  "inference.dynamic_draft_max_depth": 5,
+  "inference.dynamic_draft_top_k": 10,
+  "inference.git_diff": "",
+  "inference.git_sha1": "unknown",
+  "inference.interactive": false,
+  "inference.max_draft_growing_depth": 100,
+  "inference.max_new_tokens": 512,
+  "inference.mode": "speculative",
+  "inference.output_dir": "output",
+  "inference.run_name": "temp_run",
+  "inference.seed": 0,
+  "inference.timer": false,
+  "modeling.chat_template": "models/speculative_qwen3/chat_template.jinja2",
+  "modeling.debug": false,
+  "modeling.draft_config_modify": {
+    "moe_intermediate_size": "base.intermediate_size // draft_config.num_experts_per_tok",
+    "num_key_value_heads": "base.num_key_value_heads",
+    "rope_theta": "base.rope_theta"
+  },
+  "modeling.dtype": "torch.float32",
+  "modeling.free_base_layers": "num_hidden_layers",
+  "modeling.git_diff": "",
+  "modeling.git_sha1": "unknown",
+  "modeling.init_base_model": [
+    "SpeculativeQwen3ForCausalLM",
+    "Qwen/Qwen3-4B-Instruct-2507"
+  ],
+  "modeling.init_draft_config": [
+    "Qwen3MoeDrafter",
+    "Qwen/Qwen3-30B-A3B-Instruct-2507"
+  ],
+  "modeling.init_speculative_algorithm": [
+    "EagleV2",
+    "dict(draft_layers=1)"
+  ],
+  "modeling.model_path": null,
+  "modeling.output_dir": "output",
+  "modeling.run_name": "temp_run",
+  "modeling.seed": 0,
+  "modeling.tokenizer_add_tokens": "dict(unk_token=tokenizer.eos_token)",
+  "modeling.tokenizer_init": "",
+  "modeling.tokenizer_path": "Qwen/Qwen3-4B-Instruct-2507",
+  "training.adam_beta1": 0.9,
+  "training.adam_beta2": 0.95,
+  "training.average_tokens_across_devices": false,
+  "training.bf16": true,
+  "training.dataloader_drop_last": true,
+  "training.ddp_backend": "gloo",
+  "training.ddp_find_unused_parameters": false,
+  "training.debug": false,
+  "training.deepspeed": null,
+  "training.eval_steps": 100,
+  "training.eval_strategy": "no",
+  "training.filter_out_shorts": false,
+  "training.git_diff": "",
+  "training.git_sha1": "c545fcb3d9aa995dcf3e0ba5609559a39897f05e",
+  "training.gradient_accumulation_steps": 3,
+  "training.learning_rate": 3e-05,
+  "training.logging_first_step": true,
+  "training.logging_steps": 25,
+  "training.lr_scheduler_type": "constant_with_warmup",
+  "training.max_grad_norm": 0.5,
+  "training.max_length": 2048,
+  "training.max_steps": -1,
+  "training.model_init_ckpt": null,
+  "training.num_train_epochs": 10,
+  "training.optim": "adamw_torch_fused",
+  "training.output_dir": "output/eager-sky-65",
+  "training.overwrite_output_dir": true,
+  "training.per_device_eval_batch_size": 1,
+  "training.per_device_train_batch_size": 2,
+  "training.project": "eagle4",
+  "training.report_to": "wandb",
+  "training.resume_from_checkpoint": false,
+  "training.resume_wandb_runid": null,
+  "training.run_name": "eager-sky-65",
+  "training.save_steps": 500,
+  "training.save_strategy": "steps",
+  "training.save_total_limit": 2,
+  "training.seed": 0,
+  "training.sequential_loading": false,
+  "training.tf32": false,
+  "training.use_default_num_items_getter": true,
+  "training.use_eagle_pipeline": false,
+  "training.warmup_steps": 2000,
+  "training.world_size": 1
+}