linear-moe-hub
/

MoM-Gated-Deltanet-340M

mom_gated_deltanet

Model card Files Files and versions Community

JusenK commited on Mar 5

Commit

b8860d7

·

1 Parent(s): 8d7a027

update config

Files changed (1) hide show

config.json +2 -2

config.json CHANGED Viewed

@@ -22,11 +22,11 @@
   "model_type": "mom_gated_deltanet",
   "norm_eps": 1e-06,
   "norm_first": false,
-  "num_experts": 4,
   "num_heads": 4,
   "num_hidden_layers": 24,
   "shared_mem": true,
-  "single_expert": false,
   "tie_word_embeddings": true,
   "topk": 2,
   "torch_dtype": "bfloat16",

   "model_type": "mom_gated_deltanet",
   "norm_eps": 1e-06,
   "norm_first": false,
+  "num_memories": 4,
   "num_heads": 4,
   "num_hidden_layers": 24,
   "shared_mem": true,
+  "single_kv_proj": false,
   "tie_word_embeddings": true,
   "topk": 2,
   "torch_dtype": "bfloat16",