| { | |
| "vocab_size": 128000, | |
| "dim": 512, | |
| "num_layers": 6, | |
| "num_heads": 16, | |
| "max_recursion": 10, | |
| "ffn_expansion": 8, | |
| "max_position_embeddings": 1024, | |
| "balancing_weight": 0.01, | |
| "temperature": 1.0, | |
| "window_size": 128, | |
| "router_speed_weight": 0.1, | |
| "model_speed_weight": 0.5, | |
| "energy_weight": 0.001, | |
| "d_energy_weight": 0.005, | |
| "margin": 10.0, | |
| "latent_dim": 256, | |
| "model_type": "MoR-GAN", | |
| "architecture": "MixtureOfRecursions-GAN", | |
| "hidden_act": "gelu" | |
| } |