{
  "add_att_soft_cap": false,
  "architectures": [
    "PLM"
  ],
  "attention_soft_cap": 64.0,
  "expansion_ratio": 2.6667,
  "hidden_size": 512,
  "mlm": false,
  "num_att_tokens": 512,
  "num_attention_heads": 4,
  "num_hidden_layers": 12,
  "p_attention": false,
  "sliding_window_size": 2048,
  "soft_logit_cap": 32.0,
  "tie_embeddings": false,
  "token_dropout": true,
  "torch_dtype": "bfloat16",
  "transformers_version": "4.53.0",
  "unet": true,
  "vocab_size": 33
}