zip2zip_config.json

{
  "base_model_name_or_path": "microsoft/Phi-3-medium-4k-instruct",
  "compression": {
    "disabled_ids": [
      0,
      1,
      2,
      32000,
      32001,
      32002,
      32003,
      32004,
      32005,
      32006,
      32007,
      32008,
      32009,
      32010
    ],
    "initial_vocab_size": 32011,
    "max_codebook_size": 1024,
    "max_subtokens": 4
  },
  "encoder": {
    "hidden_size": 5120,
    "intermediate_size": null,
    "num_heads": 40,
    "num_hidden_layers": 2,
    "position_encoding": "learnable",
    "tie_encoders": true
  },
  "encoder_type": "transformer"
}