| { | |
| "base_model_name_or_path": "meta-llama/Llama-3.1-8B-Instruct", | |
| "compression": { | |
| "disabled_ids": [ | |
| 128000, | |
| 128001, | |
| 128002, | |
| 128003, | |
| 128004, | |
| 128005, | |
| 128006, | |
| 128007, | |
| 128008, | |
| 128009, | |
| 128010, | |
| 128011, | |
| 128012, | |
| 128013, | |
| 128014, | |
| 128015, | |
| 128016, | |
| 128017, | |
| 128018, | |
| 128019, | |
| 128020, | |
| 128021, | |
| 128022, | |
| 128023, | |
| 128024, | |
| 128025, | |
| 128026, | |
| 128027, | |
| 128028, | |
| 128029, | |
| 128030, | |
| 128031, | |
| 128032, | |
| 128033, | |
| 128034, | |
| 128035, | |
| 128036, | |
| 128037, | |
| 128038, | |
| 128039, | |
| 128040, | |
| 128041, | |
| 128042, | |
| 128043, | |
| 128044, | |
| 128045, | |
| 128046, | |
| 128047, | |
| 128048, | |
| 128049, | |
| 128050, | |
| 128051, | |
| 128052, | |
| 128053, | |
| 128054, | |
| 128055, | |
| 128056, | |
| 128057, | |
| 128058, | |
| 128059, | |
| 128060, | |
| 128061, | |
| 128062, | |
| 128063, | |
| 128064, | |
| 128065, | |
| 128066, | |
| 128067, | |
| 128068, | |
| 128069, | |
| 128070, | |
| 128071, | |
| 128072, | |
| 128073, | |
| 128074, | |
| 128075, | |
| 128076, | |
| 128077, | |
| 128078, | |
| 128079, | |
| 128080, | |
| 128081, | |
| 128082, | |
| 128083, | |
| 128084, | |
| 128085, | |
| 128086, | |
| 128087, | |
| 128088, | |
| 128089, | |
| 128090, | |
| 128091, | |
| 128092, | |
| 128093, | |
| 128094, | |
| 128095, | |
| 128096, | |
| 128097, | |
| 128098, | |
| 128099, | |
| 128100, | |
| 128101, | |
| 128102, | |
| 128103, | |
| 128104, | |
| 128105, | |
| 128106, | |
| 128107, | |
| 128108, | |
| 128109, | |
| 128110, | |
| 128111, | |
| 128112, | |
| 128113, | |
| 128114, | |
| 128115, | |
| 128116, | |
| 128117, | |
| 128118, | |
| 128119, | |
| 128120, | |
| 128121, | |
| 128122, | |
| 128123, | |
| 128124, | |
| 128125, | |
| 128126, | |
| 128127, | |
| 128128, | |
| 128129, | |
| 128130, | |
| 128131, | |
| 128132, | |
| 128133, | |
| 128134, | |
| 128135, | |
| 128136, | |
| 128137, | |
| 128138, | |
| 128139, | |
| 128140, | |
| 128141, | |
| 128142, | |
| 128143, | |
| 128144, | |
| 128145, | |
| 128146, | |
| 128147, | |
| 128148, | |
| 128149, | |
| 128150, | |
| 128151, | |
| 128152, | |
| 128153, | |
| 128154, | |
| 128155, | |
| 128156, | |
| 128157, | |
| 128158, | |
| 128159, | |
| 128160, | |
| 128161, | |
| 128162, | |
| 128163, | |
| 128164, | |
| 128165, | |
| 128166, | |
| 128167, | |
| 128168, | |
| 128169, | |
| 128170, | |
| 128171, | |
| 128172, | |
| 128173, | |
| 128174, | |
| 128175, | |
| 128176, | |
| 128177, | |
| 128178, | |
| 128179, | |
| 128180, | |
| 128181, | |
| 128182, | |
| 128183, | |
| 128184, | |
| 128185, | |
| 128186, | |
| 128187, | |
| 128188, | |
| 128189, | |
| 128190, | |
| 128191, | |
| 128192, | |
| 128193, | |
| 128194, | |
| 128195, | |
| 128196, | |
| 128197, | |
| 128198, | |
| 128199, | |
| 128200, | |
| 128201, | |
| 128202, | |
| 128203, | |
| 128204, | |
| 128205, | |
| 128206, | |
| 128207, | |
| 128208, | |
| 128209, | |
| 128210, | |
| 128211, | |
| 128212, | |
| 128213, | |
| 128214, | |
| 128215, | |
| 128216, | |
| 128217, | |
| 128218, | |
| 128219, | |
| 128220, | |
| 128221, | |
| 128222, | |
| 128223, | |
| 128224, | |
| 128225, | |
| 128226, | |
| 128227, | |
| 128228, | |
| 128229, | |
| 128230, | |
| 128231, | |
| 128232, | |
| 128233, | |
| 128234, | |
| 128235, | |
| 128236, | |
| 128237, | |
| 128238, | |
| 128239, | |
| 128240, | |
| 128241, | |
| 128242, | |
| 128243, | |
| 128244, | |
| 128245, | |
| 128246, | |
| 128247, | |
| 128248, | |
| 128249, | |
| 128250, | |
| 128251, | |
| 128252, | |
| 128253, | |
| 128254, | |
| 128255 | |
| ], | |
| "initial_vocab_size": 128256, | |
| "max_codebook_size": 2048, | |
| "max_subtokens": 4 | |
| }, | |
| "encoder": { | |
| "hidden_size": 4096, | |
| "intermediate_size": null, | |
| "num_heads": 32, | |
| "num_hidden_layers": 2, | |
| "position_encoding": "learnable", | |
| "tie_encoders": true | |
| }, | |
| "encoder_type": "transformer" | |
| } |