models:
  
  - model: Steelskull/L3.3-MS-Nevoria-70b
    parameters:
      weight:  
        - filter: self_attn.q_proj
          value: [1,1,1,1,1,1,1,1,1,0.9,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.9,1,1,1,1,1,1]
        - filter: self_attn.k_proj
          value: [1,1,1,1,1,1,1,1,1,0.9,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.9,1,1,1,1,1,1]
        - filter: self_attn.v_proj
          value: 1
        - filter: self_attn.o_proj
          value: 1 
        - filter: mlp.*_proj
          value: [1,1,1,1,1,1,1,1,1,1,1,1,0.9,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.9,1,1,1]
        - filter: input_layernorm
          value: 1
        - filter: post_attention_layernorm
          value: 1
        - filter: embed_tokens
          value: 0.9
        - filter: model.norm
          value: 1 
        - filter: lm_head
          value: 1
        - value: 1
      density: 0.8

  - model: Writer/Palmyra-Med-70B-32K
    parameters:
      weight: 
        - filter: self_attn.q_proj
          value: [0,0,0,0,0,0,0,0,0,0.1,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.1,0,0,0,0,0,0]
        - filter: self_attn.k_proj
          value: [0,0,0,0,0,0,0,0,0,0.1,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.1,0,0,0,0,0,0]
        - filter: self_attn.v_proj
          value: 0
        - filter: self_attn.o_proj
          value: 0
        - filter: mlp.*_proj
          value: [0,0,0,0,0,0,0,0,0,0,0,0,0.1,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.1,0,0,0]
        - filter: input_layernorm
          value: 0.0
        - filter: post_attention_layernorm
          value: 0.0
        - filter: embed_tokens
          value: 0.1
        - filter: model.norm
          value: 0
        - filter: lm_head
          value: 0
        - value: 0
      density: 0.2

merge_method: dare_ties
base_model: Steelskull/L3.3-MS-Nevoria-70b
parameters:    
    normalize: false
dtype: float32
out_dtype: bfloat16
chat_template: llama3
tokenizer:
    source: Steelskull/L3.3-MS-Nevoria-70b