Upload 2 files

ncnn/RWKV-x070-World-0.1B-v2.8-20241210-ctx4096-ncnn.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c7dab7c856261b3f2fe33de419bfa87f957b5cd29ee305f01247158c5e7a50c9
+size 382180960

ncnn/RWKV-x070-World-0.1B-v2.8-20241210-ctx4096-ncnn.param

@@ -0,0 +1,1198 @@
+7767517
+1196 1471
+MemoryData model_info 0 1 model_info 0=5 21=1
+Input input_0 0 1 token 0=1 1=1 2=1
+Input input_1 0 1 state_0_in 0=768
+Input input_2 0 1 state_1_in 0=64 1=64 2=12
+Input input_3 0 1 state_2_in 0=768
+Input input_4 0 1 state_3_in 0=768
+Input input_5 0 1 state_4_in 0=64 1=64 2=12
+Input input_6 0 1 state_5_in 0=768
+Input input_7 0 1 state_6_in 0=768
+Input input_8 0 1 state_7_in 0=64 1=64 2=12
+Input input_9 0 1 state_8_in 0=768
+Input input_10 0 1 state_9_in 0=768
+Input input_11 0 1 state_10_in 0=64 1=64 2=12
+Input input_12 0 1 state_11_in 0=768
+Input input_13 0 1 state_12_in 0=768
+Input input_14 0 1 state_13_in 0=64 1=64 2=12
+Input input_15 0 1 state_14_in 0=768
+Input input_16 0 1 state_15_in 0=768
+Input input_17 0 1 state_16_in 0=64 1=64 2=12
+Input input_18 0 1 state_17_in 0=768
+Input input_19 0 1 state_18_in 0=768
+Input input_20 0 1 state_19_in 0=64 1=64 2=12
+Input input_21 0 1 state_20_in 0=768
+Input input_22 0 1 state_21_in 0=768
+Input input_23 0 1 state_22_in 0=64 1=64 2=12
+Input input_24 0 1 state_23_in 0=768
+Input input_25 0 1 state_24_in 0=768
+Input input_26 0 1 state_25_in 0=64 1=64 2=12
+Input input_27 0 1 state_26_in 0=768
+Input input_28 0 1 state_27_in 0=768
+Input input_29 0 1 state_28_in 0=64 1=64 2=12
+Input input_30 0 1 state_29_in 0=768
+Input input_31 0 1 state_30_in 0=768
+Input input_32 0 1 state_31_in 0=64 1=64 2=12
+Input input_33 0 1 state_32_in 0=768
+Input input_34 0 1 state_33_in 0=768
+Input input_35 0 1 state_34_in 0=64 1=64 2=12
+Input input_36 0 1 state_35_in 0=768
+Embed embedding 1 1 token emb 0=768 1=65536 3=50331648
+Split split_emb 1 2 emb att_0_x_last att_0_x
+LayerNorm layernorm_0 1 1 att_0_x att_0_xx 0=768 1=0.00001 2=1
+Split split_att_0_xx 1 3 att_0_xx att_0_xx_0 att_0_xx_1 state_0_out
+BinaryOp sub_0 2 1 state_0_in att_0_xx_0 att_0_sx 0=1
+MemoryData data_att_0_maa 0 1 att_0_maa 0=768 1=6 21=0
+BinaryOp mul_0 2 1 att_0_sx att_0_maa att_0_maa_x 0=2
+BinaryOp add_0 2 1 att_0_xx_1 att_0_maa_x att_0_maa_rwkvag 0=0
+Slice slice_att_0_maa_rwkvag 1 6 att_0_maa_rwkvag att_0_mr att_0_mw att_0_mk att_0_mv att_0_ma att_0_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
+Split split_att_0_mv 1 2 att_0_mv att_0_mv_0 att_0_mv_1
+Gemm gemm_att_0_key 1 1 att_0_mk att_0_key 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_0_value 1 1 att_0_mv_0 att_0_value 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_0_receptance 1 1 att_0_mr att_0_receptance 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_0_ma_lora_0 1 1 att_0_ma att_0_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+Gemm gemm_att_0_ma_lora_1 1 1 att_0_ma_lora_0 att_0_ma_lora_1 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_0_a0 0 1 att_0_a0 0=768 21=0
+BinaryOp add_1 2 1 att_0_ma_lora_1 att_0_a0 att_0_ma_lora_2 0=0
+Sigmoid sigmoid_0 1 1 att_0_ma_lora_2 att_0_a
+Split split_att_0_a 1 2 att_0_a att_0_a_0 att_0_a_1
+Gemm gemm_att_0_mg_lora 1 1 att_0_mg att_0_mg_lora 4=0 5=1 6=0 7=0 8=128 9=768 10=-1
+Sigmoid sigmoid_1 1 1 att_0_mg_lora att_0_mg_lora_sigmoid
+Gemm gemm_att_0_gate 1 1 att_0_mg_lora_sigmoid att_0_gate 4=0 5=1 6=0 7=0 8=768 9=128 10=-1
+MemoryData data_att_0_k_k 0 1 att_0_k_k 0=768 21=0
+MemoryData data_att_0_k_a 0 1 att_0_k_a 0=768 21=0
+MemoryData data_att_0_r_k 0 1 att_0_r_k 0=768 21=0
+Split split_att_0_key 1 2 att_0_key att_0_key_0 att_0_key_1
+BinaryOp mul_1 2 1 att_0_key_0 att_0_k_k att_0_key_k 0=2
+Reshape reshape_0 1 1 att_0_key_k att_0_key_k_reshape 0=64 1=1 2=12
+Normalize l2norm_0 1 1 att_0_key_k_reshape att_0_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
+Reshape reshape_1 1 1 att_0_key_k_norm att_0_key_k_norm_reshape 0=768
+BinaryOp add_2 1 1 att_0_a_0 att_0_a_0_minus 0=0 1=1 2=-1.0
+BinaryOp mul_2 2 1 att_0_a_0_minus att_0_k_a att_0_a_minus1_mulka 0=2
+BinaryOp add_3 1 1 att_0_a_minus1_mulka att_0_a_minus1_mulka_plus1 0=0 1=1 2=1.0
+BinaryOp mul_3 2 1 att_0_a_minus1_mulka_plus1 att_0_key_1 att_0_key_a 0=2
+Split split_att_0_value 1 2 att_0_value att_0_value_final v_first_0
+Gemm gemm_att_0_mw_lora 1 1 att_0_mw att_0_mw_lora 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+TanH tanh_att_0_mw_lora 1 1 att_0_mw_lora att_0_mw_lora_tanh
+Gemm gemm_att_0_mw_lora_tanh_linear 1 1 att_0_mw_lora_tanh att_0_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_0_td 0 1 att_0_td 0=768 21=0
+BinaryOp add_4 2 1 att_0_mw_lora_tanh_linear att_0_td att_0_time_decay_pre 0=0
+Sigmoid sigmoid_2 1 1 att_0_time_decay_pre att_0_time_decay_sigmoid
+Exp exp_0 1 1 att_0_time_decay_sigmoid att_0_time_decay 1=-0.606531
+Reshape reshape_2 1 1 att_0_time_decay att_0_time_decay_reshape 0=64 1=1 2=12
+Split split_att_0_key_a 1 2 att_0_key_a att_0_key_a_0 att_0_key_a_1
+Reshape reshape_3 1 1 att_0_key_a_0 att_0_key_a_reshape 0=64 1=1 2=12
+Split split_att_0_value_final 1 3 att_0_value_final att_0_value_final_0 att_0_value_final_1 att_0_value_final_2
+Reshape reshape_4 1 1 att_0_value_final_0 att_0_value_final_0_reshape 0=64 1=12
+Reshape reshape_5 1 1 att_0_value_final_1 att_0_value_final_1_reshape 0=1 1=64 2=12
+MatMul matmul_att_0_vk 2 1 att_0_value_final_1_reshape att_0_key_a_reshape att_0_vk 0=0
+Split split_att_0_key_k_norm_reshape 1 2 att_0_key_k_norm_reshape att_0_kk_0 att_0_kk_1
+BinaryOp mul_4 1 1 att_0_kk_0 att_0_kk_0_neg 0=2 1=1 2=-1.0
+Reshape reshape_6 1 1 att_0_kk_0_neg att_0_kk_0_neg_reshape 0=1 1=64 2=12
+BinaryOp mul_5 2 1 att_0_kk_1 att_0_a_1 att_0_kk_a 0=2
+Reshape reshape_7 1 1 att_0_kk_a att_0_kk_a_reshape 0=64 1=1 2=12
+MatMul matmul_att_0_ab 2 1 att_0_kk_0_neg_reshape att_0_kk_a_reshape att_0_ab 0=0
+Split split_state_1_in 1 2 state_1_in att_0_state_prev_0 att_0_state_prev_1
+BinaryOp mul_6 2 1 att_0_state_prev_0 att_0_time_decay_reshape att_0_state_td 0=2
+MatMul matmul_att_0_sab 2 1 att_0_state_prev_1 att_0_ab att_0_sab 0=0
+BinaryOp add_5 2 1 att_0_sab att_0_vk att_0_sab_vk 0=0
+BinaryOp add_6 2 1 att_0_sab_vk att_0_state_td att_0_state_new 0=0
+Split split_att_0_state_new 1 2 att_0_state_new att_0_state_new_0 state_1_out
+Split split_att_0_receptance 1 2 att_0_receptance att_0_receptance_0 att_0_receptance_1
+Reshape reshape_8 1 1 att_0_receptance_0 att_0_receptance_reshape 0=1 1=64 2=12
+MatMul matmul_att_0_wkv_out 2 1 att_0_state_new_0 att_0_receptance_reshape att_0_wkv_out 0=0
+Reshape reshape_9 1 1 att_0_wkv_out att_0_wkv_out_flatten 0=768
+GroupNorm groupnorm_0 1 1 att_0_wkv_out_flatten att_0_x_gn 0=12 1=768 2=0.00064 3=1
+BinaryOp mul_7 2 1 att_0_receptance_1 att_0_key_a_1 att_0_receptance_key_a 0=2
+BinaryOp mul_8 2 1 att_0_receptance_key_a att_0_r_k att_0_rk 0=2
+Reshape reshape_10 1 1 att_0_rk att_0_rk_reshape 0=64 1=12
+Reduction sum_0 1 1 att_0_rk_reshape att_0_rk_sum 1=0 -23303=1,-1 4=1 5=1
+BinaryOp mul_9 2 1 att_0_value_final_0_reshape att_0_rk_sum att_0_value_final_0_rk 0=2
+Reshape reshape_11 1 1 att_0_value_final_0_rk att_0_value_final_0_rk_reshape 0=768
+BinaryOp add_7 2 1 att_0_value_final_0_rk_reshape att_0_x_gn att_0_x_gn_rkv 0=0
+BinaryOp mul_10 2 1 att_0_x_gn_rkv att_0_gate att_0_x_gate 0=2
+Gemm gemm_att_0_x_out 1 1 att_0_x_gate att_0_x_out 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+BinaryOp add_8 2 1 att_0_x_out att_0_x_last time_mixing_0_out 0=0
+Split split_time_mixing_0_out 1 2 time_mixing_0_out ffn_0_x_last ffn_0_x
+LayerNorm layernorm_1 1 1 ffn_0_x ffn_0_xx 0=768 1=0.00001 2=1
+Split split_ffn_0_xx 1 3 ffn_0_xx ffn_0_xx_0 ffn_0_xx_1 state_2_out
+BinaryOp sub_1 2 1 state_2_in ffn_0_xx_0 ffn_0_sx 0=1
+Split split_ffn_0_sx 1 1 ffn_0_sx ffn_0_sx_0
+MemoryData data_ffn_0_x_k 0 1 ffn_0_x_k 0=768 21=0
+BinaryOp mul_11 2 1 ffn_0_sx_0 ffn_0_x_k ffn_0_xk 0=2
+BinaryOp add_9 2 1 ffn_0_xk ffn_0_xx_1 ffn_0_xxk 0=0
+Gemm gemm_ffn_0_key 1 1 ffn_0_xxk ffn_0_key 4=0 5=1 6=0 7=0 8=3072 9=768 10=-1
+ReLU relu_0 1 1 ffn_0_key ffn_0_key_relu
+UnaryOp square_ffn_0_key_relu 1 1 ffn_0_key_relu ffn_0_key_relu_square 0=4
+Gemm gemm_ffn_0_value 1 1 ffn_0_key_relu_square ffn_0_value 4=0 5=1 6=0 7=0 8=768 9=3072 10=-1
+BinaryOp add_10 2 1 ffn_0_value ffn_0_x_last channel_mixing_0_out 0=0
+Split split_channel_mixing_0_out 1 2 channel_mixing_0_out att_1_x_last att_1_x
+LayerNorm layernorm_2 1 1 att_1_x att_1_xx 0=768 1=0.00001 2=1
+Split split_att_1_xx 1 3 att_1_xx att_1_xx_0 att_1_xx_1 state_3_out
+BinaryOp sub_2 2 1 state_3_in att_1_xx_0 att_1_sx 0=1
+MemoryData data_att_1_maa 0 1 att_1_maa 0=768 1=6 21=0
+BinaryOp mul_12 2 1 att_1_sx att_1_maa att_1_maa_x 0=2
+BinaryOp add_11 2 1 att_1_xx_1 att_1_maa_x att_1_maa_rwkvag 0=0
+Slice slice_att_1_maa_rwkvag 1 6 att_1_maa_rwkvag att_1_mr att_1_mw att_1_mk att_1_mv att_1_ma att_1_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
+Split split_att_1_mv 1 2 att_1_mv att_1_mv_0 att_1_mv_1
+Gemm gemm_att_1_key 1 1 att_1_mk att_1_key 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_1_value 1 1 att_1_mv_0 att_1_value 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_1_receptance 1 1 att_1_mr att_1_receptance 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_1_ma_lora_0 1 1 att_1_ma att_1_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+Gemm gemm_att_1_ma_lora_1 1 1 att_1_ma_lora_0 att_1_ma_lora_1 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_1_a0 0 1 att_1_a0 0=768 21=0
+BinaryOp add_12 2 1 att_1_ma_lora_1 att_1_a0 att_1_ma_lora_2 0=0
+Sigmoid sigmoid_3 1 1 att_1_ma_lora_2 att_1_a
+Split split_att_1_a 1 2 att_1_a att_1_a_0 att_1_a_1
+Gemm gemm_att_1_mg_lora 1 1 att_1_mg att_1_mg_lora 4=0 5=1 6=0 7=0 8=128 9=768 10=-1
+Sigmoid sigmoid_4 1 1 att_1_mg_lora att_1_mg_lora_sigmoid
+Gemm gemm_att_1_gate 1 1 att_1_mg_lora_sigmoid att_1_gate 4=0 5=1 6=0 7=0 8=768 9=128 10=-1
+MemoryData data_att_1_k_k 0 1 att_1_k_k 0=768 21=0
+MemoryData data_att_1_k_a 0 1 att_1_k_a 0=768 21=0
+MemoryData data_att_1_r_k 0 1 att_1_r_k 0=768 21=0
+Split split_att_1_key 1 2 att_1_key att_1_key_0 att_1_key_1
+BinaryOp mul_13 2 1 att_1_key_0 att_1_k_k att_1_key_k 0=2
+Reshape reshape_12 1 1 att_1_key_k att_1_key_k_reshape 0=64 1=1 2=12
+Normalize l2norm_1 1 1 att_1_key_k_reshape att_1_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
+Reshape reshape_13 1 1 att_1_key_k_norm att_1_key_k_norm_reshape 0=768
+BinaryOp add_13 1 1 att_1_a_0 att_1_a_0_minus 0=0 1=1 2=-1.0
+BinaryOp mul_14 2 1 att_1_a_0_minus att_1_k_a att_1_a_minus1_mulka 0=2
+BinaryOp add_14 1 1 att_1_a_minus1_mulka att_1_a_minus1_mulka_plus1 0=0 1=1 2=1.0
+BinaryOp mul_15 2 1 att_1_a_minus1_mulka_plus1 att_1_key_1 att_1_key_a 0=2
+Split split_att_1_value 1 2 att_1_value att_1_value_0 att_1_value_1
+Split split_v_first_0 1 2 v_first_0 att_1_v_first v_first_1
+BinaryOp minus_0 2 1 att_1_v_first att_1_value_1 att_1_vfirst_minus_value 0=1
+Gemm gemm_att_1_mv_lora_0 1 1 att_1_mv_1 att_1_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=768 10=-1
+Gemm gemm_att_1_mv_lora_1 1 1 att_1_mv_lora_0 att_1_mv_lora_1 4=0 5=1 6=0 7=0 8=768 9=32 10=-1
+MemoryData data_att_1_v0 0 1 att_1_v0 0=768 21=0
+BinaryOp add_15 2 1 att_1_mv_lora_1 att_1_v0 att_1_mv_lora_2 0=0
+Sigmoid sigmoid_5 1 1 att_1_mv_lora_2 att_1_mv_lora_sigmoid
+BinaryOp mul_16 2 1 att_1_mv_lora_sigmoid att_1_vfirst_minus_value att_1_vfirst_minus_value_mul 0=2
+BinaryOp add_16 2 1 att_1_value_0 att_1_vfirst_minus_value_mul att_1_value_final 0=0
+Gemm gemm_att_1_mw_lora 1 1 att_1_mw att_1_mw_lora 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+TanH tanh_att_1_mw_lora 1 1 att_1_mw_lora att_1_mw_lora_tanh
+Gemm gemm_att_1_mw_lora_tanh_linear 1 1 att_1_mw_lora_tanh att_1_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_1_td 0 1 att_1_td 0=768 21=0
+BinaryOp add_17 2 1 att_1_mw_lora_tanh_linear att_1_td att_1_time_decay_pre 0=0
+Sigmoid sigmoid_6 1 1 att_1_time_decay_pre att_1_time_decay_sigmoid
+Exp exp_1 1 1 att_1_time_decay_sigmoid att_1_time_decay 1=-0.606531
+Reshape reshape_14 1 1 att_1_time_decay att_1_time_decay_reshape 0=64 1=1 2=12
+Split split_att_1_key_a 1 2 att_1_key_a att_1_key_a_0 att_1_key_a_1
+Reshape reshape_15 1 1 att_1_key_a_0 att_1_key_a_reshape 0=64 1=1 2=12
+Split split_att_1_value_final 1 3 att_1_value_final att_1_value_final_0 att_1_value_final_1 att_1_value_final_2
+Reshape reshape_16 1 1 att_1_value_final_0 att_1_value_final_0_reshape 0=64 1=12
+Reshape reshape_17 1 1 att_1_value_final_1 att_1_value_final_1_reshape 0=1 1=64 2=12
+MatMul matmul_att_1_vk 2 1 att_1_value_final_1_reshape att_1_key_a_reshape att_1_vk 0=0
+Split split_att_1_key_k_norm_reshape 1 2 att_1_key_k_norm_reshape att_1_kk_0 att_1_kk_1
+BinaryOp mul_17 1 1 att_1_kk_0 att_1_kk_0_neg 0=2 1=1 2=-1.0
+Reshape reshape_18 1 1 att_1_kk_0_neg att_1_kk_0_neg_reshape 0=1 1=64 2=12
+BinaryOp mul_18 2 1 att_1_kk_1 att_1_a_1 att_1_kk_a 0=2
+Reshape reshape_19 1 1 att_1_kk_a att_1_kk_a_reshape 0=64 1=1 2=12
+MatMul matmul_att_1_ab 2 1 att_1_kk_0_neg_reshape att_1_kk_a_reshape att_1_ab 0=0
+Split split_state_4_in 1 2 state_4_in att_1_state_prev_0 att_1_state_prev_1
+BinaryOp mul_19 2 1 att_1_state_prev_0 att_1_time_decay_reshape att_1_state_td 0=2
+MatMul matmul_att_1_sab 2 1 att_1_state_prev_1 att_1_ab att_1_sab 0=0
+BinaryOp add_18 2 1 att_1_sab att_1_vk att_1_sab_vk 0=0
+BinaryOp add_19 2 1 att_1_sab_vk att_1_state_td att_1_state_new 0=0
+Split split_att_1_state_new 1 2 att_1_state_new att_1_state_new_0 state_4_out
+Split split_att_1_receptance 1 2 att_1_receptance att_1_receptance_0 att_1_receptance_1
+Reshape reshape_20 1 1 att_1_receptance_0 att_1_receptance_reshape 0=1 1=64 2=12
+MatMul matmul_att_1_wkv_out 2 1 att_1_state_new_0 att_1_receptance_reshape att_1_wkv_out 0=0
+Reshape reshape_21 1 1 att_1_wkv_out att_1_wkv_out_flatten 0=768
+GroupNorm groupnorm_1 1 1 att_1_wkv_out_flatten att_1_x_gn 0=12 1=768 2=0.00064 3=1
+BinaryOp mul_20 2 1 att_1_receptance_1 att_1_key_a_1 att_1_receptance_key_a 0=2
+BinaryOp mul_21 2 1 att_1_receptance_key_a att_1_r_k att_1_rk 0=2
+Reshape reshape_22 1 1 att_1_rk att_1_rk_reshape 0=64 1=12
+Reduction sum_1 1 1 att_1_rk_reshape att_1_rk_sum 1=0 -23303=1,-1 4=1 5=1
+BinaryOp mul_22 2 1 att_1_value_final_0_reshape att_1_rk_sum att_1_value_final_0_rk 0=2
+Reshape reshape_23 1 1 att_1_value_final_0_rk att_1_value_final_0_rk_reshape 0=768
+BinaryOp add_20 2 1 att_1_value_final_0_rk_reshape att_1_x_gn att_1_x_gn_rkv 0=0
+BinaryOp mul_23 2 1 att_1_x_gn_rkv att_1_gate att_1_x_gate 0=2
+Gemm gemm_att_1_x_out 1 1 att_1_x_gate att_1_x_out 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+BinaryOp add_21 2 1 att_1_x_out att_1_x_last time_mixing_1_out 0=0
+Split split_time_mixing_1_out 1 2 time_mixing_1_out ffn_1_x_last ffn_1_x
+LayerNorm layernorm_3 1 1 ffn_1_x ffn_1_xx 0=768 1=0.00001 2=1
+Split split_ffn_1_xx 1 3 ffn_1_xx ffn_1_xx_0 ffn_1_xx_1 state_5_out
+BinaryOp sub_3 2 1 state_5_in ffn_1_xx_0 ffn_1_sx 0=1
+Split split_ffn_1_sx 1 1 ffn_1_sx ffn_1_sx_0
+MemoryData data_ffn_1_x_k 0 1 ffn_1_x_k 0=768 21=0
+BinaryOp mul_24 2 1 ffn_1_sx_0 ffn_1_x_k ffn_1_xk 0=2
+BinaryOp add_22 2 1 ffn_1_xk ffn_1_xx_1 ffn_1_xxk 0=0
+Gemm gemm_ffn_1_key 1 1 ffn_1_xxk ffn_1_key 4=0 5=1 6=0 7=0 8=3072 9=768 10=-1
+ReLU relu_1 1 1 ffn_1_key ffn_1_key_relu
+UnaryOp square_ffn_1_key_relu 1 1 ffn_1_key_relu ffn_1_key_relu_square 0=4
+Gemm gemm_ffn_1_value 1 1 ffn_1_key_relu_square ffn_1_value 4=0 5=1 6=0 7=0 8=768 9=3072 10=-1
+BinaryOp add_23 2 1 ffn_1_value ffn_1_x_last channel_mixing_1_out 0=0
+Split split_channel_mixing_1_out 1 2 channel_mixing_1_out att_2_x_last att_2_x
+LayerNorm layernorm_4 1 1 att_2_x att_2_xx 0=768 1=0.00001 2=1
+Split split_att_2_xx 1 3 att_2_xx att_2_xx_0 att_2_xx_1 state_6_out
+BinaryOp sub_4 2 1 state_6_in att_2_xx_0 att_2_sx 0=1
+MemoryData data_att_2_maa 0 1 att_2_maa 0=768 1=6 21=0
+BinaryOp mul_25 2 1 att_2_sx att_2_maa att_2_maa_x 0=2
+BinaryOp add_24 2 1 att_2_xx_1 att_2_maa_x att_2_maa_rwkvag 0=0
+Slice slice_att_2_maa_rwkvag 1 6 att_2_maa_rwkvag att_2_mr att_2_mw att_2_mk att_2_mv att_2_ma att_2_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
+Split split_att_2_mv 1 2 att_2_mv att_2_mv_0 att_2_mv_1
+Gemm gemm_att_2_key 1 1 att_2_mk att_2_key 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_2_value 1 1 att_2_mv_0 att_2_value 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_2_receptance 1 1 att_2_mr att_2_receptance 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_2_ma_lora_0 1 1 att_2_ma att_2_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+Gemm gemm_att_2_ma_lora_1 1 1 att_2_ma_lora_0 att_2_ma_lora_1 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_2_a0 0 1 att_2_a0 0=768 21=0
+BinaryOp add_25 2 1 att_2_ma_lora_1 att_2_a0 att_2_ma_lora_2 0=0
+Sigmoid sigmoid_7 1 1 att_2_ma_lora_2 att_2_a
+Split split_att_2_a 1 2 att_2_a att_2_a_0 att_2_a_1
+Gemm gemm_att_2_mg_lora 1 1 att_2_mg att_2_mg_lora 4=0 5=1 6=0 7=0 8=128 9=768 10=-1
+Sigmoid sigmoid_8 1 1 att_2_mg_lora att_2_mg_lora_sigmoid
+Gemm gemm_att_2_gate 1 1 att_2_mg_lora_sigmoid att_2_gate 4=0 5=1 6=0 7=0 8=768 9=128 10=-1
+MemoryData data_att_2_k_k 0 1 att_2_k_k 0=768 21=0
+MemoryData data_att_2_k_a 0 1 att_2_k_a 0=768 21=0
+MemoryData data_att_2_r_k 0 1 att_2_r_k 0=768 21=0
+Split split_att_2_key 1 2 att_2_key att_2_key_0 att_2_key_1
+BinaryOp mul_26 2 1 att_2_key_0 att_2_k_k att_2_key_k 0=2
+Reshape reshape_24 1 1 att_2_key_k att_2_key_k_reshape 0=64 1=1 2=12
+Normalize l2norm_2 1 1 att_2_key_k_reshape att_2_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
+Reshape reshape_25 1 1 att_2_key_k_norm att_2_key_k_norm_reshape 0=768
+BinaryOp add_26 1 1 att_2_a_0 att_2_a_0_minus 0=0 1=1 2=-1.0
+BinaryOp mul_27 2 1 att_2_a_0_minus att_2_k_a att_2_a_minus1_mulka 0=2
+BinaryOp add_27 1 1 att_2_a_minus1_mulka att_2_a_minus1_mulka_plus1 0=0 1=1 2=1.0
+BinaryOp mul_28 2 1 att_2_a_minus1_mulka_plus1 att_2_key_1 att_2_key_a 0=2
+Split split_att_2_value 1 2 att_2_value att_2_value_0 att_2_value_1
+Split split_v_first_1 1 2 v_first_1 att_2_v_first v_first_2
+BinaryOp minus_1 2 1 att_2_v_first att_2_value_1 att_2_vfirst_minus_value 0=1
+Gemm gemm_att_2_mv_lora_0 1 1 att_2_mv_1 att_2_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=768 10=-1
+Gemm gemm_att_2_mv_lora_1 1 1 att_2_mv_lora_0 att_2_mv_lora_1 4=0 5=1 6=0 7=0 8=768 9=32 10=-1
+MemoryData data_att_2_v0 0 1 att_2_v0 0=768 21=0
+BinaryOp add_28 2 1 att_2_mv_lora_1 att_2_v0 att_2_mv_lora_2 0=0
+Sigmoid sigmoid_9 1 1 att_2_mv_lora_2 att_2_mv_lora_sigmoid
+BinaryOp mul_29 2 1 att_2_mv_lora_sigmoid att_2_vfirst_minus_value att_2_vfirst_minus_value_mul 0=2
+BinaryOp add_29 2 1 att_2_value_0 att_2_vfirst_minus_value_mul att_2_value_final 0=0
+Gemm gemm_att_2_mw_lora 1 1 att_2_mw att_2_mw_lora 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+TanH tanh_att_2_mw_lora 1 1 att_2_mw_lora att_2_mw_lora_tanh
+Gemm gemm_att_2_mw_lora_tanh_linear 1 1 att_2_mw_lora_tanh att_2_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_2_td 0 1 att_2_td 0=768 21=0
+BinaryOp add_30 2 1 att_2_mw_lora_tanh_linear att_2_td att_2_time_decay_pre 0=0
+Sigmoid sigmoid_10 1 1 att_2_time_decay_pre att_2_time_decay_sigmoid
+Exp exp_2 1 1 att_2_time_decay_sigmoid att_2_time_decay 1=-0.606531
+Reshape reshape_26 1 1 att_2_time_decay att_2_time_decay_reshape 0=64 1=1 2=12
+Split split_att_2_key_a 1 2 att_2_key_a att_2_key_a_0 att_2_key_a_1
+Reshape reshape_27 1 1 att_2_key_a_0 att_2_key_a_reshape 0=64 1=1 2=12
+Split split_att_2_value_final 1 3 att_2_value_final att_2_value_final_0 att_2_value_final_1 att_2_value_final_2
+Reshape reshape_28 1 1 att_2_value_final_0 att_2_value_final_0_reshape 0=64 1=12
+Reshape reshape_29 1 1 att_2_value_final_1 att_2_value_final_1_reshape 0=1 1=64 2=12
+MatMul matmul_att_2_vk 2 1 att_2_value_final_1_reshape att_2_key_a_reshape att_2_vk 0=0
+Split split_att_2_key_k_norm_reshape 1 2 att_2_key_k_norm_reshape att_2_kk_0 att_2_kk_1
+BinaryOp mul_30 1 1 att_2_kk_0 att_2_kk_0_neg 0=2 1=1 2=-1.0
+Reshape reshape_30 1 1 att_2_kk_0_neg att_2_kk_0_neg_reshape 0=1 1=64 2=12
+BinaryOp mul_31 2 1 att_2_kk_1 att_2_a_1 att_2_kk_a 0=2
+Reshape reshape_31 1 1 att_2_kk_a att_2_kk_a_reshape 0=64 1=1 2=12
+MatMul matmul_att_2_ab 2 1 att_2_kk_0_neg_reshape att_2_kk_a_reshape att_2_ab 0=0
+Split split_state_7_in 1 2 state_7_in att_2_state_prev_0 att_2_state_prev_1
+BinaryOp mul_32 2 1 att_2_state_prev_0 att_2_time_decay_reshape att_2_state_td 0=2
+MatMul matmul_att_2_sab 2 1 att_2_state_prev_1 att_2_ab att_2_sab 0=0
+BinaryOp add_31 2 1 att_2_sab att_2_vk att_2_sab_vk 0=0
+BinaryOp add_32 2 1 att_2_sab_vk att_2_state_td att_2_state_new 0=0
+Split split_att_2_state_new 1 2 att_2_state_new att_2_state_new_0 state_7_out
+Split split_att_2_receptance 1 2 att_2_receptance att_2_receptance_0 att_2_receptance_1
+Reshape reshape_32 1 1 att_2_receptance_0 att_2_receptance_reshape 0=1 1=64 2=12
+MatMul matmul_att_2_wkv_out 2 1 att_2_state_new_0 att_2_receptance_reshape att_2_wkv_out 0=0
+Reshape reshape_33 1 1 att_2_wkv_out att_2_wkv_out_flatten 0=768
+GroupNorm groupnorm_2 1 1 att_2_wkv_out_flatten att_2_x_gn 0=12 1=768 2=0.00064 3=1
+BinaryOp mul_33 2 1 att_2_receptance_1 att_2_key_a_1 att_2_receptance_key_a 0=2
+BinaryOp mul_34 2 1 att_2_receptance_key_a att_2_r_k att_2_rk 0=2
+Reshape reshape_34 1 1 att_2_rk att_2_rk_reshape 0=64 1=12
+Reduction sum_2 1 1 att_2_rk_reshape att_2_rk_sum 1=0 -23303=1,-1 4=1 5=1
+BinaryOp mul_35 2 1 att_2_value_final_0_reshape att_2_rk_sum att_2_value_final_0_rk 0=2
+Reshape reshape_35 1 1 att_2_value_final_0_rk att_2_value_final_0_rk_reshape 0=768
+BinaryOp add_33 2 1 att_2_value_final_0_rk_reshape att_2_x_gn att_2_x_gn_rkv 0=0
+BinaryOp mul_36 2 1 att_2_x_gn_rkv att_2_gate att_2_x_gate 0=2
+Gemm gemm_att_2_x_out 1 1 att_2_x_gate att_2_x_out 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+BinaryOp add_34 2 1 att_2_x_out att_2_x_last time_mixing_2_out 0=0
+Split split_time_mixing_2_out 1 2 time_mixing_2_out ffn_2_x_last ffn_2_x
+LayerNorm layernorm_5 1 1 ffn_2_x ffn_2_xx 0=768 1=0.00001 2=1
+Split split_ffn_2_xx 1 3 ffn_2_xx ffn_2_xx_0 ffn_2_xx_1 state_8_out
+BinaryOp sub_5 2 1 state_8_in ffn_2_xx_0 ffn_2_sx 0=1
+Split split_ffn_2_sx 1 1 ffn_2_sx ffn_2_sx_0
+MemoryData data_ffn_2_x_k 0 1 ffn_2_x_k 0=768 21=0
+BinaryOp mul_37 2 1 ffn_2_sx_0 ffn_2_x_k ffn_2_xk 0=2
+BinaryOp add_35 2 1 ffn_2_xk ffn_2_xx_1 ffn_2_xxk 0=0
+Gemm gemm_ffn_2_key 1 1 ffn_2_xxk ffn_2_key 4=0 5=1 6=0 7=0 8=3072 9=768 10=-1
+ReLU relu_2 1 1 ffn_2_key ffn_2_key_relu
+UnaryOp square_ffn_2_key_relu 1 1 ffn_2_key_relu ffn_2_key_relu_square 0=4
+Gemm gemm_ffn_2_value 1 1 ffn_2_key_relu_square ffn_2_value 4=0 5=1 6=0 7=0 8=768 9=3072 10=-1
+BinaryOp add_36 2 1 ffn_2_value ffn_2_x_last channel_mixing_2_out 0=0
+Split split_channel_mixing_2_out 1 2 channel_mixing_2_out att_3_x_last att_3_x
+LayerNorm layernorm_6 1 1 att_3_x att_3_xx 0=768 1=0.00001 2=1
+Split split_att_3_xx 1 3 att_3_xx att_3_xx_0 att_3_xx_1 state_9_out
+BinaryOp sub_6 2 1 state_9_in att_3_xx_0 att_3_sx 0=1
+MemoryData data_att_3_maa 0 1 att_3_maa 0=768 1=6 21=0
+BinaryOp mul_38 2 1 att_3_sx att_3_maa att_3_maa_x 0=2
+BinaryOp add_37 2 1 att_3_xx_1 att_3_maa_x att_3_maa_rwkvag 0=0
+Slice slice_att_3_maa_rwkvag 1 6 att_3_maa_rwkvag att_3_mr att_3_mw att_3_mk att_3_mv att_3_ma att_3_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
+Split split_att_3_mv 1 2 att_3_mv att_3_mv_0 att_3_mv_1
+Gemm gemm_att_3_key 1 1 att_3_mk att_3_key 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_3_value 1 1 att_3_mv_0 att_3_value 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_3_receptance 1 1 att_3_mr att_3_receptance 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_3_ma_lora_0 1 1 att_3_ma att_3_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+Gemm gemm_att_3_ma_lora_1 1 1 att_3_ma_lora_0 att_3_ma_lora_1 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_3_a0 0 1 att_3_a0 0=768 21=0
+BinaryOp add_38 2 1 att_3_ma_lora_1 att_3_a0 att_3_ma_lora_2 0=0
+Sigmoid sigmoid_11 1 1 att_3_ma_lora_2 att_3_a
+Split split_att_3_a 1 2 att_3_a att_3_a_0 att_3_a_1
+Gemm gemm_att_3_mg_lora 1 1 att_3_mg att_3_mg_lora 4=0 5=1 6=0 7=0 8=128 9=768 10=-1
+Sigmoid sigmoid_12 1 1 att_3_mg_lora att_3_mg_lora_sigmoid
+Gemm gemm_att_3_gate 1 1 att_3_mg_lora_sigmoid att_3_gate 4=0 5=1 6=0 7=0 8=768 9=128 10=-1
+MemoryData data_att_3_k_k 0 1 att_3_k_k 0=768 21=0
+MemoryData data_att_3_k_a 0 1 att_3_k_a 0=768 21=0
+MemoryData data_att_3_r_k 0 1 att_3_r_k 0=768 21=0
+Split split_att_3_key 1 2 att_3_key att_3_key_0 att_3_key_1
+BinaryOp mul_39 2 1 att_3_key_0 att_3_k_k att_3_key_k 0=2
+Reshape reshape_36 1 1 att_3_key_k att_3_key_k_reshape 0=64 1=1 2=12
+Normalize l2norm_3 1 1 att_3_key_k_reshape att_3_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
+Reshape reshape_37 1 1 att_3_key_k_norm att_3_key_k_norm_reshape 0=768
+BinaryOp add_39 1 1 att_3_a_0 att_3_a_0_minus 0=0 1=1 2=-1.0
+BinaryOp mul_40 2 1 att_3_a_0_minus att_3_k_a att_3_a_minus1_mulka 0=2
+BinaryOp add_40 1 1 att_3_a_minus1_mulka att_3_a_minus1_mulka_plus1 0=0 1=1 2=1.0
+BinaryOp mul_41 2 1 att_3_a_minus1_mulka_plus1 att_3_key_1 att_3_key_a 0=2
+Split split_att_3_value 1 2 att_3_value att_3_value_0 att_3_value_1
+Split split_v_first_2 1 2 v_first_2 att_3_v_first v_first_3
+BinaryOp minus_2 2 1 att_3_v_first att_3_value_1 att_3_vfirst_minus_value 0=1
+Gemm gemm_att_3_mv_lora_0 1 1 att_3_mv_1 att_3_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=768 10=-1
+Gemm gemm_att_3_mv_lora_1 1 1 att_3_mv_lora_0 att_3_mv_lora_1 4=0 5=1 6=0 7=0 8=768 9=32 10=-1
+MemoryData data_att_3_v0 0 1 att_3_v0 0=768 21=0
+BinaryOp add_41 2 1 att_3_mv_lora_1 att_3_v0 att_3_mv_lora_2 0=0
+Sigmoid sigmoid_13 1 1 att_3_mv_lora_2 att_3_mv_lora_sigmoid
+BinaryOp mul_42 2 1 att_3_mv_lora_sigmoid att_3_vfirst_minus_value att_3_vfirst_minus_value_mul 0=2
+BinaryOp add_42 2 1 att_3_value_0 att_3_vfirst_minus_value_mul att_3_value_final 0=0
+Gemm gemm_att_3_mw_lora 1 1 att_3_mw att_3_mw_lora 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+TanH tanh_att_3_mw_lora 1 1 att_3_mw_lora att_3_mw_lora_tanh
+Gemm gemm_att_3_mw_lora_tanh_linear 1 1 att_3_mw_lora_tanh att_3_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_3_td 0 1 att_3_td 0=768 21=0
+BinaryOp add_43 2 1 att_3_mw_lora_tanh_linear att_3_td att_3_time_decay_pre 0=0
+Sigmoid sigmoid_14 1 1 att_3_time_decay_pre att_3_time_decay_sigmoid
+Exp exp_3 1 1 att_3_time_decay_sigmoid att_3_time_decay 1=-0.606531
+Reshape reshape_38 1 1 att_3_time_decay att_3_time_decay_reshape 0=64 1=1 2=12
+Split split_att_3_key_a 1 2 att_3_key_a att_3_key_a_0 att_3_key_a_1
+Reshape reshape_39 1 1 att_3_key_a_0 att_3_key_a_reshape 0=64 1=1 2=12
+Split split_att_3_value_final 1 3 att_3_value_final att_3_value_final_0 att_3_value_final_1 att_3_value_final_2
+Reshape reshape_40 1 1 att_3_value_final_0 att_3_value_final_0_reshape 0=64 1=12
+Reshape reshape_41 1 1 att_3_value_final_1 att_3_value_final_1_reshape 0=1 1=64 2=12
+MatMul matmul_att_3_vk 2 1 att_3_value_final_1_reshape att_3_key_a_reshape att_3_vk 0=0
+Split split_att_3_key_k_norm_reshape 1 2 att_3_key_k_norm_reshape att_3_kk_0 att_3_kk_1
+BinaryOp mul_43 1 1 att_3_kk_0 att_3_kk_0_neg 0=2 1=1 2=-1.0
+Reshape reshape_42 1 1 att_3_kk_0_neg att_3_kk_0_neg_reshape 0=1 1=64 2=12
+BinaryOp mul_44 2 1 att_3_kk_1 att_3_a_1 att_3_kk_a 0=2
+Reshape reshape_43 1 1 att_3_kk_a att_3_kk_a_reshape 0=64 1=1 2=12
+MatMul matmul_att_3_ab 2 1 att_3_kk_0_neg_reshape att_3_kk_a_reshape att_3_ab 0=0
+Split split_state_10_in 1 2 state_10_in att_3_state_prev_0 att_3_state_prev_1
+BinaryOp mul_45 2 1 att_3_state_prev_0 att_3_time_decay_reshape att_3_state_td 0=2
+MatMul matmul_att_3_sab 2 1 att_3_state_prev_1 att_3_ab att_3_sab 0=0
+BinaryOp add_44 2 1 att_3_sab att_3_vk att_3_sab_vk 0=0
+BinaryOp add_45 2 1 att_3_sab_vk att_3_state_td att_3_state_new 0=0
+Split split_att_3_state_new 1 2 att_3_state_new att_3_state_new_0 state_10_out
+Split split_att_3_receptance 1 2 att_3_receptance att_3_receptance_0 att_3_receptance_1
+Reshape reshape_44 1 1 att_3_receptance_0 att_3_receptance_reshape 0=1 1=64 2=12
+MatMul matmul_att_3_wkv_out 2 1 att_3_state_new_0 att_3_receptance_reshape att_3_wkv_out 0=0
+Reshape reshape_45 1 1 att_3_wkv_out att_3_wkv_out_flatten 0=768
+GroupNorm groupnorm_3 1 1 att_3_wkv_out_flatten att_3_x_gn 0=12 1=768 2=0.00064 3=1
+BinaryOp mul_46 2 1 att_3_receptance_1 att_3_key_a_1 att_3_receptance_key_a 0=2
+BinaryOp mul_47 2 1 att_3_receptance_key_a att_3_r_k att_3_rk 0=2
+Reshape reshape_46 1 1 att_3_rk att_3_rk_reshape 0=64 1=12
+Reduction sum_3 1 1 att_3_rk_reshape att_3_rk_sum 1=0 -23303=1,-1 4=1 5=1
+BinaryOp mul_48 2 1 att_3_value_final_0_reshape att_3_rk_sum att_3_value_final_0_rk 0=2
+Reshape reshape_47 1 1 att_3_value_final_0_rk att_3_value_final_0_rk_reshape 0=768
+BinaryOp add_46 2 1 att_3_value_final_0_rk_reshape att_3_x_gn att_3_x_gn_rkv 0=0
+BinaryOp mul_49 2 1 att_3_x_gn_rkv att_3_gate att_3_x_gate 0=2
+Gemm gemm_att_3_x_out 1 1 att_3_x_gate att_3_x_out 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+BinaryOp add_47 2 1 att_3_x_out att_3_x_last time_mixing_3_out 0=0
+Split split_time_mixing_3_out 1 2 time_mixing_3_out ffn_3_x_last ffn_3_x
+LayerNorm layernorm_7 1 1 ffn_3_x ffn_3_xx 0=768 1=0.00001 2=1
+Split split_ffn_3_xx 1 3 ffn_3_xx ffn_3_xx_0 ffn_3_xx_1 state_11_out
+BinaryOp sub_7 2 1 state_11_in ffn_3_xx_0 ffn_3_sx 0=1
+Split split_ffn_3_sx 1 1 ffn_3_sx ffn_3_sx_0
+MemoryData data_ffn_3_x_k 0 1 ffn_3_x_k 0=768 21=0
+BinaryOp mul_50 2 1 ffn_3_sx_0 ffn_3_x_k ffn_3_xk 0=2
+BinaryOp add_48 2 1 ffn_3_xk ffn_3_xx_1 ffn_3_xxk 0=0
+Gemm gemm_ffn_3_key 1 1 ffn_3_xxk ffn_3_key 4=0 5=1 6=0 7=0 8=3072 9=768 10=-1
+ReLU relu_3 1 1 ffn_3_key ffn_3_key_relu
+UnaryOp square_ffn_3_key_relu 1 1 ffn_3_key_relu ffn_3_key_relu_square 0=4
+Gemm gemm_ffn_3_value 1 1 ffn_3_key_relu_square ffn_3_value 4=0 5=1 6=0 7=0 8=768 9=3072 10=-1
+BinaryOp add_49 2 1 ffn_3_value ffn_3_x_last channel_mixing_3_out 0=0
+Split split_channel_mixing_3_out 1 2 channel_mixing_3_out att_4_x_last att_4_x
+LayerNorm layernorm_8 1 1 att_4_x att_4_xx 0=768 1=0.00001 2=1
+Split split_att_4_xx 1 3 att_4_xx att_4_xx_0 att_4_xx_1 state_12_out
+BinaryOp sub_8 2 1 state_12_in att_4_xx_0 att_4_sx 0=1
+MemoryData data_att_4_maa 0 1 att_4_maa 0=768 1=6 21=0
+BinaryOp mul_51 2 1 att_4_sx att_4_maa att_4_maa_x 0=2
+BinaryOp add_50 2 1 att_4_xx_1 att_4_maa_x att_4_maa_rwkvag 0=0
+Slice slice_att_4_maa_rwkvag 1 6 att_4_maa_rwkvag att_4_mr att_4_mw att_4_mk att_4_mv att_4_ma att_4_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
+Split split_att_4_mv 1 2 att_4_mv att_4_mv_0 att_4_mv_1
+Gemm gemm_att_4_key 1 1 att_4_mk att_4_key 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_4_value 1 1 att_4_mv_0 att_4_value 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_4_receptance 1 1 att_4_mr att_4_receptance 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_4_ma_lora_0 1 1 att_4_ma att_4_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+Gemm gemm_att_4_ma_lora_1 1 1 att_4_ma_lora_0 att_4_ma_lora_1 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_4_a0 0 1 att_4_a0 0=768 21=0
+BinaryOp add_51 2 1 att_4_ma_lora_1 att_4_a0 att_4_ma_lora_2 0=0
+Sigmoid sigmoid_15 1 1 att_4_ma_lora_2 att_4_a
+Split split_att_4_a 1 2 att_4_a att_4_a_0 att_4_a_1
+Gemm gemm_att_4_mg_lora 1 1 att_4_mg att_4_mg_lora 4=0 5=1 6=0 7=0 8=128 9=768 10=-1
+Sigmoid sigmoid_16 1 1 att_4_mg_lora att_4_mg_lora_sigmoid
+Gemm gemm_att_4_gate 1 1 att_4_mg_lora_sigmoid att_4_gate 4=0 5=1 6=0 7=0 8=768 9=128 10=-1
+MemoryData data_att_4_k_k 0 1 att_4_k_k 0=768 21=0
+MemoryData data_att_4_k_a 0 1 att_4_k_a 0=768 21=0
+MemoryData data_att_4_r_k 0 1 att_4_r_k 0=768 21=0
+Split split_att_4_key 1 2 att_4_key att_4_key_0 att_4_key_1
+BinaryOp mul_52 2 1 att_4_key_0 att_4_k_k att_4_key_k 0=2
+Reshape reshape_48 1 1 att_4_key_k att_4_key_k_reshape 0=64 1=1 2=12
+Normalize l2norm_4 1 1 att_4_key_k_reshape att_4_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
+Reshape reshape_49 1 1 att_4_key_k_norm att_4_key_k_norm_reshape 0=768
+BinaryOp add_52 1 1 att_4_a_0 att_4_a_0_minus 0=0 1=1 2=-1.0
+BinaryOp mul_53 2 1 att_4_a_0_minus att_4_k_a att_4_a_minus1_mulka 0=2
+BinaryOp add_53 1 1 att_4_a_minus1_mulka att_4_a_minus1_mulka_plus1 0=0 1=1 2=1.0
+BinaryOp mul_54 2 1 att_4_a_minus1_mulka_plus1 att_4_key_1 att_4_key_a 0=2
+Split split_att_4_value 1 2 att_4_value att_4_value_0 att_4_value_1
+Split split_v_first_3 1 2 v_first_3 att_4_v_first v_first_4
+BinaryOp minus_3 2 1 att_4_v_first att_4_value_1 att_4_vfirst_minus_value 0=1
+Gemm gemm_att_4_mv_lora_0 1 1 att_4_mv_1 att_4_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=768 10=-1
+Gemm gemm_att_4_mv_lora_1 1 1 att_4_mv_lora_0 att_4_mv_lora_1 4=0 5=1 6=0 7=0 8=768 9=32 10=-1
+MemoryData data_att_4_v0 0 1 att_4_v0 0=768 21=0
+BinaryOp add_54 2 1 att_4_mv_lora_1 att_4_v0 att_4_mv_lora_2 0=0
+Sigmoid sigmoid_17 1 1 att_4_mv_lora_2 att_4_mv_lora_sigmoid
+BinaryOp mul_55 2 1 att_4_mv_lora_sigmoid att_4_vfirst_minus_value att_4_vfirst_minus_value_mul 0=2
+BinaryOp add_55 2 1 att_4_value_0 att_4_vfirst_minus_value_mul att_4_value_final 0=0
+Gemm gemm_att_4_mw_lora 1 1 att_4_mw att_4_mw_lora 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+TanH tanh_att_4_mw_lora 1 1 att_4_mw_lora att_4_mw_lora_tanh
+Gemm gemm_att_4_mw_lora_tanh_linear 1 1 att_4_mw_lora_tanh att_4_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_4_td 0 1 att_4_td 0=768 21=0
+BinaryOp add_56 2 1 att_4_mw_lora_tanh_linear att_4_td att_4_time_decay_pre 0=0
+Sigmoid sigmoid_18 1 1 att_4_time_decay_pre att_4_time_decay_sigmoid
+Exp exp_4 1 1 att_4_time_decay_sigmoid att_4_time_decay 1=-0.606531
+Reshape reshape_50 1 1 att_4_time_decay att_4_time_decay_reshape 0=64 1=1 2=12
+Split split_att_4_key_a 1 2 att_4_key_a att_4_key_a_0 att_4_key_a_1
+Reshape reshape_51 1 1 att_4_key_a_0 att_4_key_a_reshape 0=64 1=1 2=12
+Split split_att_4_value_final 1 3 att_4_value_final att_4_value_final_0 att_4_value_final_1 att_4_value_final_2
+Reshape reshape_52 1 1 att_4_value_final_0 att_4_value_final_0_reshape 0=64 1=12
+Reshape reshape_53 1 1 att_4_value_final_1 att_4_value_final_1_reshape 0=1 1=64 2=12
+MatMul matmul_att_4_vk 2 1 att_4_value_final_1_reshape att_4_key_a_reshape att_4_vk 0=0
+Split split_att_4_key_k_norm_reshape 1 2 att_4_key_k_norm_reshape att_4_kk_0 att_4_kk_1
+BinaryOp mul_56 1 1 att_4_kk_0 att_4_kk_0_neg 0=2 1=1 2=-1.0
+Reshape reshape_54 1 1 att_4_kk_0_neg att_4_kk_0_neg_reshape 0=1 1=64 2=12
+BinaryOp mul_57 2 1 att_4_kk_1 att_4_a_1 att_4_kk_a 0=2
+Reshape reshape_55 1 1 att_4_kk_a att_4_kk_a_reshape 0=64 1=1 2=12
+MatMul matmul_att_4_ab 2 1 att_4_kk_0_neg_reshape att_4_kk_a_reshape att_4_ab 0=0
+Split split_state_13_in 1 2 state_13_in att_4_state_prev_0 att_4_state_prev_1
+BinaryOp mul_58 2 1 att_4_state_prev_0 att_4_time_decay_reshape att_4_state_td 0=2
+MatMul matmul_att_4_sab 2 1 att_4_state_prev_1 att_4_ab att_4_sab 0=0
+BinaryOp add_57 2 1 att_4_sab att_4_vk att_4_sab_vk 0=0
+BinaryOp add_58 2 1 att_4_sab_vk att_4_state_td att_4_state_new 0=0
+Split split_att_4_state_new 1 2 att_4_state_new att_4_state_new_0 state_13_out
+Split split_att_4_receptance 1 2 att_4_receptance att_4_receptance_0 att_4_receptance_1
+Reshape reshape_56 1 1 att_4_receptance_0 att_4_receptance_reshape 0=1 1=64 2=12
+MatMul matmul_att_4_wkv_out 2 1 att_4_state_new_0 att_4_receptance_reshape att_4_wkv_out 0=0
+Reshape reshape_57 1 1 att_4_wkv_out att_4_wkv_out_flatten 0=768
+GroupNorm groupnorm_4 1 1 att_4_wkv_out_flatten att_4_x_gn 0=12 1=768 2=0.00064 3=1
+BinaryOp mul_59 2 1 att_4_receptance_1 att_4_key_a_1 att_4_receptance_key_a 0=2
+BinaryOp mul_60 2 1 att_4_receptance_key_a att_4_r_k att_4_rk 0=2
+Reshape reshape_58 1 1 att_4_rk att_4_rk_reshape 0=64 1=12
+Reduction sum_4 1 1 att_4_rk_reshape att_4_rk_sum 1=0 -23303=1,-1 4=1 5=1
+BinaryOp mul_61 2 1 att_4_value_final_0_reshape att_4_rk_sum att_4_value_final_0_rk 0=2
+Reshape reshape_59 1 1 att_4_value_final_0_rk att_4_value_final_0_rk_reshape 0=768
+BinaryOp add_59 2 1 att_4_value_final_0_rk_reshape att_4_x_gn att_4_x_gn_rkv 0=0
+BinaryOp mul_62 2 1 att_4_x_gn_rkv att_4_gate att_4_x_gate 0=2
+Gemm gemm_att_4_x_out 1 1 att_4_x_gate att_4_x_out 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+BinaryOp add_60 2 1 att_4_x_out att_4_x_last time_mixing_4_out 0=0
+Split split_time_mixing_4_out 1 2 time_mixing_4_out ffn_4_x_last ffn_4_x
+LayerNorm layernorm_9 1 1 ffn_4_x ffn_4_xx 0=768 1=0.00001 2=1
+Split split_ffn_4_xx 1 3 ffn_4_xx ffn_4_xx_0 ffn_4_xx_1 state_14_out
+BinaryOp sub_9 2 1 state_14_in ffn_4_xx_0 ffn_4_sx 0=1
+Split split_ffn_4_sx 1 1 ffn_4_sx ffn_4_sx_0
+MemoryData data_ffn_4_x_k 0 1 ffn_4_x_k 0=768 21=0
+BinaryOp mul_63 2 1 ffn_4_sx_0 ffn_4_x_k ffn_4_xk 0=2
+BinaryOp add_61 2 1 ffn_4_xk ffn_4_xx_1 ffn_4_xxk 0=0
+Gemm gemm_ffn_4_key 1 1 ffn_4_xxk ffn_4_key 4=0 5=1 6=0 7=0 8=3072 9=768 10=-1
+ReLU relu_4 1 1 ffn_4_key ffn_4_key_relu
+UnaryOp square_ffn_4_key_relu 1 1 ffn_4_key_relu ffn_4_key_relu_square 0=4
+Gemm gemm_ffn_4_value 1 1 ffn_4_key_relu_square ffn_4_value 4=0 5=1 6=0 7=0 8=768 9=3072 10=-1
+BinaryOp add_62 2 1 ffn_4_value ffn_4_x_last channel_mixing_4_out 0=0
+Split split_channel_mixing_4_out 1 2 channel_mixing_4_out att_5_x_last att_5_x
+LayerNorm layernorm_10 1 1 att_5_x att_5_xx 0=768 1=0.00001 2=1
+Split split_att_5_xx 1 3 att_5_xx att_5_xx_0 att_5_xx_1 state_15_out
+BinaryOp sub_10 2 1 state_15_in att_5_xx_0 att_5_sx 0=1
+MemoryData data_att_5_maa 0 1 att_5_maa 0=768 1=6 21=0
+BinaryOp mul_64 2 1 att_5_sx att_5_maa att_5_maa_x 0=2
+BinaryOp add_63 2 1 att_5_xx_1 att_5_maa_x att_5_maa_rwkvag 0=0
+Slice slice_att_5_maa_rwkvag 1 6 att_5_maa_rwkvag att_5_mr att_5_mw att_5_mk att_5_mv att_5_ma att_5_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
+Split split_att_5_mv 1 2 att_5_mv att_5_mv_0 att_5_mv_1
+Gemm gemm_att_5_key 1 1 att_5_mk att_5_key 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_5_value 1 1 att_5_mv_0 att_5_value 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_5_receptance 1 1 att_5_mr att_5_receptance 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_5_ma_lora_0 1 1 att_5_ma att_5_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+Gemm gemm_att_5_ma_lora_1 1 1 att_5_ma_lora_0 att_5_ma_lora_1 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_5_a0 0 1 att_5_a0 0=768 21=0
+BinaryOp add_64 2 1 att_5_ma_lora_1 att_5_a0 att_5_ma_lora_2 0=0
+Sigmoid sigmoid_19 1 1 att_5_ma_lora_2 att_5_a
+Split split_att_5_a 1 2 att_5_a att_5_a_0 att_5_a_1
+Gemm gemm_att_5_mg_lora 1 1 att_5_mg att_5_mg_lora 4=0 5=1 6=0 7=0 8=128 9=768 10=-1
+Sigmoid sigmoid_20 1 1 att_5_mg_lora att_5_mg_lora_sigmoid
+Gemm gemm_att_5_gate 1 1 att_5_mg_lora_sigmoid att_5_gate 4=0 5=1 6=0 7=0 8=768 9=128 10=-1
+MemoryData data_att_5_k_k 0 1 att_5_k_k 0=768 21=0
+MemoryData data_att_5_k_a 0 1 att_5_k_a 0=768 21=0
+MemoryData data_att_5_r_k 0 1 att_5_r_k 0=768 21=0
+Split split_att_5_key 1 2 att_5_key att_5_key_0 att_5_key_1
+BinaryOp mul_65 2 1 att_5_key_0 att_5_k_k att_5_key_k 0=2
+Reshape reshape_60 1 1 att_5_key_k att_5_key_k_reshape 0=64 1=1 2=12
+Normalize l2norm_5 1 1 att_5_key_k_reshape att_5_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
+Reshape reshape_61 1 1 att_5_key_k_norm att_5_key_k_norm_reshape 0=768
+BinaryOp add_65 1 1 att_5_a_0 att_5_a_0_minus 0=0 1=1 2=-1.0
+BinaryOp mul_66 2 1 att_5_a_0_minus att_5_k_a att_5_a_minus1_mulka 0=2
+BinaryOp add_66 1 1 att_5_a_minus1_mulka att_5_a_minus1_mulka_plus1 0=0 1=1 2=1.0
+BinaryOp mul_67 2 1 att_5_a_minus1_mulka_plus1 att_5_key_1 att_5_key_a 0=2
+Split split_att_5_value 1 2 att_5_value att_5_value_0 att_5_value_1
+Split split_v_first_4 1 2 v_first_4 att_5_v_first v_first_5
+BinaryOp minus_4 2 1 att_5_v_first att_5_value_1 att_5_vfirst_minus_value 0=1
+Gemm gemm_att_5_mv_lora_0 1 1 att_5_mv_1 att_5_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=768 10=-1
+Gemm gemm_att_5_mv_lora_1 1 1 att_5_mv_lora_0 att_5_mv_lora_1 4=0 5=1 6=0 7=0 8=768 9=32 10=-1
+MemoryData data_att_5_v0 0 1 att_5_v0 0=768 21=0
+BinaryOp add_67 2 1 att_5_mv_lora_1 att_5_v0 att_5_mv_lora_2 0=0
+Sigmoid sigmoid_21 1 1 att_5_mv_lora_2 att_5_mv_lora_sigmoid
+BinaryOp mul_68 2 1 att_5_mv_lora_sigmoid att_5_vfirst_minus_value att_5_vfirst_minus_value_mul 0=2
+BinaryOp add_68 2 1 att_5_value_0 att_5_vfirst_minus_value_mul att_5_value_final 0=0
+Gemm gemm_att_5_mw_lora 1 1 att_5_mw att_5_mw_lora 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+TanH tanh_att_5_mw_lora 1 1 att_5_mw_lora att_5_mw_lora_tanh
+Gemm gemm_att_5_mw_lora_tanh_linear 1 1 att_5_mw_lora_tanh att_5_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_5_td 0 1 att_5_td 0=768 21=0
+BinaryOp add_69 2 1 att_5_mw_lora_tanh_linear att_5_td att_5_time_decay_pre 0=0
+Sigmoid sigmoid_22 1 1 att_5_time_decay_pre att_5_time_decay_sigmoid
+Exp exp_5 1 1 att_5_time_decay_sigmoid att_5_time_decay 1=-0.606531
+Reshape reshape_62 1 1 att_5_time_decay att_5_time_decay_reshape 0=64 1=1 2=12
+Split split_att_5_key_a 1 2 att_5_key_a att_5_key_a_0 att_5_key_a_1
+Reshape reshape_63 1 1 att_5_key_a_0 att_5_key_a_reshape 0=64 1=1 2=12
+Split split_att_5_value_final 1 3 att_5_value_final att_5_value_final_0 att_5_value_final_1 att_5_value_final_2
+Reshape reshape_64 1 1 att_5_value_final_0 att_5_value_final_0_reshape 0=64 1=12
+Reshape reshape_65 1 1 att_5_value_final_1 att_5_value_final_1_reshape 0=1 1=64 2=12
+MatMul matmul_att_5_vk 2 1 att_5_value_final_1_reshape att_5_key_a_reshape att_5_vk 0=0
+Split split_att_5_key_k_norm_reshape 1 2 att_5_key_k_norm_reshape att_5_kk_0 att_5_kk_1
+BinaryOp mul_69 1 1 att_5_kk_0 att_5_kk_0_neg 0=2 1=1 2=-1.0
+Reshape reshape_66 1 1 att_5_kk_0_neg att_5_kk_0_neg_reshape 0=1 1=64 2=12
+BinaryOp mul_70 2 1 att_5_kk_1 att_5_a_1 att_5_kk_a 0=2
+Reshape reshape_67 1 1 att_5_kk_a att_5_kk_a_reshape 0=64 1=1 2=12
+MatMul matmul_att_5_ab 2 1 att_5_kk_0_neg_reshape att_5_kk_a_reshape att_5_ab 0=0
+Split split_state_16_in 1 2 state_16_in att_5_state_prev_0 att_5_state_prev_1
+BinaryOp mul_71 2 1 att_5_state_prev_0 att_5_time_decay_reshape att_5_state_td 0=2
+MatMul matmul_att_5_sab 2 1 att_5_state_prev_1 att_5_ab att_5_sab 0=0
+BinaryOp add_70 2 1 att_5_sab att_5_vk att_5_sab_vk 0=0
+BinaryOp add_71 2 1 att_5_sab_vk att_5_state_td att_5_state_new 0=0
+Split split_att_5_state_new 1 2 att_5_state_new att_5_state_new_0 state_16_out
+Split split_att_5_receptance 1 2 att_5_receptance att_5_receptance_0 att_5_receptance_1
+Reshape reshape_68 1 1 att_5_receptance_0 att_5_receptance_reshape 0=1 1=64 2=12
+MatMul matmul_att_5_wkv_out 2 1 att_5_state_new_0 att_5_receptance_reshape att_5_wkv_out 0=0
+Reshape reshape_69 1 1 att_5_wkv_out att_5_wkv_out_flatten 0=768
+GroupNorm groupnorm_5 1 1 att_5_wkv_out_flatten att_5_x_gn 0=12 1=768 2=0.00064 3=1
+BinaryOp mul_72 2 1 att_5_receptance_1 att_5_key_a_1 att_5_receptance_key_a 0=2
+BinaryOp mul_73 2 1 att_5_receptance_key_a att_5_r_k att_5_rk 0=2
+Reshape reshape_70 1 1 att_5_rk att_5_rk_reshape 0=64 1=12
+Reduction sum_5 1 1 att_5_rk_reshape att_5_rk_sum 1=0 -23303=1,-1 4=1 5=1
+BinaryOp mul_74 2 1 att_5_value_final_0_reshape att_5_rk_sum att_5_value_final_0_rk 0=2
+Reshape reshape_71 1 1 att_5_value_final_0_rk att_5_value_final_0_rk_reshape 0=768
+BinaryOp add_72 2 1 att_5_value_final_0_rk_reshape att_5_x_gn att_5_x_gn_rkv 0=0
+BinaryOp mul_75 2 1 att_5_x_gn_rkv att_5_gate att_5_x_gate 0=2
+Gemm gemm_att_5_x_out 1 1 att_5_x_gate att_5_x_out 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+BinaryOp add_73 2 1 att_5_x_out att_5_x_last time_mixing_5_out 0=0
+Split split_time_mixing_5_out 1 2 time_mixing_5_out ffn_5_x_last ffn_5_x
+LayerNorm layernorm_11 1 1 ffn_5_x ffn_5_xx 0=768 1=0.00001 2=1
+Split split_ffn_5_xx 1 3 ffn_5_xx ffn_5_xx_0 ffn_5_xx_1 state_17_out
+BinaryOp sub_11 2 1 state_17_in ffn_5_xx_0 ffn_5_sx 0=1
+Split split_ffn_5_sx 1 1 ffn_5_sx ffn_5_sx_0
+MemoryData data_ffn_5_x_k 0 1 ffn_5_x_k 0=768 21=0
+BinaryOp mul_76 2 1 ffn_5_sx_0 ffn_5_x_k ffn_5_xk 0=2
+BinaryOp add_74 2 1 ffn_5_xk ffn_5_xx_1 ffn_5_xxk 0=0
+Gemm gemm_ffn_5_key 1 1 ffn_5_xxk ffn_5_key 4=0 5=1 6=0 7=0 8=3072 9=768 10=-1
+ReLU relu_5 1 1 ffn_5_key ffn_5_key_relu
+UnaryOp square_ffn_5_key_relu 1 1 ffn_5_key_relu ffn_5_key_relu_square 0=4
+Gemm gemm_ffn_5_value 1 1 ffn_5_key_relu_square ffn_5_value 4=0 5=1 6=0 7=0 8=768 9=3072 10=-1
+BinaryOp add_75 2 1 ffn_5_value ffn_5_x_last channel_mixing_5_out 0=0
+Split split_channel_mixing_5_out 1 2 channel_mixing_5_out att_6_x_last att_6_x
+LayerNorm layernorm_12 1 1 att_6_x att_6_xx 0=768 1=0.00001 2=1
+Split split_att_6_xx 1 3 att_6_xx att_6_xx_0 att_6_xx_1 state_18_out
+BinaryOp sub_12 2 1 state_18_in att_6_xx_0 att_6_sx 0=1
+MemoryData data_att_6_maa 0 1 att_6_maa 0=768 1=6 21=0
+BinaryOp mul_77 2 1 att_6_sx att_6_maa att_6_maa_x 0=2
+BinaryOp add_76 2 1 att_6_xx_1 att_6_maa_x att_6_maa_rwkvag 0=0
+Slice slice_att_6_maa_rwkvag 1 6 att_6_maa_rwkvag att_6_mr att_6_mw att_6_mk att_6_mv att_6_ma att_6_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
+Split split_att_6_mv 1 2 att_6_mv att_6_mv_0 att_6_mv_1
+Gemm gemm_att_6_key 1 1 att_6_mk att_6_key 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_6_value 1 1 att_6_mv_0 att_6_value 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_6_receptance 1 1 att_6_mr att_6_receptance 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_6_ma_lora_0 1 1 att_6_ma att_6_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+Gemm gemm_att_6_ma_lora_1 1 1 att_6_ma_lora_0 att_6_ma_lora_1 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_6_a0 0 1 att_6_a0 0=768 21=0
+BinaryOp add_77 2 1 att_6_ma_lora_1 att_6_a0 att_6_ma_lora_2 0=0
+Sigmoid sigmoid_23 1 1 att_6_ma_lora_2 att_6_a
+Split split_att_6_a 1 2 att_6_a att_6_a_0 att_6_a_1
+Gemm gemm_att_6_mg_lora 1 1 att_6_mg att_6_mg_lora 4=0 5=1 6=0 7=0 8=128 9=768 10=-1
+Sigmoid sigmoid_24 1 1 att_6_mg_lora att_6_mg_lora_sigmoid
+Gemm gemm_att_6_gate 1 1 att_6_mg_lora_sigmoid att_6_gate 4=0 5=1 6=0 7=0 8=768 9=128 10=-1
+MemoryData data_att_6_k_k 0 1 att_6_k_k 0=768 21=0
+MemoryData data_att_6_k_a 0 1 att_6_k_a 0=768 21=0
+MemoryData data_att_6_r_k 0 1 att_6_r_k 0=768 21=0
+Split split_att_6_key 1 2 att_6_key att_6_key_0 att_6_key_1
+BinaryOp mul_78 2 1 att_6_key_0 att_6_k_k att_6_key_k 0=2
+Reshape reshape_72 1 1 att_6_key_k att_6_key_k_reshape 0=64 1=1 2=12
+Normalize l2norm_6 1 1 att_6_key_k_reshape att_6_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
+Reshape reshape_73 1 1 att_6_key_k_norm att_6_key_k_norm_reshape 0=768
+BinaryOp add_78 1 1 att_6_a_0 att_6_a_0_minus 0=0 1=1 2=-1.0
+BinaryOp mul_79 2 1 att_6_a_0_minus att_6_k_a att_6_a_minus1_mulka 0=2
+BinaryOp add_79 1 1 att_6_a_minus1_mulka att_6_a_minus1_mulka_plus1 0=0 1=1 2=1.0
+BinaryOp mul_80 2 1 att_6_a_minus1_mulka_plus1 att_6_key_1 att_6_key_a 0=2
+Split split_att_6_value 1 2 att_6_value att_6_value_0 att_6_value_1
+Split split_v_first_5 1 2 v_first_5 att_6_v_first v_first_6
+BinaryOp minus_5 2 1 att_6_v_first att_6_value_1 att_6_vfirst_minus_value 0=1
+Gemm gemm_att_6_mv_lora_0 1 1 att_6_mv_1 att_6_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=768 10=-1
+Gemm gemm_att_6_mv_lora_1 1 1 att_6_mv_lora_0 att_6_mv_lora_1 4=0 5=1 6=0 7=0 8=768 9=32 10=-1
+MemoryData data_att_6_v0 0 1 att_6_v0 0=768 21=0
+BinaryOp add_80 2 1 att_6_mv_lora_1 att_6_v0 att_6_mv_lora_2 0=0
+Sigmoid sigmoid_25 1 1 att_6_mv_lora_2 att_6_mv_lora_sigmoid
+BinaryOp mul_81 2 1 att_6_mv_lora_sigmoid att_6_vfirst_minus_value att_6_vfirst_minus_value_mul 0=2
+BinaryOp add_81 2 1 att_6_value_0 att_6_vfirst_minus_value_mul att_6_value_final 0=0
+Gemm gemm_att_6_mw_lora 1 1 att_6_mw att_6_mw_lora 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+TanH tanh_att_6_mw_lora 1 1 att_6_mw_lora att_6_mw_lora_tanh
+Gemm gemm_att_6_mw_lora_tanh_linear 1 1 att_6_mw_lora_tanh att_6_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_6_td 0 1 att_6_td 0=768 21=0
+BinaryOp add_82 2 1 att_6_mw_lora_tanh_linear att_6_td att_6_time_decay_pre 0=0
+Sigmoid sigmoid_26 1 1 att_6_time_decay_pre att_6_time_decay_sigmoid
+Exp exp_6 1 1 att_6_time_decay_sigmoid att_6_time_decay 1=-0.606531
+Reshape reshape_74 1 1 att_6_time_decay att_6_time_decay_reshape 0=64 1=1 2=12
+Split split_att_6_key_a 1 2 att_6_key_a att_6_key_a_0 att_6_key_a_1
+Reshape reshape_75 1 1 att_6_key_a_0 att_6_key_a_reshape 0=64 1=1 2=12
+Split split_att_6_value_final 1 3 att_6_value_final att_6_value_final_0 att_6_value_final_1 att_6_value_final_2
+Reshape reshape_76 1 1 att_6_value_final_0 att_6_value_final_0_reshape 0=64 1=12
+Reshape reshape_77 1 1 att_6_value_final_1 att_6_value_final_1_reshape 0=1 1=64 2=12
+MatMul matmul_att_6_vk 2 1 att_6_value_final_1_reshape att_6_key_a_reshape att_6_vk 0=0
+Split split_att_6_key_k_norm_reshape 1 2 att_6_key_k_norm_reshape att_6_kk_0 att_6_kk_1
+BinaryOp mul_82 1 1 att_6_kk_0 att_6_kk_0_neg 0=2 1=1 2=-1.0
+Reshape reshape_78 1 1 att_6_kk_0_neg att_6_kk_0_neg_reshape 0=1 1=64 2=12
+BinaryOp mul_83 2 1 att_6_kk_1 att_6_a_1 att_6_kk_a 0=2
+Reshape reshape_79 1 1 att_6_kk_a att_6_kk_a_reshape 0=64 1=1 2=12
+MatMul matmul_att_6_ab 2 1 att_6_kk_0_neg_reshape att_6_kk_a_reshape att_6_ab 0=0
+Split split_state_19_in 1 2 state_19_in att_6_state_prev_0 att_6_state_prev_1
+BinaryOp mul_84 2 1 att_6_state_prev_0 att_6_time_decay_reshape att_6_state_td 0=2
+MatMul matmul_att_6_sab 2 1 att_6_state_prev_1 att_6_ab att_6_sab 0=0
+BinaryOp add_83 2 1 att_6_sab att_6_vk att_6_sab_vk 0=0
+BinaryOp add_84 2 1 att_6_sab_vk att_6_state_td att_6_state_new 0=0
+Split split_att_6_state_new 1 2 att_6_state_new att_6_state_new_0 state_19_out
+Split split_att_6_receptance 1 2 att_6_receptance att_6_receptance_0 att_6_receptance_1
+Reshape reshape_80 1 1 att_6_receptance_0 att_6_receptance_reshape 0=1 1=64 2=12
+MatMul matmul_att_6_wkv_out 2 1 att_6_state_new_0 att_6_receptance_reshape att_6_wkv_out 0=0
+Reshape reshape_81 1 1 att_6_wkv_out att_6_wkv_out_flatten 0=768
+GroupNorm groupnorm_6 1 1 att_6_wkv_out_flatten att_6_x_gn 0=12 1=768 2=0.00064 3=1
+BinaryOp mul_85 2 1 att_6_receptance_1 att_6_key_a_1 att_6_receptance_key_a 0=2
+BinaryOp mul_86 2 1 att_6_receptance_key_a att_6_r_k att_6_rk 0=2
+Reshape reshape_82 1 1 att_6_rk att_6_rk_reshape 0=64 1=12
+Reduction sum_6 1 1 att_6_rk_reshape att_6_rk_sum 1=0 -23303=1,-1 4=1 5=1
+BinaryOp mul_87 2 1 att_6_value_final_0_reshape att_6_rk_sum att_6_value_final_0_rk 0=2
+Reshape reshape_83 1 1 att_6_value_final_0_rk att_6_value_final_0_rk_reshape 0=768
+BinaryOp add_85 2 1 att_6_value_final_0_rk_reshape att_6_x_gn att_6_x_gn_rkv 0=0
+BinaryOp mul_88 2 1 att_6_x_gn_rkv att_6_gate att_6_x_gate 0=2
+Gemm gemm_att_6_x_out 1 1 att_6_x_gate att_6_x_out 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+BinaryOp add_86 2 1 att_6_x_out att_6_x_last time_mixing_6_out 0=0
+Split split_time_mixing_6_out 1 2 time_mixing_6_out ffn_6_x_last ffn_6_x
+LayerNorm layernorm_13 1 1 ffn_6_x ffn_6_xx 0=768 1=0.00001 2=1
+Split split_ffn_6_xx 1 3 ffn_6_xx ffn_6_xx_0 ffn_6_xx_1 state_20_out
+BinaryOp sub_13 2 1 state_20_in ffn_6_xx_0 ffn_6_sx 0=1
+Split split_ffn_6_sx 1 1 ffn_6_sx ffn_6_sx_0
+MemoryData data_ffn_6_x_k 0 1 ffn_6_x_k 0=768 21=0
+BinaryOp mul_89 2 1 ffn_6_sx_0 ffn_6_x_k ffn_6_xk 0=2
+BinaryOp add_87 2 1 ffn_6_xk ffn_6_xx_1 ffn_6_xxk 0=0
+Gemm gemm_ffn_6_key 1 1 ffn_6_xxk ffn_6_key 4=0 5=1 6=0 7=0 8=3072 9=768 10=-1
+ReLU relu_6 1 1 ffn_6_key ffn_6_key_relu
+UnaryOp square_ffn_6_key_relu 1 1 ffn_6_key_relu ffn_6_key_relu_square 0=4
+Gemm gemm_ffn_6_value 1 1 ffn_6_key_relu_square ffn_6_value 4=0 5=1 6=0 7=0 8=768 9=3072 10=-1
+BinaryOp add_88 2 1 ffn_6_value ffn_6_x_last channel_mixing_6_out 0=0
+Split split_channel_mixing_6_out 1 2 channel_mixing_6_out att_7_x_last att_7_x
+LayerNorm layernorm_14 1 1 att_7_x att_7_xx 0=768 1=0.00001 2=1
+Split split_att_7_xx 1 3 att_7_xx att_7_xx_0 att_7_xx_1 state_21_out
+BinaryOp sub_14 2 1 state_21_in att_7_xx_0 att_7_sx 0=1
+MemoryData data_att_7_maa 0 1 att_7_maa 0=768 1=6 21=0
+BinaryOp mul_90 2 1 att_7_sx att_7_maa att_7_maa_x 0=2
+BinaryOp add_89 2 1 att_7_xx_1 att_7_maa_x att_7_maa_rwkvag 0=0
+Slice slice_att_7_maa_rwkvag 1 6 att_7_maa_rwkvag att_7_mr att_7_mw att_7_mk att_7_mv att_7_ma att_7_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
+Split split_att_7_mv 1 2 att_7_mv att_7_mv_0 att_7_mv_1
+Gemm gemm_att_7_key 1 1 att_7_mk att_7_key 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_7_value 1 1 att_7_mv_0 att_7_value 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_7_receptance 1 1 att_7_mr att_7_receptance 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_7_ma_lora_0 1 1 att_7_ma att_7_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+Gemm gemm_att_7_ma_lora_1 1 1 att_7_ma_lora_0 att_7_ma_lora_1 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_7_a0 0 1 att_7_a0 0=768 21=0
+BinaryOp add_90 2 1 att_7_ma_lora_1 att_7_a0 att_7_ma_lora_2 0=0
+Sigmoid sigmoid_27 1 1 att_7_ma_lora_2 att_7_a
+Split split_att_7_a 1 2 att_7_a att_7_a_0 att_7_a_1
+Gemm gemm_att_7_mg_lora 1 1 att_7_mg att_7_mg_lora 4=0 5=1 6=0 7=0 8=128 9=768 10=-1
+Sigmoid sigmoid_28 1 1 att_7_mg_lora att_7_mg_lora_sigmoid
+Gemm gemm_att_7_gate 1 1 att_7_mg_lora_sigmoid att_7_gate 4=0 5=1 6=0 7=0 8=768 9=128 10=-1
+MemoryData data_att_7_k_k 0 1 att_7_k_k 0=768 21=0
+MemoryData data_att_7_k_a 0 1 att_7_k_a 0=768 21=0
+MemoryData data_att_7_r_k 0 1 att_7_r_k 0=768 21=0
+Split split_att_7_key 1 2 att_7_key att_7_key_0 att_7_key_1
+BinaryOp mul_91 2 1 att_7_key_0 att_7_k_k att_7_key_k 0=2
+Reshape reshape_84 1 1 att_7_key_k att_7_key_k_reshape 0=64 1=1 2=12
+Normalize l2norm_7 1 1 att_7_key_k_reshape att_7_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
+Reshape reshape_85 1 1 att_7_key_k_norm att_7_key_k_norm_reshape 0=768
+BinaryOp add_91 1 1 att_7_a_0 att_7_a_0_minus 0=0 1=1 2=-1.0
+BinaryOp mul_92 2 1 att_7_a_0_minus att_7_k_a att_7_a_minus1_mulka 0=2
+BinaryOp add_92 1 1 att_7_a_minus1_mulka att_7_a_minus1_mulka_plus1 0=0 1=1 2=1.0
+BinaryOp mul_93 2 1 att_7_a_minus1_mulka_plus1 att_7_key_1 att_7_key_a 0=2
+Split split_att_7_value 1 2 att_7_value att_7_value_0 att_7_value_1
+Split split_v_first_6 1 2 v_first_6 att_7_v_first v_first_7
+BinaryOp minus_6 2 1 att_7_v_first att_7_value_1 att_7_vfirst_minus_value 0=1
+Gemm gemm_att_7_mv_lora_0 1 1 att_7_mv_1 att_7_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=768 10=-1
+Gemm gemm_att_7_mv_lora_1 1 1 att_7_mv_lora_0 att_7_mv_lora_1 4=0 5=1 6=0 7=0 8=768 9=32 10=-1
+MemoryData data_att_7_v0 0 1 att_7_v0 0=768 21=0
+BinaryOp add_93 2 1 att_7_mv_lora_1 att_7_v0 att_7_mv_lora_2 0=0
+Sigmoid sigmoid_29 1 1 att_7_mv_lora_2 att_7_mv_lora_sigmoid
+BinaryOp mul_94 2 1 att_7_mv_lora_sigmoid att_7_vfirst_minus_value att_7_vfirst_minus_value_mul 0=2
+BinaryOp add_94 2 1 att_7_value_0 att_7_vfirst_minus_value_mul att_7_value_final 0=0
+Gemm gemm_att_7_mw_lora 1 1 att_7_mw att_7_mw_lora 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+TanH tanh_att_7_mw_lora 1 1 att_7_mw_lora att_7_mw_lora_tanh
+Gemm gemm_att_7_mw_lora_tanh_linear 1 1 att_7_mw_lora_tanh att_7_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_7_td 0 1 att_7_td 0=768 21=0
+BinaryOp add_95 2 1 att_7_mw_lora_tanh_linear att_7_td att_7_time_decay_pre 0=0
+Sigmoid sigmoid_30 1 1 att_7_time_decay_pre att_7_time_decay_sigmoid
+Exp exp_7 1 1 att_7_time_decay_sigmoid att_7_time_decay 1=-0.606531
+Reshape reshape_86 1 1 att_7_time_decay att_7_time_decay_reshape 0=64 1=1 2=12
+Split split_att_7_key_a 1 2 att_7_key_a att_7_key_a_0 att_7_key_a_1
+Reshape reshape_87 1 1 att_7_key_a_0 att_7_key_a_reshape 0=64 1=1 2=12
+Split split_att_7_value_final 1 3 att_7_value_final att_7_value_final_0 att_7_value_final_1 att_7_value_final_2
+Reshape reshape_88 1 1 att_7_value_final_0 att_7_value_final_0_reshape 0=64 1=12
+Reshape reshape_89 1 1 att_7_value_final_1 att_7_value_final_1_reshape 0=1 1=64 2=12
+MatMul matmul_att_7_vk 2 1 att_7_value_final_1_reshape att_7_key_a_reshape att_7_vk 0=0
+Split split_att_7_key_k_norm_reshape 1 2 att_7_key_k_norm_reshape att_7_kk_0 att_7_kk_1
+BinaryOp mul_95 1 1 att_7_kk_0 att_7_kk_0_neg 0=2 1=1 2=-1.0
+Reshape reshape_90 1 1 att_7_kk_0_neg att_7_kk_0_neg_reshape 0=1 1=64 2=12
+BinaryOp mul_96 2 1 att_7_kk_1 att_7_a_1 att_7_kk_a 0=2
+Reshape reshape_91 1 1 att_7_kk_a att_7_kk_a_reshape 0=64 1=1 2=12
+MatMul matmul_att_7_ab 2 1 att_7_kk_0_neg_reshape att_7_kk_a_reshape att_7_ab 0=0
+Split split_state_22_in 1 2 state_22_in att_7_state_prev_0 att_7_state_prev_1
+BinaryOp mul_97 2 1 att_7_state_prev_0 att_7_time_decay_reshape att_7_state_td 0=2
+MatMul matmul_att_7_sab 2 1 att_7_state_prev_1 att_7_ab att_7_sab 0=0
+BinaryOp add_96 2 1 att_7_sab att_7_vk att_7_sab_vk 0=0
+BinaryOp add_97 2 1 att_7_sab_vk att_7_state_td att_7_state_new 0=0
+Split split_att_7_state_new 1 2 att_7_state_new att_7_state_new_0 state_22_out
+Split split_att_7_receptance 1 2 att_7_receptance att_7_receptance_0 att_7_receptance_1
+Reshape reshape_92 1 1 att_7_receptance_0 att_7_receptance_reshape 0=1 1=64 2=12
+MatMul matmul_att_7_wkv_out 2 1 att_7_state_new_0 att_7_receptance_reshape att_7_wkv_out 0=0
+Reshape reshape_93 1 1 att_7_wkv_out att_7_wkv_out_flatten 0=768
+GroupNorm groupnorm_7 1 1 att_7_wkv_out_flatten att_7_x_gn 0=12 1=768 2=0.00064 3=1
+BinaryOp mul_98 2 1 att_7_receptance_1 att_7_key_a_1 att_7_receptance_key_a 0=2
+BinaryOp mul_99 2 1 att_7_receptance_key_a att_7_r_k att_7_rk 0=2
+Reshape reshape_94 1 1 att_7_rk att_7_rk_reshape 0=64 1=12
+Reduction sum_7 1 1 att_7_rk_reshape att_7_rk_sum 1=0 -23303=1,-1 4=1 5=1
+BinaryOp mul_100 2 1 att_7_value_final_0_reshape att_7_rk_sum att_7_value_final_0_rk 0=2
+Reshape reshape_95 1 1 att_7_value_final_0_rk att_7_value_final_0_rk_reshape 0=768
+BinaryOp add_98 2 1 att_7_value_final_0_rk_reshape att_7_x_gn att_7_x_gn_rkv 0=0
+BinaryOp mul_101 2 1 att_7_x_gn_rkv att_7_gate att_7_x_gate 0=2
+Gemm gemm_att_7_x_out 1 1 att_7_x_gate att_7_x_out 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+BinaryOp add_99 2 1 att_7_x_out att_7_x_last time_mixing_7_out 0=0
+Split split_time_mixing_7_out 1 2 time_mixing_7_out ffn_7_x_last ffn_7_x
+LayerNorm layernorm_15 1 1 ffn_7_x ffn_7_xx 0=768 1=0.00001 2=1
+Split split_ffn_7_xx 1 3 ffn_7_xx ffn_7_xx_0 ffn_7_xx_1 state_23_out
+BinaryOp sub_15 2 1 state_23_in ffn_7_xx_0 ffn_7_sx 0=1
+Split split_ffn_7_sx 1 1 ffn_7_sx ffn_7_sx_0
+MemoryData data_ffn_7_x_k 0 1 ffn_7_x_k 0=768 21=0
+BinaryOp mul_102 2 1 ffn_7_sx_0 ffn_7_x_k ffn_7_xk 0=2
+BinaryOp add_100 2 1 ffn_7_xk ffn_7_xx_1 ffn_7_xxk 0=0
+Gemm gemm_ffn_7_key 1 1 ffn_7_xxk ffn_7_key 4=0 5=1 6=0 7=0 8=3072 9=768 10=-1
+ReLU relu_7 1 1 ffn_7_key ffn_7_key_relu
+UnaryOp square_ffn_7_key_relu 1 1 ffn_7_key_relu ffn_7_key_relu_square 0=4
+Gemm gemm_ffn_7_value 1 1 ffn_7_key_relu_square ffn_7_value 4=0 5=1 6=0 7=0 8=768 9=3072 10=-1
+BinaryOp add_101 2 1 ffn_7_value ffn_7_x_last channel_mixing_7_out 0=0
+Split split_channel_mixing_7_out 1 2 channel_mixing_7_out att_8_x_last att_8_x
+LayerNorm layernorm_16 1 1 att_8_x att_8_xx 0=768 1=0.00001 2=1
+Split split_att_8_xx 1 3 att_8_xx att_8_xx_0 att_8_xx_1 state_24_out
+BinaryOp sub_16 2 1 state_24_in att_8_xx_0 att_8_sx 0=1
+MemoryData data_att_8_maa 0 1 att_8_maa 0=768 1=6 21=0
+BinaryOp mul_103 2 1 att_8_sx att_8_maa att_8_maa_x 0=2
+BinaryOp add_102 2 1 att_8_xx_1 att_8_maa_x att_8_maa_rwkvag 0=0
+Slice slice_att_8_maa_rwkvag 1 6 att_8_maa_rwkvag att_8_mr att_8_mw att_8_mk att_8_mv att_8_ma att_8_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
+Split split_att_8_mv 1 2 att_8_mv att_8_mv_0 att_8_mv_1
+Gemm gemm_att_8_key 1 1 att_8_mk att_8_key 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_8_value 1 1 att_8_mv_0 att_8_value 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_8_receptance 1 1 att_8_mr att_8_receptance 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_8_ma_lora_0 1 1 att_8_ma att_8_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+Gemm gemm_att_8_ma_lora_1 1 1 att_8_ma_lora_0 att_8_ma_lora_1 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_8_a0 0 1 att_8_a0 0=768 21=0
+BinaryOp add_103 2 1 att_8_ma_lora_1 att_8_a0 att_8_ma_lora_2 0=0
+Sigmoid sigmoid_31 1 1 att_8_ma_lora_2 att_8_a
+Split split_att_8_a 1 2 att_8_a att_8_a_0 att_8_a_1
+Gemm gemm_att_8_mg_lora 1 1 att_8_mg att_8_mg_lora 4=0 5=1 6=0 7=0 8=128 9=768 10=-1
+Sigmoid sigmoid_32 1 1 att_8_mg_lora att_8_mg_lora_sigmoid
+Gemm gemm_att_8_gate 1 1 att_8_mg_lora_sigmoid att_8_gate 4=0 5=1 6=0 7=0 8=768 9=128 10=-1
+MemoryData data_att_8_k_k 0 1 att_8_k_k 0=768 21=0
+MemoryData data_att_8_k_a 0 1 att_8_k_a 0=768 21=0
+MemoryData data_att_8_r_k 0 1 att_8_r_k 0=768 21=0
+Split split_att_8_key 1 2 att_8_key att_8_key_0 att_8_key_1
+BinaryOp mul_104 2 1 att_8_key_0 att_8_k_k att_8_key_k 0=2
+Reshape reshape_96 1 1 att_8_key_k att_8_key_k_reshape 0=64 1=1 2=12
+Normalize l2norm_8 1 1 att_8_key_k_reshape att_8_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
+Reshape reshape_97 1 1 att_8_key_k_norm att_8_key_k_norm_reshape 0=768
+BinaryOp add_104 1 1 att_8_a_0 att_8_a_0_minus 0=0 1=1 2=-1.0
+BinaryOp mul_105 2 1 att_8_a_0_minus att_8_k_a att_8_a_minus1_mulka 0=2
+BinaryOp add_105 1 1 att_8_a_minus1_mulka att_8_a_minus1_mulka_plus1 0=0 1=1 2=1.0
+BinaryOp mul_106 2 1 att_8_a_minus1_mulka_plus1 att_8_key_1 att_8_key_a 0=2
+Split split_att_8_value 1 2 att_8_value att_8_value_0 att_8_value_1
+Split split_v_first_7 1 2 v_first_7 att_8_v_first v_first_8
+BinaryOp minus_7 2 1 att_8_v_first att_8_value_1 att_8_vfirst_minus_value 0=1
+Gemm gemm_att_8_mv_lora_0 1 1 att_8_mv_1 att_8_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=768 10=-1
+Gemm gemm_att_8_mv_lora_1 1 1 att_8_mv_lora_0 att_8_mv_lora_1 4=0 5=1 6=0 7=0 8=768 9=32 10=-1
+MemoryData data_att_8_v0 0 1 att_8_v0 0=768 21=0
+BinaryOp add_106 2 1 att_8_mv_lora_1 att_8_v0 att_8_mv_lora_2 0=0
+Sigmoid sigmoid_33 1 1 att_8_mv_lora_2 att_8_mv_lora_sigmoid
+BinaryOp mul_107 2 1 att_8_mv_lora_sigmoid att_8_vfirst_minus_value att_8_vfirst_minus_value_mul 0=2
+BinaryOp add_107 2 1 att_8_value_0 att_8_vfirst_minus_value_mul att_8_value_final 0=0
+Gemm gemm_att_8_mw_lora 1 1 att_8_mw att_8_mw_lora 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+TanH tanh_att_8_mw_lora 1 1 att_8_mw_lora att_8_mw_lora_tanh
+Gemm gemm_att_8_mw_lora_tanh_linear 1 1 att_8_mw_lora_tanh att_8_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_8_td 0 1 att_8_td 0=768 21=0
+BinaryOp add_108 2 1 att_8_mw_lora_tanh_linear att_8_td att_8_time_decay_pre 0=0
+Sigmoid sigmoid_34 1 1 att_8_time_decay_pre att_8_time_decay_sigmoid
+Exp exp_8 1 1 att_8_time_decay_sigmoid att_8_time_decay 1=-0.606531
+Reshape reshape_98 1 1 att_8_time_decay att_8_time_decay_reshape 0=64 1=1 2=12
+Split split_att_8_key_a 1 2 att_8_key_a att_8_key_a_0 att_8_key_a_1
+Reshape reshape_99 1 1 att_8_key_a_0 att_8_key_a_reshape 0=64 1=1 2=12
+Split split_att_8_value_final 1 3 att_8_value_final att_8_value_final_0 att_8_value_final_1 att_8_value_final_2
+Reshape reshape_100 1 1 att_8_value_final_0 att_8_value_final_0_reshape 0=64 1=12
+Reshape reshape_101 1 1 att_8_value_final_1 att_8_value_final_1_reshape 0=1 1=64 2=12
+MatMul matmul_att_8_vk 2 1 att_8_value_final_1_reshape att_8_key_a_reshape att_8_vk 0=0
+Split split_att_8_key_k_norm_reshape 1 2 att_8_key_k_norm_reshape att_8_kk_0 att_8_kk_1
+BinaryOp mul_108 1 1 att_8_kk_0 att_8_kk_0_neg 0=2 1=1 2=-1.0
+Reshape reshape_102 1 1 att_8_kk_0_neg att_8_kk_0_neg_reshape 0=1 1=64 2=12
+BinaryOp mul_109 2 1 att_8_kk_1 att_8_a_1 att_8_kk_a 0=2
+Reshape reshape_103 1 1 att_8_kk_a att_8_kk_a_reshape 0=64 1=1 2=12
+MatMul matmul_att_8_ab 2 1 att_8_kk_0_neg_reshape att_8_kk_a_reshape att_8_ab 0=0
+Split split_state_25_in 1 2 state_25_in att_8_state_prev_0 att_8_state_prev_1
+BinaryOp mul_110 2 1 att_8_state_prev_0 att_8_time_decay_reshape att_8_state_td 0=2
+MatMul matmul_att_8_sab 2 1 att_8_state_prev_1 att_8_ab att_8_sab 0=0
+BinaryOp add_109 2 1 att_8_sab att_8_vk att_8_sab_vk 0=0
+BinaryOp add_110 2 1 att_8_sab_vk att_8_state_td att_8_state_new 0=0
+Split split_att_8_state_new 1 2 att_8_state_new att_8_state_new_0 state_25_out
+Split split_att_8_receptance 1 2 att_8_receptance att_8_receptance_0 att_8_receptance_1
+Reshape reshape_104 1 1 att_8_receptance_0 att_8_receptance_reshape 0=1 1=64 2=12
+MatMul matmul_att_8_wkv_out 2 1 att_8_state_new_0 att_8_receptance_reshape att_8_wkv_out 0=0
+Reshape reshape_105 1 1 att_8_wkv_out att_8_wkv_out_flatten 0=768
+GroupNorm groupnorm_8 1 1 att_8_wkv_out_flatten att_8_x_gn 0=12 1=768 2=0.00064 3=1
+BinaryOp mul_111 2 1 att_8_receptance_1 att_8_key_a_1 att_8_receptance_key_a 0=2
+BinaryOp mul_112 2 1 att_8_receptance_key_a att_8_r_k att_8_rk 0=2
+Reshape reshape_106 1 1 att_8_rk att_8_rk_reshape 0=64 1=12
+Reduction sum_8 1 1 att_8_rk_reshape att_8_rk_sum 1=0 -23303=1,-1 4=1 5=1
+BinaryOp mul_113 2 1 att_8_value_final_0_reshape att_8_rk_sum att_8_value_final_0_rk 0=2
+Reshape reshape_107 1 1 att_8_value_final_0_rk att_8_value_final_0_rk_reshape 0=768
+BinaryOp add_111 2 1 att_8_value_final_0_rk_reshape att_8_x_gn att_8_x_gn_rkv 0=0
+BinaryOp mul_114 2 1 att_8_x_gn_rkv att_8_gate att_8_x_gate 0=2
+Gemm gemm_att_8_x_out 1 1 att_8_x_gate att_8_x_out 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+BinaryOp add_112 2 1 att_8_x_out att_8_x_last time_mixing_8_out 0=0
+Split split_time_mixing_8_out 1 2 time_mixing_8_out ffn_8_x_last ffn_8_x
+LayerNorm layernorm_17 1 1 ffn_8_x ffn_8_xx 0=768 1=0.00001 2=1
+Split split_ffn_8_xx 1 3 ffn_8_xx ffn_8_xx_0 ffn_8_xx_1 state_26_out
+BinaryOp sub_17 2 1 state_26_in ffn_8_xx_0 ffn_8_sx 0=1
+Split split_ffn_8_sx 1 1 ffn_8_sx ffn_8_sx_0
+MemoryData data_ffn_8_x_k 0 1 ffn_8_x_k 0=768 21=0
+BinaryOp mul_115 2 1 ffn_8_sx_0 ffn_8_x_k ffn_8_xk 0=2
+BinaryOp add_113 2 1 ffn_8_xk ffn_8_xx_1 ffn_8_xxk 0=0
+Gemm gemm_ffn_8_key 1 1 ffn_8_xxk ffn_8_key 4=0 5=1 6=0 7=0 8=3072 9=768 10=-1
+ReLU relu_8 1 1 ffn_8_key ffn_8_key_relu
+UnaryOp square_ffn_8_key_relu 1 1 ffn_8_key_relu ffn_8_key_relu_square 0=4
+Gemm gemm_ffn_8_value 1 1 ffn_8_key_relu_square ffn_8_value 4=0 5=1 6=0 7=0 8=768 9=3072 10=-1
+BinaryOp add_114 2 1 ffn_8_value ffn_8_x_last channel_mixing_8_out 0=0
+Split split_channel_mixing_8_out 1 2 channel_mixing_8_out att_9_x_last att_9_x
+LayerNorm layernorm_18 1 1 att_9_x att_9_xx 0=768 1=0.00001 2=1
+Split split_att_9_xx 1 3 att_9_xx att_9_xx_0 att_9_xx_1 state_27_out
+BinaryOp sub_18 2 1 state_27_in att_9_xx_0 att_9_sx 0=1
+MemoryData data_att_9_maa 0 1 att_9_maa 0=768 1=6 21=0
+BinaryOp mul_116 2 1 att_9_sx att_9_maa att_9_maa_x 0=2
+BinaryOp add_115 2 1 att_9_xx_1 att_9_maa_x att_9_maa_rwkvag 0=0
+Slice slice_att_9_maa_rwkvag 1 6 att_9_maa_rwkvag att_9_mr att_9_mw att_9_mk att_9_mv att_9_ma att_9_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
+Split split_att_9_mv 1 2 att_9_mv att_9_mv_0 att_9_mv_1
+Gemm gemm_att_9_key 1 1 att_9_mk att_9_key 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_9_value 1 1 att_9_mv_0 att_9_value 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_9_receptance 1 1 att_9_mr att_9_receptance 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_9_ma_lora_0 1 1 att_9_ma att_9_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+Gemm gemm_att_9_ma_lora_1 1 1 att_9_ma_lora_0 att_9_ma_lora_1 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_9_a0 0 1 att_9_a0 0=768 21=0
+BinaryOp add_116 2 1 att_9_ma_lora_1 att_9_a0 att_9_ma_lora_2 0=0
+Sigmoid sigmoid_35 1 1 att_9_ma_lora_2 att_9_a
+Split split_att_9_a 1 2 att_9_a att_9_a_0 att_9_a_1
+Gemm gemm_att_9_mg_lora 1 1 att_9_mg att_9_mg_lora 4=0 5=1 6=0 7=0 8=128 9=768 10=-1
+Sigmoid sigmoid_36 1 1 att_9_mg_lora att_9_mg_lora_sigmoid
+Gemm gemm_att_9_gate 1 1 att_9_mg_lora_sigmoid att_9_gate 4=0 5=1 6=0 7=0 8=768 9=128 10=-1
+MemoryData data_att_9_k_k 0 1 att_9_k_k 0=768 21=0
+MemoryData data_att_9_k_a 0 1 att_9_k_a 0=768 21=0
+MemoryData data_att_9_r_k 0 1 att_9_r_k 0=768 21=0
+Split split_att_9_key 1 2 att_9_key att_9_key_0 att_9_key_1
+BinaryOp mul_117 2 1 att_9_key_0 att_9_k_k att_9_key_k 0=2
+Reshape reshape_108 1 1 att_9_key_k att_9_key_k_reshape 0=64 1=1 2=12
+Normalize l2norm_9 1 1 att_9_key_k_reshape att_9_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
+Reshape reshape_109 1 1 att_9_key_k_norm att_9_key_k_norm_reshape 0=768
+BinaryOp add_117 1 1 att_9_a_0 att_9_a_0_minus 0=0 1=1 2=-1.0
+BinaryOp mul_118 2 1 att_9_a_0_minus att_9_k_a att_9_a_minus1_mulka 0=2
+BinaryOp add_118 1 1 att_9_a_minus1_mulka att_9_a_minus1_mulka_plus1 0=0 1=1 2=1.0
+BinaryOp mul_119 2 1 att_9_a_minus1_mulka_plus1 att_9_key_1 att_9_key_a 0=2
+Split split_att_9_value 1 2 att_9_value att_9_value_0 att_9_value_1
+Split split_v_first_8 1 2 v_first_8 att_9_v_first v_first_9
+BinaryOp minus_8 2 1 att_9_v_first att_9_value_1 att_9_vfirst_minus_value 0=1
+Gemm gemm_att_9_mv_lora_0 1 1 att_9_mv_1 att_9_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=768 10=-1
+Gemm gemm_att_9_mv_lora_1 1 1 att_9_mv_lora_0 att_9_mv_lora_1 4=0 5=1 6=0 7=0 8=768 9=32 10=-1
+MemoryData data_att_9_v0 0 1 att_9_v0 0=768 21=0
+BinaryOp add_119 2 1 att_9_mv_lora_1 att_9_v0 att_9_mv_lora_2 0=0
+Sigmoid sigmoid_37 1 1 att_9_mv_lora_2 att_9_mv_lora_sigmoid
+BinaryOp mul_120 2 1 att_9_mv_lora_sigmoid att_9_vfirst_minus_value att_9_vfirst_minus_value_mul 0=2
+BinaryOp add_120 2 1 att_9_value_0 att_9_vfirst_minus_value_mul att_9_value_final 0=0
+Gemm gemm_att_9_mw_lora 1 1 att_9_mw att_9_mw_lora 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+TanH tanh_att_9_mw_lora 1 1 att_9_mw_lora att_9_mw_lora_tanh
+Gemm gemm_att_9_mw_lora_tanh_linear 1 1 att_9_mw_lora_tanh att_9_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_9_td 0 1 att_9_td 0=768 21=0
+BinaryOp add_121 2 1 att_9_mw_lora_tanh_linear att_9_td att_9_time_decay_pre 0=0
+Sigmoid sigmoid_38 1 1 att_9_time_decay_pre att_9_time_decay_sigmoid
+Exp exp_9 1 1 att_9_time_decay_sigmoid att_9_time_decay 1=-0.606531
+Reshape reshape_110 1 1 att_9_time_decay att_9_time_decay_reshape 0=64 1=1 2=12
+Split split_att_9_key_a 1 2 att_9_key_a att_9_key_a_0 att_9_key_a_1
+Reshape reshape_111 1 1 att_9_key_a_0 att_9_key_a_reshape 0=64 1=1 2=12
+Split split_att_9_value_final 1 3 att_9_value_final att_9_value_final_0 att_9_value_final_1 att_9_value_final_2
+Reshape reshape_112 1 1 att_9_value_final_0 att_9_value_final_0_reshape 0=64 1=12
+Reshape reshape_113 1 1 att_9_value_final_1 att_9_value_final_1_reshape 0=1 1=64 2=12
+MatMul matmul_att_9_vk 2 1 att_9_value_final_1_reshape att_9_key_a_reshape att_9_vk 0=0
+Split split_att_9_key_k_norm_reshape 1 2 att_9_key_k_norm_reshape att_9_kk_0 att_9_kk_1
+BinaryOp mul_121 1 1 att_9_kk_0 att_9_kk_0_neg 0=2 1=1 2=-1.0
+Reshape reshape_114 1 1 att_9_kk_0_neg att_9_kk_0_neg_reshape 0=1 1=64 2=12
+BinaryOp mul_122 2 1 att_9_kk_1 att_9_a_1 att_9_kk_a 0=2
+Reshape reshape_115 1 1 att_9_kk_a att_9_kk_a_reshape 0=64 1=1 2=12
+MatMul matmul_att_9_ab 2 1 att_9_kk_0_neg_reshape att_9_kk_a_reshape att_9_ab 0=0
+Split split_state_28_in 1 2 state_28_in att_9_state_prev_0 att_9_state_prev_1
+BinaryOp mul_123 2 1 att_9_state_prev_0 att_9_time_decay_reshape att_9_state_td 0=2
+MatMul matmul_att_9_sab 2 1 att_9_state_prev_1 att_9_ab att_9_sab 0=0
+BinaryOp add_122 2 1 att_9_sab att_9_vk att_9_sab_vk 0=0
+BinaryOp add_123 2 1 att_9_sab_vk att_9_state_td att_9_state_new 0=0
+Split split_att_9_state_new 1 2 att_9_state_new att_9_state_new_0 state_28_out
+Split split_att_9_receptance 1 2 att_9_receptance att_9_receptance_0 att_9_receptance_1
+Reshape reshape_116 1 1 att_9_receptance_0 att_9_receptance_reshape 0=1 1=64 2=12
+MatMul matmul_att_9_wkv_out 2 1 att_9_state_new_0 att_9_receptance_reshape att_9_wkv_out 0=0
+Reshape reshape_117 1 1 att_9_wkv_out att_9_wkv_out_flatten 0=768
+GroupNorm groupnorm_9 1 1 att_9_wkv_out_flatten att_9_x_gn 0=12 1=768 2=0.00064 3=1
+BinaryOp mul_124 2 1 att_9_receptance_1 att_9_key_a_1 att_9_receptance_key_a 0=2
+BinaryOp mul_125 2 1 att_9_receptance_key_a att_9_r_k att_9_rk 0=2
+Reshape reshape_118 1 1 att_9_rk att_9_rk_reshape 0=64 1=12
+Reduction sum_9 1 1 att_9_rk_reshape att_9_rk_sum 1=0 -23303=1,-1 4=1 5=1
+BinaryOp mul_126 2 1 att_9_value_final_0_reshape att_9_rk_sum att_9_value_final_0_rk 0=2
+Reshape reshape_119 1 1 att_9_value_final_0_rk att_9_value_final_0_rk_reshape 0=768
+BinaryOp add_124 2 1 att_9_value_final_0_rk_reshape att_9_x_gn att_9_x_gn_rkv 0=0
+BinaryOp mul_127 2 1 att_9_x_gn_rkv att_9_gate att_9_x_gate 0=2
+Gemm gemm_att_9_x_out 1 1 att_9_x_gate att_9_x_out 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+BinaryOp add_125 2 1 att_9_x_out att_9_x_last time_mixing_9_out 0=0
+Split split_time_mixing_9_out 1 2 time_mixing_9_out ffn_9_x_last ffn_9_x
+LayerNorm layernorm_19 1 1 ffn_9_x ffn_9_xx 0=768 1=0.00001 2=1
+Split split_ffn_9_xx 1 3 ffn_9_xx ffn_9_xx_0 ffn_9_xx_1 state_29_out
+BinaryOp sub_19 2 1 state_29_in ffn_9_xx_0 ffn_9_sx 0=1
+Split split_ffn_9_sx 1 1 ffn_9_sx ffn_9_sx_0
+MemoryData data_ffn_9_x_k 0 1 ffn_9_x_k 0=768 21=0
+BinaryOp mul_128 2 1 ffn_9_sx_0 ffn_9_x_k ffn_9_xk 0=2
+BinaryOp add_126 2 1 ffn_9_xk ffn_9_xx_1 ffn_9_xxk 0=0
+Gemm gemm_ffn_9_key 1 1 ffn_9_xxk ffn_9_key 4=0 5=1 6=0 7=0 8=3072 9=768 10=-1
+ReLU relu_9 1 1 ffn_9_key ffn_9_key_relu
+UnaryOp square_ffn_9_key_relu 1 1 ffn_9_key_relu ffn_9_key_relu_square 0=4
+Gemm gemm_ffn_9_value 1 1 ffn_9_key_relu_square ffn_9_value 4=0 5=1 6=0 7=0 8=768 9=3072 10=-1
+BinaryOp add_127 2 1 ffn_9_value ffn_9_x_last channel_mixing_9_out 0=0
+Split split_channel_mixing_9_out 1 2 channel_mixing_9_out att_10_x_last att_10_x
+LayerNorm layernorm_20 1 1 att_10_x att_10_xx 0=768 1=0.00001 2=1
+Split split_att_10_xx 1 3 att_10_xx att_10_xx_0 att_10_xx_1 state_30_out
+BinaryOp sub_20 2 1 state_30_in att_10_xx_0 att_10_sx 0=1
+MemoryData data_att_10_maa 0 1 att_10_maa 0=768 1=6 21=0
+BinaryOp mul_129 2 1 att_10_sx att_10_maa att_10_maa_x 0=2
+BinaryOp add_128 2 1 att_10_xx_1 att_10_maa_x att_10_maa_rwkvag 0=0
+Slice slice_att_10_maa_rwkvag 1 6 att_10_maa_rwkvag att_10_mr att_10_mw att_10_mk att_10_mv att_10_ma att_10_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
+Split split_att_10_mv 1 2 att_10_mv att_10_mv_0 att_10_mv_1
+Gemm gemm_att_10_key 1 1 att_10_mk att_10_key 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_10_value 1 1 att_10_mv_0 att_10_value 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_10_receptance 1 1 att_10_mr att_10_receptance 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_10_ma_lora_0 1 1 att_10_ma att_10_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+Gemm gemm_att_10_ma_lora_1 1 1 att_10_ma_lora_0 att_10_ma_lora_1 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_10_a0 0 1 att_10_a0 0=768 21=0
+BinaryOp add_129 2 1 att_10_ma_lora_1 att_10_a0 att_10_ma_lora_2 0=0
+Sigmoid sigmoid_39 1 1 att_10_ma_lora_2 att_10_a
+Split split_att_10_a 1 2 att_10_a att_10_a_0 att_10_a_1
+Gemm gemm_att_10_mg_lora 1 1 att_10_mg att_10_mg_lora 4=0 5=1 6=0 7=0 8=128 9=768 10=-1
+Sigmoid sigmoid_40 1 1 att_10_mg_lora att_10_mg_lora_sigmoid
+Gemm gemm_att_10_gate 1 1 att_10_mg_lora_sigmoid att_10_gate 4=0 5=1 6=0 7=0 8=768 9=128 10=-1
+MemoryData data_att_10_k_k 0 1 att_10_k_k 0=768 21=0
+MemoryData data_att_10_k_a 0 1 att_10_k_a 0=768 21=0
+MemoryData data_att_10_r_k 0 1 att_10_r_k 0=768 21=0
+Split split_att_10_key 1 2 att_10_key att_10_key_0 att_10_key_1
+BinaryOp mul_130 2 1 att_10_key_0 att_10_k_k att_10_key_k 0=2
+Reshape reshape_120 1 1 att_10_key_k att_10_key_k_reshape 0=64 1=1 2=12
+Normalize l2norm_10 1 1 att_10_key_k_reshape att_10_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
+Reshape reshape_121 1 1 att_10_key_k_norm att_10_key_k_norm_reshape 0=768
+BinaryOp add_130 1 1 att_10_a_0 att_10_a_0_minus 0=0 1=1 2=-1.0
+BinaryOp mul_131 2 1 att_10_a_0_minus att_10_k_a att_10_a_minus1_mulka 0=2
+BinaryOp add_131 1 1 att_10_a_minus1_mulka att_10_a_minus1_mulka_plus1 0=0 1=1 2=1.0
+BinaryOp mul_132 2 1 att_10_a_minus1_mulka_plus1 att_10_key_1 att_10_key_a 0=2
+Split split_att_10_value 1 2 att_10_value att_10_value_0 att_10_value_1
+Split split_v_first_9 1 2 v_first_9 att_10_v_first v_first_10
+BinaryOp minus_9 2 1 att_10_v_first att_10_value_1 att_10_vfirst_minus_value 0=1
+Gemm gemm_att_10_mv_lora_0 1 1 att_10_mv_1 att_10_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=768 10=-1
+Gemm gemm_att_10_mv_lora_1 1 1 att_10_mv_lora_0 att_10_mv_lora_1 4=0 5=1 6=0 7=0 8=768 9=32 10=-1
+MemoryData data_att_10_v0 0 1 att_10_v0 0=768 21=0
+BinaryOp add_132 2 1 att_10_mv_lora_1 att_10_v0 att_10_mv_lora_2 0=0
+Sigmoid sigmoid_41 1 1 att_10_mv_lora_2 att_10_mv_lora_sigmoid
+BinaryOp mul_133 2 1 att_10_mv_lora_sigmoid att_10_vfirst_minus_value att_10_vfirst_minus_value_mul 0=2
+BinaryOp add_133 2 1 att_10_value_0 att_10_vfirst_minus_value_mul att_10_value_final 0=0
+Gemm gemm_att_10_mw_lora 1 1 att_10_mw att_10_mw_lora 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+TanH tanh_att_10_mw_lora 1 1 att_10_mw_lora att_10_mw_lora_tanh
+Gemm gemm_att_10_mw_lora_tanh_linear 1 1 att_10_mw_lora_tanh att_10_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_10_td 0 1 att_10_td 0=768 21=0
+BinaryOp add_134 2 1 att_10_mw_lora_tanh_linear att_10_td att_10_time_decay_pre 0=0
+Sigmoid sigmoid_42 1 1 att_10_time_decay_pre att_10_time_decay_sigmoid
+Exp exp_10 1 1 att_10_time_decay_sigmoid att_10_time_decay 1=-0.606531
+Reshape reshape_122 1 1 att_10_time_decay att_10_time_decay_reshape 0=64 1=1 2=12
+Split split_att_10_key_a 1 2 att_10_key_a att_10_key_a_0 att_10_key_a_1
+Reshape reshape_123 1 1 att_10_key_a_0 att_10_key_a_reshape 0=64 1=1 2=12
+Split split_att_10_value_final 1 3 att_10_value_final att_10_value_final_0 att_10_value_final_1 att_10_value_final_2
+Reshape reshape_124 1 1 att_10_value_final_0 att_10_value_final_0_reshape 0=64 1=12
+Reshape reshape_125 1 1 att_10_value_final_1 att_10_value_final_1_reshape 0=1 1=64 2=12
+MatMul matmul_att_10_vk 2 1 att_10_value_final_1_reshape att_10_key_a_reshape att_10_vk 0=0
+Split split_att_10_key_k_norm_reshape 1 2 att_10_key_k_norm_reshape att_10_kk_0 att_10_kk_1
+BinaryOp mul_134 1 1 att_10_kk_0 att_10_kk_0_neg 0=2 1=1 2=-1.0
+Reshape reshape_126 1 1 att_10_kk_0_neg att_10_kk_0_neg_reshape 0=1 1=64 2=12
+BinaryOp mul_135 2 1 att_10_kk_1 att_10_a_1 att_10_kk_a 0=2
+Reshape reshape_127 1 1 att_10_kk_a att_10_kk_a_reshape 0=64 1=1 2=12
+MatMul matmul_att_10_ab 2 1 att_10_kk_0_neg_reshape att_10_kk_a_reshape att_10_ab 0=0
+Split split_state_31_in 1 2 state_31_in att_10_state_prev_0 att_10_state_prev_1
+BinaryOp mul_136 2 1 att_10_state_prev_0 att_10_time_decay_reshape att_10_state_td 0=2
+MatMul matmul_att_10_sab 2 1 att_10_state_prev_1 att_10_ab att_10_sab 0=0
+BinaryOp add_135 2 1 att_10_sab att_10_vk att_10_sab_vk 0=0
+BinaryOp add_136 2 1 att_10_sab_vk att_10_state_td att_10_state_new 0=0
+Split split_att_10_state_new 1 2 att_10_state_new att_10_state_new_0 state_31_out
+Split split_att_10_receptance 1 2 att_10_receptance att_10_receptance_0 att_10_receptance_1
+Reshape reshape_128 1 1 att_10_receptance_0 att_10_receptance_reshape 0=1 1=64 2=12
+MatMul matmul_att_10_wkv_out 2 1 att_10_state_new_0 att_10_receptance_reshape att_10_wkv_out 0=0
+Reshape reshape_129 1 1 att_10_wkv_out att_10_wkv_out_flatten 0=768
+GroupNorm groupnorm_10 1 1 att_10_wkv_out_flatten att_10_x_gn 0=12 1=768 2=0.00064 3=1
+BinaryOp mul_137 2 1 att_10_receptance_1 att_10_key_a_1 att_10_receptance_key_a 0=2
+BinaryOp mul_138 2 1 att_10_receptance_key_a att_10_r_k att_10_rk 0=2
+Reshape reshape_130 1 1 att_10_rk att_10_rk_reshape 0=64 1=12
+Reduction sum_10 1 1 att_10_rk_reshape att_10_rk_sum 1=0 -23303=1,-1 4=1 5=1
+BinaryOp mul_139 2 1 att_10_value_final_0_reshape att_10_rk_sum att_10_value_final_0_rk 0=2
+Reshape reshape_131 1 1 att_10_value_final_0_rk att_10_value_final_0_rk_reshape 0=768
+BinaryOp add_137 2 1 att_10_value_final_0_rk_reshape att_10_x_gn att_10_x_gn_rkv 0=0
+BinaryOp mul_140 2 1 att_10_x_gn_rkv att_10_gate att_10_x_gate 0=2
+Gemm gemm_att_10_x_out 1 1 att_10_x_gate att_10_x_out 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+BinaryOp add_138 2 1 att_10_x_out att_10_x_last time_mixing_10_out 0=0
+Split split_time_mixing_10_out 1 2 time_mixing_10_out ffn_10_x_last ffn_10_x
+LayerNorm layernorm_21 1 1 ffn_10_x ffn_10_xx 0=768 1=0.00001 2=1
+Split split_ffn_10_xx 1 3 ffn_10_xx ffn_10_xx_0 ffn_10_xx_1 state_32_out
+BinaryOp sub_21 2 1 state_32_in ffn_10_xx_0 ffn_10_sx 0=1
+Split split_ffn_10_sx 1 1 ffn_10_sx ffn_10_sx_0
+MemoryData data_ffn_10_x_k 0 1 ffn_10_x_k 0=768 21=0
+BinaryOp mul_141 2 1 ffn_10_sx_0 ffn_10_x_k ffn_10_xk 0=2
+BinaryOp add_139 2 1 ffn_10_xk ffn_10_xx_1 ffn_10_xxk 0=0
+Gemm gemm_ffn_10_key 1 1 ffn_10_xxk ffn_10_key 4=0 5=1 6=0 7=0 8=3072 9=768 10=-1
+ReLU relu_10 1 1 ffn_10_key ffn_10_key_relu
+UnaryOp square_ffn_10_key_relu 1 1 ffn_10_key_relu ffn_10_key_relu_square 0=4
+Gemm gemm_ffn_10_value 1 1 ffn_10_key_relu_square ffn_10_value 4=0 5=1 6=0 7=0 8=768 9=3072 10=-1
+BinaryOp add_140 2 1 ffn_10_value ffn_10_x_last channel_mixing_10_out 0=0
+Split split_channel_mixing_10_out 1 2 channel_mixing_10_out att_11_x_last att_11_x
+LayerNorm layernorm_22 1 1 att_11_x att_11_xx 0=768 1=0.00001 2=1
+Split split_att_11_xx 1 3 att_11_xx att_11_xx_0 att_11_xx_1 state_33_out
+BinaryOp sub_22 2 1 state_33_in att_11_xx_0 att_11_sx 0=1
+MemoryData data_att_11_maa 0 1 att_11_maa 0=768 1=6 21=0
+BinaryOp mul_142 2 1 att_11_sx att_11_maa att_11_maa_x 0=2
+BinaryOp add_141 2 1 att_11_xx_1 att_11_maa_x att_11_maa_rwkvag 0=0
+Slice slice_att_11_maa_rwkvag 1 6 att_11_maa_rwkvag att_11_mr att_11_mw att_11_mk att_11_mv att_11_ma att_11_mg -23300=6,-233,-233,-233,-233,-233,-233 1=0
+Split split_att_11_mv 1 2 att_11_mv att_11_mv_0 att_11_mv_1
+Gemm gemm_att_11_key 1 1 att_11_mk att_11_key 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_11_value 1 1 att_11_mv_0 att_11_value 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_11_receptance 1 1 att_11_mr att_11_receptance 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+Gemm gemm_att_11_ma_lora_0 1 1 att_11_ma att_11_ma_lora_0 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+Gemm gemm_att_11_ma_lora_1 1 1 att_11_ma_lora_0 att_11_ma_lora_1 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_11_a0 0 1 att_11_a0 0=768 21=0
+BinaryOp add_142 2 1 att_11_ma_lora_1 att_11_a0 att_11_ma_lora_2 0=0
+Sigmoid sigmoid_43 1 1 att_11_ma_lora_2 att_11_a
+Split split_att_11_a 1 2 att_11_a att_11_a_0 att_11_a_1
+Gemm gemm_att_11_mg_lora 1 1 att_11_mg att_11_mg_lora 4=0 5=1 6=0 7=0 8=128 9=768 10=-1
+Sigmoid sigmoid_44 1 1 att_11_mg_lora att_11_mg_lora_sigmoid
+Gemm gemm_att_11_gate 1 1 att_11_mg_lora_sigmoid att_11_gate 4=0 5=1 6=0 7=0 8=768 9=128 10=-1
+MemoryData data_att_11_k_k 0 1 att_11_k_k 0=768 21=0
+MemoryData data_att_11_k_a 0 1 att_11_k_a 0=768 21=0
+MemoryData data_att_11_r_k 0 1 att_11_r_k 0=768 21=0
+Split split_att_11_key 1 2 att_11_key att_11_key_0 att_11_key_1
+BinaryOp mul_143 2 1 att_11_key_0 att_11_k_k att_11_key_k 0=2
+Reshape reshape_132 1 1 att_11_key_k att_11_key_k_reshape 0=64 1=1 2=12
+Normalize l2norm_11 1 1 att_11_key_k_reshape att_11_key_k_norm 0=1 1=1 2=0.0000001 3=1 4=0 9=1
+Reshape reshape_133 1 1 att_11_key_k_norm att_11_key_k_norm_reshape 0=768
+BinaryOp add_143 1 1 att_11_a_0 att_11_a_0_minus 0=0 1=1 2=-1.0
+BinaryOp mul_144 2 1 att_11_a_0_minus att_11_k_a att_11_a_minus1_mulka 0=2
+BinaryOp add_144 1 1 att_11_a_minus1_mulka att_11_a_minus1_mulka_plus1 0=0 1=1 2=1.0
+BinaryOp mul_145 2 1 att_11_a_minus1_mulka_plus1 att_11_key_1 att_11_key_a 0=2
+Split split_att_11_value 1 2 att_11_value att_11_value_0 att_11_value_1
+Split split_v_first_10 1 2 v_first_10 att_11_v_first v_first_11
+BinaryOp minus_10 2 1 att_11_v_first att_11_value_1 att_11_vfirst_minus_value 0=1
+Gemm gemm_att_11_mv_lora_0 1 1 att_11_mv_1 att_11_mv_lora_0 4=0 5=1 6=0 7=0 8=32 9=768 10=-1
+Gemm gemm_att_11_mv_lora_1 1 1 att_11_mv_lora_0 att_11_mv_lora_1 4=0 5=1 6=0 7=0 8=768 9=32 10=-1
+MemoryData data_att_11_v0 0 1 att_11_v0 0=768 21=0
+BinaryOp add_145 2 1 att_11_mv_lora_1 att_11_v0 att_11_mv_lora_2 0=0
+Sigmoid sigmoid_45 1 1 att_11_mv_lora_2 att_11_mv_lora_sigmoid
+BinaryOp mul_146 2 1 att_11_mv_lora_sigmoid att_11_vfirst_minus_value att_11_vfirst_minus_value_mul 0=2
+BinaryOp add_146 2 1 att_11_value_0 att_11_vfirst_minus_value_mul att_11_value_final 0=0
+Gemm gemm_att_11_mw_lora 1 1 att_11_mw att_11_mw_lora 4=0 5=1 6=0 7=0 8=64 9=768 10=-1
+TanH tanh_att_11_mw_lora 1 1 att_11_mw_lora att_11_mw_lora_tanh
+Gemm gemm_att_11_mw_lora_tanh_linear 1 1 att_11_mw_lora_tanh att_11_mw_lora_tanh_linear 4=0 5=1 6=0 7=0 8=768 9=64 10=-1
+MemoryData data_att_11_td 0 1 att_11_td 0=768 21=0
+BinaryOp add_147 2 1 att_11_mw_lora_tanh_linear att_11_td att_11_time_decay_pre 0=0
+Sigmoid sigmoid_46 1 1 att_11_time_decay_pre att_11_time_decay_sigmoid
+Exp exp_11 1 1 att_11_time_decay_sigmoid att_11_time_decay 1=-0.606531
+Reshape reshape_134 1 1 att_11_time_decay att_11_time_decay_reshape 0=64 1=1 2=12
+Split split_att_11_key_a 1 2 att_11_key_a att_11_key_a_0 att_11_key_a_1
+Reshape reshape_135 1 1 att_11_key_a_0 att_11_key_a_reshape 0=64 1=1 2=12
+Split split_att_11_value_final 1 3 att_11_value_final att_11_value_final_0 att_11_value_final_1 att_11_value_final_2
+Reshape reshape_136 1 1 att_11_value_final_0 att_11_value_final_0_reshape 0=64 1=12
+Reshape reshape_137 1 1 att_11_value_final_1 att_11_value_final_1_reshape 0=1 1=64 2=12
+MatMul matmul_att_11_vk 2 1 att_11_value_final_1_reshape att_11_key_a_reshape att_11_vk 0=0
+Split split_att_11_key_k_norm_reshape 1 2 att_11_key_k_norm_reshape att_11_kk_0 att_11_kk_1
+BinaryOp mul_147 1 1 att_11_kk_0 att_11_kk_0_neg 0=2 1=1 2=-1.0
+Reshape reshape_138 1 1 att_11_kk_0_neg att_11_kk_0_neg_reshape 0=1 1=64 2=12
+BinaryOp mul_148 2 1 att_11_kk_1 att_11_a_1 att_11_kk_a 0=2
+Reshape reshape_139 1 1 att_11_kk_a att_11_kk_a_reshape 0=64 1=1 2=12
+MatMul matmul_att_11_ab 2 1 att_11_kk_0_neg_reshape att_11_kk_a_reshape att_11_ab 0=0
+Split split_state_34_in 1 2 state_34_in att_11_state_prev_0 att_11_state_prev_1
+BinaryOp mul_149 2 1 att_11_state_prev_0 att_11_time_decay_reshape att_11_state_td 0=2
+MatMul matmul_att_11_sab 2 1 att_11_state_prev_1 att_11_ab att_11_sab 0=0
+BinaryOp add_148 2 1 att_11_sab att_11_vk att_11_sab_vk 0=0
+BinaryOp add_149 2 1 att_11_sab_vk att_11_state_td att_11_state_new 0=0
+Split split_att_11_state_new 1 2 att_11_state_new att_11_state_new_0 state_34_out
+Split split_att_11_receptance 1 2 att_11_receptance att_11_receptance_0 att_11_receptance_1
+Reshape reshape_140 1 1 att_11_receptance_0 att_11_receptance_reshape 0=1 1=64 2=12
+MatMul matmul_att_11_wkv_out 2 1 att_11_state_new_0 att_11_receptance_reshape att_11_wkv_out 0=0
+Reshape reshape_141 1 1 att_11_wkv_out att_11_wkv_out_flatten 0=768
+GroupNorm groupnorm_11 1 1 att_11_wkv_out_flatten att_11_x_gn 0=12 1=768 2=0.00064 3=1
+BinaryOp mul_150 2 1 att_11_receptance_1 att_11_key_a_1 att_11_receptance_key_a 0=2
+BinaryOp mul_151 2 1 att_11_receptance_key_a att_11_r_k att_11_rk 0=2
+Reshape reshape_142 1 1 att_11_rk att_11_rk_reshape 0=64 1=12
+Reduction sum_11 1 1 att_11_rk_reshape att_11_rk_sum 1=0 -23303=1,-1 4=1 5=1
+BinaryOp mul_152 2 1 att_11_value_final_0_reshape att_11_rk_sum att_11_value_final_0_rk 0=2
+Reshape reshape_143 1 1 att_11_value_final_0_rk att_11_value_final_0_rk_reshape 0=768
+BinaryOp add_150 2 1 att_11_value_final_0_rk_reshape att_11_x_gn att_11_x_gn_rkv 0=0
+BinaryOp mul_153 2 1 att_11_x_gn_rkv att_11_gate att_11_x_gate 0=2
+Gemm gemm_att_11_x_out 1 1 att_11_x_gate att_11_x_out 4=0 5=1 6=0 7=0 8=768 9=768 10=-1
+BinaryOp add_151 2 1 att_11_x_out att_11_x_last time_mixing_11_out 0=0
+Split split_time_mixing_11_out 1 2 time_mixing_11_out ffn_11_x_last ffn_11_x
+LayerNorm layernorm_23 1 1 ffn_11_x ffn_11_xx 0=768 1=0.00001 2=1
+Split split_ffn_11_xx 1 3 ffn_11_xx ffn_11_xx_0 ffn_11_xx_1 state_35_out
+BinaryOp sub_23 2 1 state_35_in ffn_11_xx_0 ffn_11_sx 0=1
+Split split_ffn_11_sx 1 1 ffn_11_sx ffn_11_sx_0
+MemoryData data_ffn_11_x_k 0 1 ffn_11_x_k 0=768 21=0
+BinaryOp mul_154 2 1 ffn_11_sx_0 ffn_11_x_k ffn_11_xk 0=2
+BinaryOp add_152 2 1 ffn_11_xk ffn_11_xx_1 ffn_11_xxk 0=0
+Gemm gemm_ffn_11_key 1 1 ffn_11_xxk ffn_11_key 4=0 5=1 6=0 7=0 8=3072 9=768 10=-1
+ReLU relu_11 1 1 ffn_11_key ffn_11_key_relu
+UnaryOp square_ffn_11_key_relu 1 1 ffn_11_key_relu ffn_11_key_relu_square 0=4
+Gemm gemm_ffn_11_value 1 1 ffn_11_key_relu_square ffn_11_value 4=0 5=1 6=0 7=0 8=768 9=3072 10=-1
+BinaryOp add_153 2 1 ffn_11_value ffn_11_x_last channel_mixing_11_out 0=0
+LayerNorm layernorm_24 1 1 channel_mixing_11_out norm_head 0=768 1=0.00001 2=1
+Gemm gemm_logits 1 1 norm_head logits 4=0 5=1 6=0 7=0 8=65536 9=768 10=-1