RikkaBotan/fineweb_model

itr: 9000, Loss: 2.591947

9099it [49:44, 3.11it/s]itr: 9100, Loss: 2.681541

9199it [50:16, 3.11it/s]itr: 9200, Loss: 2.705647

9299it [50:48, 3.11it/s]itr: 9300, Loss: 2.701258

9399it [51:20, 3.11it/s]itr: 9400, Loss: 2.688398

    vocab_size= 32768,
    hidden_size=512,
    state_size=1024,
    segment_size = 8,
    heads=8,
    si_groups=2,
    sea_inter_size=1024,
    slc_inter_size=768,
    slc_kernel_size=3,
    ssrm_num=6,
    slc_ssr_num=1,
    sea_ssr_num=8,
    layers=6,
    rep=1,
    slc_multi=1.0,
    bias=False,
    device=device,
    max_position=16384,
    top_k=64, #1024,
    tiny_mode=False,
    parallel=2.0,
    mask_prob=0.0