This is a fast 15B parameter 2x Self Merge MoE

1.5 Epoch of my sythetic dataset
Eval Loss : ~0.4

model-00001-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7b58ce785ecd7bbc157c747d6e367f55d5e2116e12094532265392991ee899ad
+oid sha256:d5a6c784638d4a1db392a7e53b2a5362832b304cdddf3c34f7dec23122808ea8
 size 4915719208

model-00002-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e094b6e3dcb3ed1f5c3ef4edfdf5521b35194356a4c9e437625076d6fd950ecf
+oid sha256:615fd0ef52b63e08739547c895045f22384c4959490718c96bc4d117f198c199
 size 4996656336

model-00003-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:89cf28b918c5a0536a57984e9f98b1938b16b3021441bf2fbadf58c00aeb63ce
+oid sha256:9a632611124f73db00a86890209a095901054defa8817772c677f412215d3f2d
 size 4959956208

model-00004-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dcd65405c1f4b809a736c013bb5dc3156406a2b212791580c8cdcf6bc19fa93c
+oid sha256:6cc4ef8495a4d93add5e9839b13f72af374cdae79f06f1474839002a0fa775a9
 size 4870847960

model-00005-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5d5dc3b0e8f80f0780f0a128da3ad0ed43d878cb66e97db0773c1fc10142e02b
+oid sha256:aad09cf95dfaa212b3c56ebe1981d5344df4d40578f2245097905155e3440fe4
 size 4996656392

model-00006-of-00007.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3bdcf19c7e20e3ebeb67ad8148b69e77d2eb4b0d5055c2beb6410457d16dcb91
+oid sha256:4a00ccfa365cb3d54b3dc6f9ef548197984255cbae5fbd18432e700366500fde
 size 4959956224