Epoch 0 - Val loss 1.1416

README.md

@@ -0,0 +1,12 @@
+---
+license: apache-2.0
+pipeline_tag: fill-mask
+tags:
+- model_hub_mixin
+- pytorch_model_hub_mixin
+---
+
+This model has been pushed to the Hub using the [PytorchModelHubMixin](https://huggingface.co/docs/huggingface_hub/package_reference/mixins#huggingface_hub.PyTorchModelHubMixin) integration:
+- Code: [More Information Needed]
+- Paper: [More Information Needed]
+- Docs: [More Information Needed]

config.json

@@ -0,0 +1,18 @@
+{
+  "att_groups": 8,
+  "att_heads": 16,
+  "att_query_groups": 8,
+  "cross_att_type": "sqa",
+  "embed_dim": 256,
+  "ff_activation": "silu",
+  "ff_dim": 768,
+  "ff_dropout": 0.1,
+  "num_layers": 10,
+  "self_att_type": "sqa",
+  "seq_len": 256,
+  "skip_memory_cross_attention": true,
+  "stm_size": 256,
+  "use_flash_attention": false,
+  "use_gated": true,
+  "vocab_size": 10000
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9f8f982a15c83f6369f0754b8c09d6253d75e15a2f37c109fff845f7de7f77aa
+size 41821168