model upload

README.md

@@ -1,3 +1,54 @@
----
-license: apache-2.0
----
+---
+base_model: []
+library_name: transformers
+tags:
+- mergekit
+- merge
+
+---
+# mixtral-upscaled
+
+This is a merge of pre-trained language models created using [mergekit](https://github.com/cg123/mergekit).
+
+## Merge Details
+### Merge Method
+
+This model was merged using the passthrough merge method.
+
+### Models Merged
+
+The following models were included in the merge:
+* ./Mixtral-8x7B-Instruct-v0.1
+
+### Configuration
+
+The following YAML configuration was used to produce this model:
+
+```yaml
+merge_method: passthrough
+slices:
+  - sources:
+      - model: ./Mixtral-8x7B-Instruct-v0.1
+        layer_range: [0, 8]
+  - sources:
+      - model: ./Mixtral-8x7B-Instruct-v0.1
+        layer_range: [4, 12]
+  - sources:
+      - model: ./Mixtral-8x7B-Instruct-v0.1
+        layer_range: [8, 16]
+  - sources:
+      - model: ./Mixtral-8x7B-Instruct-v0.1
+        layer_range: [12, 20]
+  - sources:
+      - model: ./Mixtral-8x7B-Instruct-v0.1
+        layer_range: [16, 24]
+  - sources:
+      - model: ./Mixtral-8x7B-Instruct-v0.1
+        layer_range: [20, 28]
+  - sources:
+      - model: ./Mixtral-8x7B-Instruct-v0.1
+        layer_range: [24, 32]
+dtype: bfloat16
+tokenizer_source: base
+
+```

config.json

@@ -0,0 +1,30 @@
+{
+  "_name_or_path": "./Mixtral-8x7B-Instruct-v0.1",
+  "architectures": [
+    "MixtralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mixtral",
+  "num_attention_heads": 32,
+  "num_experts_per_tok": 2,
+  "num_hidden_layers": 56,
+  "num_key_value_heads": 8,
+  "num_local_experts": 8,
+  "output_router_logits": false,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 1000000.0,
+  "router_aux_loss_coef": 0.02,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.38.1",
+  "use_cache": true,
+  "vocab_size": 32000
+}

mergekit_config.yml

@@ -0,0 +1,25 @@
+merge_method: passthrough
+slices:
+  - sources:
+      - model: ./Mixtral-8x7B-Instruct-v0.1
+        layer_range: [0, 8]
+  - sources:
+      - model: ./Mixtral-8x7B-Instruct-v0.1
+        layer_range: [4, 12]
+  - sources:
+      - model: ./Mixtral-8x7B-Instruct-v0.1
+        layer_range: [8, 16]
+  - sources:
+      - model: ./Mixtral-8x7B-Instruct-v0.1
+        layer_range: [12, 20]
+  - sources:
+      - model: ./Mixtral-8x7B-Instruct-v0.1
+        layer_range: [16, 24]
+  - sources:
+      - model: ./Mixtral-8x7B-Instruct-v0.1
+        layer_range: [20, 28]
+  - sources:
+      - model: ./Mixtral-8x7B-Instruct-v0.1
+        layer_range: [24, 32]
+dtype: bfloat16
+tokenizer_source: base

model-00001-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e6c09933a9a453c4c5ebf8f43ea35f0e3dd16f86cc33c54323f2ed6e080df32d
+size 9993254680

model-00002-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:74c215757d5a143741f8d5008dd149ab56e9b55e72273938dec941723836f407
+size 9899014072

model-00003-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0402a2e86bf72d5758cd9eec0768c5f7cc6b7a846dac18aa35b31c394f0ef0a0
+size 9915511056

model-00004-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a10a453dcc571e97c65c9ad41a2f585a6fdbb6db1460f4513438d23a8dc373e
+size 9899064448

model-00005-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75d5575c027b0cd0c457f170a73264c1ce0fe952aa6317b27c2ca9aa24d90543
+size 9915511120

model-00006-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df8fe2359a81096208a31b96b92f5aa6a50cd316f3073172f5a36611d02271dc
+size 9915511120

model-00007-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c513b3ea3c2c9ea00e15d483c4e9f45761aa2e3172c85bc4c94277b9de856bef
+size 9899064480

model-00008-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:470b515fd86aea4b0451c5b179fc067bab1dea3ddc4f4083e713aa2db3b29802
+size 9915511120

model-00009-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:68272ae6467c46927d0e94465fc3833f86a58cf2c20b6fe0fdb90dc0f7446cdb
+size 9915511120

model-00010-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc1dd70c8d64d9b4d4e36cfa794ebfee95257b5c43d241b61c685bd9db8e3d7d
+size 9966008120

model-00011-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7f489a10aefe8f11bd34995e85e850ce76bbdab697b6cb2b01e9140107525ddc
+size 9899064488

model-00012-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6272a0673ceab217cf8fb13c9b1aca0e4b237073b129280f5e4566ff178d55d0
+size 9915511120

model-00013-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:497852cdbcc4faa1edddcfb72aa78d5dbc3c0b6824471661aa91c94a8ff6b5d0
+size 9915511120

model-00014-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7aaf11afb1c7b80a168a338977ee562f66585836552a40fecbda9fb52ec71c04
+size 9899064488

model-00015-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:720c4c02274d2da5555af238b8072cbb426ab3073346c22fa932567cb47ceaaa
+size 9948999560

model-00016-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63022a88a71114e047d7b257f4cf72bb125cd8cf8930cd47bb106a4ff4eb6569
+size 9915544352

model-00017-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aea3ee3b8bbc0a0f66042e09e0b41965c531b674288c6b75a295b85181a06d4f
+size 4339119736

special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception('Conversation roles must alternate user/assistant/user/assistant/...') }}{% endif %}{% if message['role'] == 'user' %}{{ '[INST] ' + message['content'] + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ message['content'] + eos_token}}{% else %}{{ raise_exception('Only user and assistant roles are supported!') }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": null,
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}