Hugging Face's logo

rAIfle
/
QwQonsortium-8x32B-RAW

Text Generation
Transformers
Safetensors
English
qwen2_moe
conversational
Inference Endpoints
Model card Files Community
rAIfle commited on
Commit
7d37d64
·
verified ·
1 Parent(s): 4b48bce

Add files using upload-large-folder tool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +1 -0
  2. added_tokens.json +28 -0
  3. config.json +38 -0
  4. mergekit_moe_config.yml +16 -0
  5. merges.txt +0 -0
  6. model-00001-of-00103.safetensors +3 -0
  7. model-00002-of-00103.safetensors +3 -0
  8. model-00003-of-00103.safetensors +3 -0
  9. model-00004-of-00103.safetensors +3 -0
  10. model-00005-of-00103.safetensors +3 -0
  11. model-00006-of-00103.safetensors +3 -0
  12. model-00007-of-00103.safetensors +3 -0
  13. model-00008-of-00103.safetensors +3 -0
  14. model-00009-of-00103.safetensors +3 -0
  15. model-00010-of-00103.safetensors +3 -0
  16. model-00011-of-00103.safetensors +3 -0
  17. model-00012-of-00103.safetensors +3 -0
  18. model-00013-of-00103.safetensors +3 -0
  19. model-00014-of-00103.safetensors +3 -0
  20. model-00015-of-00103.safetensors +3 -0
  21. model-00016-of-00103.safetensors +3 -0
  22. model-00017-of-00103.safetensors +3 -0
  23. model-00018-of-00103.safetensors +3 -0
  24. model-00019-of-00103.safetensors +3 -0
  25. model-00020-of-00103.safetensors +3 -0
  26. model-00021-of-00103.safetensors +3 -0
  27. model-00022-of-00103.safetensors +3 -0
  28. model-00023-of-00103.safetensors +3 -0
  29. model-00024-of-00103.safetensors +3 -0
  30. model-00025-of-00103.safetensors +3 -0
  31. model-00026-of-00103.safetensors +3 -0
  32. model-00027-of-00103.safetensors +3 -0
  33. model-00028-of-00103.safetensors +3 -0
  34. model-00029-of-00103.safetensors +3 -0
  35. model-00030-of-00103.safetensors +3 -0
  36. model-00031-of-00103.safetensors +3 -0
  37. model-00032-of-00103.safetensors +3 -0
  38. model-00033-of-00103.safetensors +3 -0
  39. model-00034-of-00103.safetensors +3 -0
  40. model-00035-of-00103.safetensors +3 -0
  41. model-00036-of-00103.safetensors +3 -0
  42. model-00037-of-00103.safetensors +3 -0
  43. model-00038-of-00103.safetensors +3 -0
  44. model-00039-of-00103.safetensors +3 -0
  45. model-00040-of-00103.safetensors +3 -0
  46. model-00041-of-00103.safetensors +3 -0
  47. model-00042-of-00103.safetensors +3 -0
  48. model-00043-of-00103.safetensors +3 -0
  49. model-00044-of-00103.safetensors +3 -0
  50. model-00045-of-00103.safetensors +3 -0
.gitattributes CHANGED
@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
added_tokens.json ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "</think>": 151668,
3
+ "</tool_call>": 151658,
4
+ "</tool_response>": 151666,
5
+ "<think>": 151667,
6
+ "<tool_call>": 151657,
7
+ "<tool_response>": 151665,
8
+ "<|box_end|>": 151649,
9
+ "<|box_start|>": 151648,
10
+ "<|endoftext|>": 151643,
11
+ "<|file_sep|>": 151664,
12
+ "<|fim_middle|>": 151660,
13
+ "<|fim_pad|>": 151662,
14
+ "<|fim_prefix|>": 151659,
15
+ "<|fim_suffix|>": 151661,
16
+ "<|im_end|>": 151645,
17
+ "<|im_start|>": 151644,
18
+ "<|image_pad|>": 151655,
19
+ "<|object_ref_end|>": 151647,
20
+ "<|object_ref_start|>": 151646,
21
+ "<|quad_end|>": 151651,
22
+ "<|quad_start|>": 151650,
23
+ "<|repo_name|>": 151663,
24
+ "<|video_pad|>": 151656,
25
+ "<|vision_end|>": 151653,
26
+ "<|vision_pad|>": 151654,
27
+ "<|vision_start|>": 151652
28
+ }
config.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "Qwen/QwQ-32B",
3
+ "architectures": [
4
+ "Qwen2MoeForCausalLM"
5
+ ],
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 151643,
8
+ "decoder_sparse_step": 1,
9
+ "eos_token_id": 151645,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 5120,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 27648,
14
+ "max_position_embeddings": 131072,
15
+ "max_window_layers": 64,
16
+ "mlp_only_layers": [],
17
+ "model_type": "qwen2_moe",
18
+ "moe_intermediate_size": 27648,
19
+ "norm_topk_prob": true,
20
+ "num_attention_heads": 40,
21
+ "num_experts": 8,
22
+ "num_experts_per_tok": 2,
23
+ "num_hidden_layers": 64,
24
+ "num_key_value_heads": 8,
25
+ "output_router_logits": false,
26
+ "rms_norm_eps": 1e-05,
27
+ "rope_scaling": null,
28
+ "rope_theta": 1000000.0,
29
+ "router_aux_loss_coef": 0.001,
30
+ "shared_expert_intermediate_size": 27648,
31
+ "sliding_window": null,
32
+ "tie_word_embeddings": false,
33
+ "torch_dtype": "bfloat16",
34
+ "transformers_version": "4.49.0",
35
+ "use_cache": true,
36
+ "use_sliding_window": false,
37
+ "vocab_size": 152064
38
+ }
mergekit_moe_config.yml ADDED
@@ -0,0 +1,16 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ base_model: Qwen/QwQ-32B
2
+ gate_mode: random
3
+ architecture: qwen
4
+ dtype: bfloat16
5
+ experts:
6
+ - source_model: Qwen/QwQ-32B
7
+ - source_model: Qwen/QwQ-32B
8
+ - source_model: Qwen/QwQ-32B
9
+ - source_model: Qwen/QwQ-32B
10
+ - source_model: Qwen/QwQ-32B
11
+ - source_model: Qwen/QwQ-32B
12
+ - source_model: Qwen/QwQ-32B
13
+ - source_model: Qwen/QwQ-32B
14
+ shared_experts:
15
+ - source_model: Qwen/QwQ-32B
16
+ residual_scale: 0.1 # downweight output from shared expert to prevent overcooking the model
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
model-00001-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e5f0a154afbb55b9331bbc52514a7b407f3c35d82e30b7b407c4e0bd9ab1f9fe
3
+ size 4954533592
model-00002-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bc64421bba824cd9dbb957eca45fee41fbdcefe102528d0e78dd53385e322ef5
3
+ size 4938830952
model-00003-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b9180b60c366361c34cac2d902f87b5618b22ca7a630b6f276b0403c2c404535
3
+ size 4812966000
model-00004-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0ee7c7368676b2ea48c6a7825520d6d61c65a98a28985766135edb64a3ae9c11
3
+ size 4938830952
model-00005-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c40de39f05880c569d52a43c5820c72562fe0a6919ee341f9eb46645199f75a0
3
+ size 4812965984
model-00006-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5b8a8e05350fb2fc2444da5fb2fb4ca5a21850e5f156ce171531e682208bcce7
3
+ size 4938830944
model-00007-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f05b270a56fd01d6299febd739e58dbfbb9df8b7fb102f2a16d2f38b08da2288
3
+ size 4938830952
model-00008-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:921ea409832ed43f86a39beb845082e02868c8800be46319529fd3740ca30aa6
3
+ size 4812965992
model-00009-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:20240ba25026a589f400977f1cee4eb034baf49575d13c591b650f7cfa411f27
3
+ size 4938830960
model-00010-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aac6a5576ce234871f2cd5381f22910ce44ae02c8fdf7060b82ac97a274f9e73
3
+ size 4938830952
model-00011-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a29269ad51e4ec550260789acab1f5195b7c66147b3e1c454bb2d30a20f6b9b9
3
+ size 4812966000
model-00012-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9f190a4f5ced7b6ef2a75f8dd28814c5a6010c6adf5a9d35b0b76b63c19434e1
3
+ size 4938830952
model-00013-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3834fb912aa90433e166c0a7a4d785eca2b7e803b6245a7e9d073d8be9b97b3b
3
+ size 4938830952
model-00014-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:73fb229beb780adc236a6a381ac74651f9b4ef391987a726f3491a252928ce78
3
+ size 4812966000
model-00015-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1553d6751e5093287e1f854d386c26c9c4c634a04fddfb6c4cbfd5f928d2e369
3
+ size 4938830952
model-00016-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0132b281d4d0b83d2be3e8f5dd6615533ea37e206d3714c5492dbd8202f09f63
3
+ size 4812965992
model-00017-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5223a77fd27272f848c42b456ddf84ddc78f17ca29e53fc248b1369d536cb69e
3
+ size 4938830968
model-00018-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd4aefccef5ef31ed846ee44a1c1bccabf218f0140bc74895d3a8255e014ce45
3
+ size 4938830976
model-00019-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8639494f84a72856cae60532df11f6c10f28ef22ead8d89f417082a39916da44
3
+ size 4812966016
model-00020-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:520cc18fca3797508f3070ec7c9fa32b3481c50deb660152ad2ce56ac5f6ff8c
3
+ size 4938830984
model-00021-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4dbd52980a6590477c904063c2204f84cb37a2f82c90c00153f1992187fbaa0e
3
+ size 4938830976
model-00022-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d3d93ed001a5b84e21322f3c38dcf972c64fba766756a55b9dfbd95f1b8a6676
3
+ size 4812966016
model-00023-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:77e1eacc704e74926613a2ca5288fae4b4d57528475729c4e7003e0cd6bc8b99
3
+ size 4938830968
model-00024-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d34d55d8cd08fe912ce2b0230d22acb27019d3d85d001bed37af66a273209438
3
+ size 4812966008
model-00025-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:422b48e39f804a6d0acbb18ba4cfe792a3ccea67a2a054b360e36d83724f94fc
3
+ size 4938830968
model-00026-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9894a338d98f330e5183347cd36b6bcd8ea4ef899fd4174b2ebe9830ae801554
3
+ size 4938830976
model-00027-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f813d75c60a609895afab822e3e6980d1bc27f089a623c9ba5a62108a5c5c2a7
3
+ size 4812966008
model-00028-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f8e9083273bf014aa808d2012264c74da155daaf9d16a21cde1987a89f99c447
3
+ size 4938830984
model-00029-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:37e01351b52152bf7c474ae89172a464d592e7a5b573a657f4534e35bfe5b521
3
+ size 4938830976
model-00030-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:278d7c1e382bc65c355514a71dcd0a9c81a94dd4b16effd471b796191c686db3
3
+ size 4812966016
model-00031-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a463e638dc5c1ef8fcc7863fca6b29fefc876c3471bfa259047dfee6209878a2
3
+ size 4938830976
model-00032-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bbc2e1b468f8588132141b331f75c171745fedd12d57a33e504d4f76e23ea50e
3
+ size 4812966000
model-00033-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f87b14acf8b7b39a43e65514f08e28cea4bbd0bc3bc23f5cf9d25b109a0210d
3
+ size 4938830976
model-00034-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:79c7cf77aaa70d7832a5063c712ae5216a868d3cd2cd1f584eca5025d7ae67f0
3
+ size 4938830976
model-00035-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:86be05420dbbd3c1b481016cd8694fc38f9222cef101ba0930f4228f5b063080
3
+ size 4812966008
model-00036-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7c3d9219e4f2f2973488d1809059c56cdae4dd2fa2f3f8b6b683d43660be4d80
3
+ size 4938830984
model-00037-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:68c499015950f37d174356124ffcd1db87a20712241fa4c81883e51fb2f8c30b
3
+ size 4938830976
model-00038-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6d3f4f8e92ea7fa675b0349e96444c0abee993a89c035f3b4067dd6987cb62ed
3
+ size 4812966016
model-00039-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c20378027cdffcb85a8df09103bb708fac051909132f4abcf3ca3233f860c3d8
3
+ size 4938830976
model-00040-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:46106e04a887fdfeedcf94e073def7acf2132ed2da1727df3c373da85fc99b10
3
+ size 4938830976
model-00041-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0e5daa198e7b7f76f7d70eb65a6d9db5d3a7f6238197841c2acf94e72247a469
3
+ size 4812966016
model-00042-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6edf216fd74db8ca8da2e2e206ba7bd49780aa1ad31e39c79c13162ceafac554
3
+ size 4938830976
model-00043-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4840d9a6d5018193517e1208675bc4b5d571a39067a1324e26a92d7a134a6395
3
+ size 4812966008
model-00044-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ddf59c35f45bd48e669b4f7ff7e1ec4761b9df5a352b6f4acfad84b2143bdaac
3
+ size 4938830968
model-00045-of-00103.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ac799b1ae464992c6e3cb320b0f053bb5d06ba4b9032c4537bb81210dc7f0b3e
3
+ size 4938830976