Hugging Face's logo

ConvexAI
/
Seraphim-8x10.7B-bf16

Text Generation
Transformers
Safetensors
English
mixtral
Merge
Mixture of Experts
conversational
text-generation-inference
Inference Endpoints
Model card Files Community
1
Kquant03 commited on
Commit
5da2056
·
verified ·
1 Parent(s): 0ee04da

Upload 19 files

Browse files
Files changed (19) hide show
  1. config.json +34 -0
  2. mergekit_moe_config.yml +67 -0
  3. model-00001-of-00015.safetensors +3 -0
  4. model-00002-of-00015.safetensors +3 -0
  5. model-00003-of-00015.safetensors +3 -0
  6. model-00004-of-00015.safetensors +3 -0
  7. model-00005-of-00015.safetensors +3 -0
  8. model-00006-of-00015.safetensors +3 -0
  9. model-00007-of-00015.safetensors +3 -0
  10. model-00008-of-00015.safetensors +3 -0
  11. model-00009-of-00015.safetensors +3 -0
  12. model-00010-of-00015.safetensors +3 -0
  13. model-00011-of-00015.safetensors +3 -0
  14. model-00012-of-00015.safetensors +3 -0
  15. model-00013-of-00015.safetensors +3 -0
  16. model-00014-of-00015.safetensors +3 -0
  17. model-00015-of-00015.safetensors +3 -0
  18. model.safetensors.index.json +0 -0
  19. special_tokens_map.json +24 -0
config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "bhavinjawade/SOLAR-10B-Nector-DPO-Jawade",
3
+ "architectures": [
4
+ "MixtralForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "eos_token_id": 2,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 4096,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 14336,
14
+ "max_position_embeddings": 4096,
15
+ "model_type": "mixtral",
16
+ "num_attention_heads": 32,
17
+ "num_experts_per_tok": 2,
18
+ "num_hidden_layers": 48,
19
+ "num_key_value_heads": 8,
20
+ "num_local_experts": 8,
21
+ "output_router_logits": false,
22
+ "pad_token_id": 2,
23
+ "pretraining_tp": 1,
24
+ "rms_norm_eps": 1e-05,
25
+ "rope_scaling": null,
26
+ "rope_theta": 10000.0,
27
+ "router_aux_loss_coef": 0.001,
28
+ "sliding_window": null,
29
+ "tie_word_embeddings": false,
30
+ "torch_dtype": "float32",
31
+ "transformers_version": "4.37.0.dev0",
32
+ "use_cache": true,
33
+ "vocab_size": 32000
34
+ }
mergekit_moe_config.yml ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ base_model: bhavinjawade/SOLAR-10B-Nector-DPO-Jawade
2
+ gate_mode: hidden
3
+ dtype: bfloat16
4
+ experts:
5
+ - source_model: bhavinjawade/SOLAR-10B-Nector-DPO-Jawade
6
+ positive_prompts:
7
+ - "helpful"
8
+ - "Relevant"
9
+ - "Factual"
10
+ - "Precise"
11
+ - source_model: NousResearch/Nous-Hermes-2-SOLAR-10.7B
12
+ positive_prompts:
13
+ - "Math"
14
+ - "Science"
15
+ negative_prompts:
16
+ - "inaccurate"
17
+ - "incorrect"
18
+ - source_model: Sao10K/Sensualize-Solar-10.7B
19
+ positive_prompts:
20
+ - "discuss"
21
+ - "chat"
22
+ - "culture"
23
+ - "world"
24
+ negative_prompts:
25
+ - "Sorry"
26
+ - "As an AI"
27
+ - "cannot"
28
+ - "not capable"
29
+ - source_model: NousResearch/Nous-Hermes-2-SOLAR-10.7B
30
+ positive_prompts:
31
+ - "calculate"
32
+ - "compute"
33
+ - "solve"
34
+ - "work"
35
+ negative_prompts:
36
+ - "mistake"
37
+ - "inaccurate"
38
+ - source_model: bhavinjawade/SOLAR-10B-Nector-DPO-Jawade
39
+ positive_prompts:
40
+ - "assist"
41
+ - "logic"
42
+ - "sense"
43
+ - source_model: Sao10K/Sensualize-Solar-10.7B
44
+ positive_prompts:
45
+ - "form"
46
+ - "connect"
47
+ - "try"
48
+ negative_prompts:
49
+ - "cannot"
50
+ - "incapable"
51
+ - source_model: Sao10K/SOLAR-10.7B-NahIdWin
52
+ positive_prompts:
53
+ - "percieve"
54
+ - "discern"
55
+ - "recognize"
56
+ negative_prompts:
57
+ - "don't"
58
+ - "cannot"
59
+ - source_model: bhavinjawade/SOLAR-10B-Nector-DPO-Jawade
60
+ positive_prompts:
61
+ - "core"
62
+ - "common"
63
+ - "basic"
64
+ - "intuitive"
65
+ negative_prompts:
66
+ - "boring"
67
+ - "lifeless"
model-00001-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8a4da1dae58157fd8c0f5ce04141fce609524956a99461d18acbb88faff0d63e
3
+ size 9919946528
model-00002-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:92311140fa6641a97e5ff7c2fc1d83c514172582eeae3111f48a67adb178a200
3
+ size 9982454760
model-00003-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6b597eb21a0bd9a65c0c00a8c521a332eea716e074c1be6f4a790a550ea29b0b
3
+ size 9982454760
model-00004-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:31015ac6c03a1b57aed16efe3fdf88ffd00d46a92331d2772b2e6829b68831a1
3
+ size 9982454760
model-00005-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7ae7304f41c46cc310a99c759bd130cd4d47446ab54e386eab9f658ec4bb6240
3
+ size 9982454720
model-00006-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ec8f12632c51055169b82bd9312270de2fdcd5e7c8e25a952e27f391085c0ab7
3
+ size 9982454712
model-00007-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d5bbfd929a98f8fe7e81a189d70311f3dc9987ac10e3cd9914cd2d336d510311
3
+ size 9982454760
model-00008-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f15460c4e20a25e9842b46a73025ee80d25743811d20cf33a659d9140780eddd
3
+ size 9982454760
model-00009-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3bdb44ce0c1a0cf1d86775d4e02b01e4dd6f208ace6a7ea7e93e21720e65106d
3
+ size 9982454760
model-00010-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a7db6fb2741fe1a87bef0db77505c75c9f3606969b1b299586d22f9bc4504965
3
+ size 9982454680
model-00011-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6878b64394ba493d75be0158f4e98b8207bc3d1bb994adb18952cbc979514176
3
+ size 9982454752
model-00012-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c961dfae6cc16870a519e21d9f3324c3b08cebb8cb355b00b21bccd7d3956f68
3
+ size 9982454760
model-00013-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5f9e5e2e7559aaa42435289b1b858e7bb8ffc3ed3a265648bd0c681a76668082
3
+ size 9982454760
model-00014-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d85158847d4e64b8fc4e04458cab3b89a9ce4751cf1d9f5f3aef2c39636366ca
3
+ size 9999648648
model-00015-of-00015.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:bba5f16822308da9428731c3eae310870c0625cc307ac8a95a0b5b2d68b37276
3
+ size 137369408
model.safetensors.index.json ADDED
The diff for this file is too large to render. See raw diff
 
special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "<s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }