change weight format

config.json

@@ -1,4 +1,5 @@
 {
+  "model_type": "jetmoe",
   "activation_function": "silu",
   "architectures": [
     "JetMoEForCausalLM"

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec472ef5ccae7be6181856a8dac3f4c2743fff491606d82266b6f82e94fa274e
-size 4879574808
+oid sha256:a755dde976d3e9bffc2f04cd3849b9ab190f0f9f8ce6f96f46c2ee048d60a6aa
+size 4879573992

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4ee73ffb225275e9cced7473181f9dcb12a79c27d5d6fa31c2ae09964233e057
-size 4933085104
+oid sha256:1c2129367d853e58d035d164081775ffe71b26e283328db30be62536029e9894
+size 4933084288

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:562ea54a02baa7ed6d250bdb7835a9867dbb9ceaacd0a65d127eee5257a94b16
-size 4933085160
+oid sha256:1c5f1f807fe65c7c9272ed2ef7fac260b43f3296fd977996a444248a7abe51ca
+size 4933084344

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fb1997ac5dfbe2fddbfc45644db3b2963427adda13202bf4cf8c38a6daee8029
-size 2298765920
+oid sha256:99ffc25291c1300b916147cfbe3a603d6e5409788fbc0037cfdd43992a69be2b
+size 2298765576

model.safetensors.index.json

@@ -6,7 +6,6 @@
     "model.embed_tokens.weight": "model-00001-of-00004.safetensors",
     "model.layers.0.input_layernorm.weight": "model-00001-of-00004.safetensors",
     "model.layers.0.mlp.bias": "model-00001-of-00004.safetensors",
-    "model.layers.0.mlp.glu_linear.weight": "model-00001-of-00004.safetensors",
     "model.layers.0.mlp.input_linear.weight": "model-00001-of-00004.safetensors",
     "model.layers.0.mlp.output_linear.weight": "model-00001-of-00004.safetensors",
     "model.layers.0.mlp.router.layer.weight": "model-00001-of-00004.safetensors",
@@ -18,7 +17,6 @@
     "model.layers.0.self_attention.kv_proj.weight": "model-00001-of-00004.safetensors",
     "model.layers.1.input_layernorm.weight": "model-00001-of-00004.safetensors",
     "model.layers.1.mlp.bias": "model-00001-of-00004.safetensors",
-    "model.layers.1.mlp.glu_linear.weight": "model-00001-of-00004.safetensors",
     "model.layers.1.mlp.input_linear.weight": "model-00001-of-00004.safetensors",
     "model.layers.1.mlp.output_linear.weight": "model-00001-of-00004.safetensors",
     "model.layers.1.mlp.router.layer.weight": "model-00001-of-00004.safetensors",
@@ -30,7 +28,6 @@
     "model.layers.1.self_attention.kv_proj.weight": "model-00001-of-00004.safetensors",
     "model.layers.10.input_layernorm.weight": "model-00002-of-00004.safetensors",
     "model.layers.10.mlp.bias": "model-00002-of-00004.safetensors",
-    "model.layers.10.mlp.glu_linear.weight": "model-00002-of-00004.safetensors",
     "model.layers.10.mlp.input_linear.weight": "model-00002-of-00004.safetensors",
     "model.layers.10.mlp.output_linear.weight": "model-00002-of-00004.safetensors",
     "model.layers.10.mlp.router.layer.weight": "model-00002-of-00004.safetensors",
@@ -42,7 +39,6 @@
     "model.layers.10.self_attention.kv_proj.weight": "model-00002-of-00004.safetensors",
     "model.layers.11.input_layernorm.weight": "model-00002-of-00004.safetensors",
     "model.layers.11.mlp.bias": "model-00002-of-00004.safetensors",
-    "model.layers.11.mlp.glu_linear.weight": "model-00002-of-00004.safetensors",
     "model.layers.11.mlp.input_linear.weight": "model-00002-of-00004.safetensors",
     "model.layers.11.mlp.output_linear.weight": "model-00002-of-00004.safetensors",
     "model.layers.11.mlp.router.layer.weight": "model-00002-of-00004.safetensors",
@@ -54,7 +50,6 @@
     "model.layers.11.self_attention.kv_proj.weight": "model-00002-of-00004.safetensors",
     "model.layers.12.input_layernorm.weight": "model-00002-of-00004.safetensors",
     "model.layers.12.mlp.bias": "model-00002-of-00004.safetensors",
-    "model.layers.12.mlp.glu_linear.weight": "model-00002-of-00004.safetensors",
     "model.layers.12.mlp.input_linear.weight": "model-00002-of-00004.safetensors",
     "model.layers.12.mlp.output_linear.weight": "model-00002-of-00004.safetensors",
     "model.layers.12.mlp.router.layer.weight": "model-00002-of-00004.safetensors",
@@ -66,7 +61,6 @@
     "model.layers.12.self_attention.kv_proj.weight": "model-00002-of-00004.safetensors",
     "model.layers.13.input_layernorm.weight": "model-00002-of-00004.safetensors",
     "model.layers.13.mlp.bias": "model-00002-of-00004.safetensors",
-    "model.layers.13.mlp.glu_linear.weight": "model-00002-of-00004.safetensors",
     "model.layers.13.mlp.input_linear.weight": "model-00002-of-00004.safetensors",
     "model.layers.13.mlp.output_linear.weight": "model-00003-of-00004.safetensors",
     "model.layers.13.mlp.router.layer.weight": "model-00003-of-00004.safetensors",
@@ -78,7 +72,6 @@
     "model.layers.13.self_attention.kv_proj.weight": "model-00002-of-00004.safetensors",
     "model.layers.14.input_layernorm.weight": "model-00003-of-00004.safetensors",
     "model.layers.14.mlp.bias": "model-00003-of-00004.safetensors",
-    "model.layers.14.mlp.glu_linear.weight": "model-00003-of-00004.safetensors",
     "model.layers.14.mlp.input_linear.weight": "model-00003-of-00004.safetensors",
     "model.layers.14.mlp.output_linear.weight": "model-00003-of-00004.safetensors",
     "model.layers.14.mlp.router.layer.weight": "model-00003-of-00004.safetensors",
@@ -90,7 +83,6 @@
     "model.layers.14.self_attention.kv_proj.weight": "model-00003-of-00004.safetensors",
     "model.layers.15.input_layernorm.weight": "model-00003-of-00004.safetensors",
     "model.layers.15.mlp.bias": "model-00003-of-00004.safetensors",
-    "model.layers.15.mlp.glu_linear.weight": "model-00003-of-00004.safetensors",
     "model.layers.15.mlp.input_linear.weight": "model-00003-of-00004.safetensors",
     "model.layers.15.mlp.output_linear.weight": "model-00003-of-00004.safetensors",
     "model.layers.15.mlp.router.layer.weight": "model-00003-of-00004.safetensors",
@@ -102,7 +94,6 @@
     "model.layers.15.self_attention.kv_proj.weight": "model-00003-of-00004.safetensors",
     "model.layers.16.input_layernorm.weight": "model-00003-of-00004.safetensors",
     "model.layers.16.mlp.bias": "model-00003-of-00004.safetensors",
-    "model.layers.16.mlp.glu_linear.weight": "model-00003-of-00004.safetensors",
     "model.layers.16.mlp.input_linear.weight": "model-00003-of-00004.safetensors",
     "model.layers.16.mlp.output_linear.weight": "model-00003-of-00004.safetensors",
     "model.layers.16.mlp.router.layer.weight": "model-00003-of-00004.safetensors",
@@ -114,7 +105,6 @@
     "model.layers.16.self_attention.kv_proj.weight": "model-00003-of-00004.safetensors",
     "model.layers.17.input_layernorm.weight": "model-00003-of-00004.safetensors",
     "model.layers.17.mlp.bias": "model-00003-of-00004.safetensors",
-    "model.layers.17.mlp.glu_linear.weight": "model-00003-of-00004.safetensors",
     "model.layers.17.mlp.input_linear.weight": "model-00003-of-00004.safetensors",
     "model.layers.17.mlp.output_linear.weight": "model-00003-of-00004.safetensors",
     "model.layers.17.mlp.router.layer.weight": "model-00003-of-00004.safetensors",
@@ -126,7 +116,6 @@
     "model.layers.17.self_attention.kv_proj.weight": "model-00003-of-00004.safetensors",
     "model.layers.18.input_layernorm.weight": "model-00003-of-00004.safetensors",
     "model.layers.18.mlp.bias": "model-00003-of-00004.safetensors",
-    "model.layers.18.mlp.glu_linear.weight": "model-00003-of-00004.safetensors",
     "model.layers.18.mlp.input_linear.weight": "model-00003-of-00004.safetensors",
     "model.layers.18.mlp.output_linear.weight": "model-00003-of-00004.safetensors",
     "model.layers.18.mlp.router.layer.weight": "model-00003-of-00004.safetensors",
@@ -138,7 +127,6 @@
     "model.layers.18.self_attention.kv_proj.weight": "model-00003-of-00004.safetensors",
     "model.layers.19.input_layernorm.weight": "model-00003-of-00004.safetensors",
     "model.layers.19.mlp.bias": "model-00003-of-00004.safetensors",
-    "model.layers.19.mlp.glu_linear.weight": "model-00003-of-00004.safetensors",
     "model.layers.19.mlp.input_linear.weight": "model-00003-of-00004.safetensors",
     "model.layers.19.mlp.output_linear.weight": "model-00003-of-00004.safetensors",
     "model.layers.19.mlp.router.layer.weight": "model-00003-of-00004.safetensors",
@@ -150,7 +138,6 @@
     "model.layers.19.self_attention.kv_proj.weight": "model-00003-of-00004.safetensors",
     "model.layers.2.input_layernorm.weight": "model-00001-of-00004.safetensors",
     "model.layers.2.mlp.bias": "model-00001-of-00004.safetensors",
-    "model.layers.2.mlp.glu_linear.weight": "model-00001-of-00004.safetensors",
     "model.layers.2.mlp.input_linear.weight": "model-00001-of-00004.safetensors",
     "model.layers.2.mlp.output_linear.weight": "model-00001-of-00004.safetensors",
     "model.layers.2.mlp.router.layer.weight": "model-00001-of-00004.safetensors",
@@ -162,7 +149,6 @@
     "model.layers.2.self_attention.kv_proj.weight": "model-00001-of-00004.safetensors",
     "model.layers.20.input_layernorm.weight": "model-00003-of-00004.safetensors",
     "model.layers.20.mlp.bias": "model-00003-of-00004.safetensors",
-    "model.layers.20.mlp.glu_linear.weight": "model-00003-of-00004.safetensors",
     "model.layers.20.mlp.input_linear.weight": "model-00003-of-00004.safetensors",
     "model.layers.20.mlp.output_linear.weight": "model-00004-of-00004.safetensors",
     "model.layers.20.mlp.router.layer.weight": "model-00004-of-00004.safetensors",
@@ -174,7 +160,6 @@
     "model.layers.20.self_attention.kv_proj.weight": "model-00003-of-00004.safetensors",
     "model.layers.21.input_layernorm.weight": "model-00004-of-00004.safetensors",
     "model.layers.21.mlp.bias": "model-00004-of-00004.safetensors",
-    "model.layers.21.mlp.glu_linear.weight": "model-00004-of-00004.safetensors",
     "model.layers.21.mlp.input_linear.weight": "model-00004-of-00004.safetensors",
     "model.layers.21.mlp.output_linear.weight": "model-00004-of-00004.safetensors",
     "model.layers.21.mlp.router.layer.weight": "model-00004-of-00004.safetensors",
@@ -186,7 +171,6 @@
     "model.layers.21.self_attention.kv_proj.weight": "model-00004-of-00004.safetensors",
     "model.layers.22.input_layernorm.weight": "model-00004-of-00004.safetensors",
     "model.layers.22.mlp.bias": "model-00004-of-00004.safetensors",
-    "model.layers.22.mlp.glu_linear.weight": "model-00004-of-00004.safetensors",
     "model.layers.22.mlp.input_linear.weight": "model-00004-of-00004.safetensors",
     "model.layers.22.mlp.output_linear.weight": "model-00004-of-00004.safetensors",
     "model.layers.22.mlp.router.layer.weight": "model-00004-of-00004.safetensors",
@@ -198,7 +182,6 @@
     "model.layers.22.self_attention.kv_proj.weight": "model-00004-of-00004.safetensors",
     "model.layers.23.input_layernorm.weight": "model-00004-of-00004.safetensors",
     "model.layers.23.mlp.bias": "model-00004-of-00004.safetensors",
-    "model.layers.23.mlp.glu_linear.weight": "model-00004-of-00004.safetensors",
     "model.layers.23.mlp.input_linear.weight": "model-00004-of-00004.safetensors",
     "model.layers.23.mlp.output_linear.weight": "model-00004-of-00004.safetensors",
     "model.layers.23.mlp.router.layer.weight": "model-00004-of-00004.safetensors",
@@ -210,7 +193,6 @@
     "model.layers.23.self_attention.kv_proj.weight": "model-00004-of-00004.safetensors",
     "model.layers.3.input_layernorm.weight": "model-00001-of-00004.safetensors",
     "model.layers.3.mlp.bias": "model-00001-of-00004.safetensors",
-    "model.layers.3.mlp.glu_linear.weight": "model-00001-of-00004.safetensors",
     "model.layers.3.mlp.input_linear.weight": "model-00001-of-00004.safetensors",
     "model.layers.3.mlp.output_linear.weight": "model-00001-of-00004.safetensors",
     "model.layers.3.mlp.router.layer.weight": "model-00001-of-00004.safetensors",
@@ -222,7 +204,6 @@
     "model.layers.3.self_attention.kv_proj.weight": "model-00001-of-00004.safetensors",
     "model.layers.4.input_layernorm.weight": "model-00001-of-00004.safetensors",
     "model.layers.4.mlp.bias": "model-00001-of-00004.safetensors",
-    "model.layers.4.mlp.glu_linear.weight": "model-00001-of-00004.safetensors",
     "model.layers.4.mlp.input_linear.weight": "model-00001-of-00004.safetensors",
     "model.layers.4.mlp.output_linear.weight": "model-00001-of-00004.safetensors",
     "model.layers.4.mlp.router.layer.weight": "model-00001-of-00004.safetensors",
@@ -234,7 +215,6 @@
     "model.layers.4.self_attention.kv_proj.weight": "model-00001-of-00004.safetensors",
     "model.layers.5.input_layernorm.weight": "model-00001-of-00004.safetensors",
     "model.layers.5.mlp.bias": "model-00001-of-00004.safetensors",
-    "model.layers.5.mlp.glu_linear.weight": "model-00001-of-00004.safetensors",
     "model.layers.5.mlp.input_linear.weight": "model-00001-of-00004.safetensors",
     "model.layers.5.mlp.output_linear.weight": "model-00001-of-00004.safetensors",
     "model.layers.5.mlp.router.layer.weight": "model-00001-of-00004.safetensors",
@@ -246,7 +226,6 @@
     "model.layers.5.self_attention.kv_proj.weight": "model-00001-of-00004.safetensors",
     "model.layers.6.input_layernorm.weight": "model-00001-of-00004.safetensors",
     "model.layers.6.mlp.bias": "model-00001-of-00004.safetensors",
-    "model.layers.6.mlp.glu_linear.weight": "model-00001-of-00004.safetensors",
     "model.layers.6.mlp.input_linear.weight": "model-00001-of-00004.safetensors",
     "model.layers.6.mlp.output_linear.weight": "model-00002-of-00004.safetensors",
     "model.layers.6.mlp.router.layer.weight": "model-00002-of-00004.safetensors",
@@ -258,7 +237,6 @@
     "model.layers.6.self_attention.kv_proj.weight": "model-00001-of-00004.safetensors",
     "model.layers.7.input_layernorm.weight": "model-00002-of-00004.safetensors",
     "model.layers.7.mlp.bias": "model-00002-of-00004.safetensors",
-    "model.layers.7.mlp.glu_linear.weight": "model-00002-of-00004.safetensors",
     "model.layers.7.mlp.input_linear.weight": "model-00002-of-00004.safetensors",
     "model.layers.7.mlp.output_linear.weight": "model-00002-of-00004.safetensors",
     "model.layers.7.mlp.router.layer.weight": "model-00002-of-00004.safetensors",
@@ -270,7 +248,6 @@
     "model.layers.7.self_attention.kv_proj.weight": "model-00002-of-00004.safetensors",
     "model.layers.8.input_layernorm.weight": "model-00002-of-00004.safetensors",
     "model.layers.8.mlp.bias": "model-00002-of-00004.safetensors",
-    "model.layers.8.mlp.glu_linear.weight": "model-00002-of-00004.safetensors",
     "model.layers.8.mlp.input_linear.weight": "model-00002-of-00004.safetensors",
     "model.layers.8.mlp.output_linear.weight": "model-00002-of-00004.safetensors",
     "model.layers.8.mlp.router.layer.weight": "model-00002-of-00004.safetensors",
@@ -282,7 +259,6 @@
     "model.layers.8.self_attention.kv_proj.weight": "model-00002-of-00004.safetensors",
     "model.layers.9.input_layernorm.weight": "model-00002-of-00004.safetensors",
     "model.layers.9.mlp.bias": "model-00002-of-00004.safetensors",
-    "model.layers.9.mlp.glu_linear.weight": "model-00002-of-00004.safetensors",
     "model.layers.9.mlp.input_linear.weight": "model-00002-of-00004.safetensors",
     "model.layers.9.mlp.output_linear.weight": "model-00002-of-00004.safetensors",
     "model.layers.9.mlp.router.layer.weight": "model-00002-of-00004.safetensors",