Training in progress, step 1000

config.json

@@ -8,12 +8,12 @@
   "dtype": "float32",
   "embedding_size": 128,
   "expert_intermediate_size": 2624,
-  "group_depth": 4,
+  "group_depth": 3,
   "hidden_act": "gelu",
   "hidden_dropout_prob": 0.1,
-  "hidden_size": 1024,
+  "hidden_size": 768,
   "initializer_range": 0.02,
-  "intermediate_size": 2624,
+  "intermediate_size": 1152,
   "layer_norm_eps": 1e-06,
   "load_balancing_loss_coef": 0.2,
   "lora_alpha": 32,
@@ -23,8 +23,7 @@
   "num_attention_heads": 16,
   "num_expert_modules": 3,
   "num_experts": 8,
-  "num_hidden_layers": 16,
-  "output_hidden_states": true,
+  "num_hidden_layers": 12,
   "pad_token_id": 0,
   "router_jitter_noise": 0.01,
   "top_k": 2,

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:923f7d1b566247ede9b22278980822eae56cadb9a9b3dd9262ae20fbe9e710b1
-size 1059459406
+oid sha256:1ce76e7ff5e00c2b43727da522f29f397bdc16bd061c317acc62bd8f0fa104fe
+size 383474230

runs/Oct09_16-06-08_nid006852/events.out.tfevents.1760018783.nid006852.205420.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a511eda6dd8b00603ffd39f43cc6ba138048f3e6811d23150a075c1e70729a3
+size 15859

runs/Sep27_10-53-04_nid007099/events.out.tfevents.1758963191.nid007099.167413.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:520a51bd0ea389e20c9063d3dfe5fedd0d8fa5e3dcf3dcc4845beff6466eea17
-size 220332
+oid sha256:75c790adca0ae4dafa1d43ce1496b4891ddfb85b2252124c5a6cefd4df129b4b
+size 231082

runs/Sep27_21-55-44_nid006631/events.out.tfevents.1759002954.nid006631.150629.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5e6ca037b3ccd7d914ace4a9720206d1c078cde552bb787fec64a618ba32061f
+size 15867

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:19c9bbb6b985d4f48fe80fcb93df10124c24f15065a7691fa4973605b097bf6b
+oid sha256:119e924a11235114f963ea267ac6873181a4ff9d54cf5938dedce89a3b292868
 size 5432