Checkpoint at step 100 (FP16)

checkpoint-100/config.json

@@ -0,0 +1,38 @@
+{
+  "architectures": [
+    "Qwen3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "head_dim": 128,
+  "hidden_act": "silu",
+  "hidden_size": 384,
+  "initializer_range": 0.02,
+  "intermediate_size": 1536,
+  "layer_types": [
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention",
+    "full_attention"
+  ],
+  "max_position_embeddings": 32768,
+  "max_window_layers": 28,
+  "model_type": "qwen3",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 8,
+  "num_key_value_heads": 12,
+  "rms_norm_eps": 1e-06,
+  "rope_scaling": null,
+  "rope_theta": 1000000.0,
+  "sliding_window": null,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.53.2",
+  "use_cache": true,
+  "use_sliding_window": false,
+  "vocab_size": 50257
+}

checkpoint-100/generation_config.json

@@ -0,0 +1,4 @@
+{
+  "_from_model_config": true,
+  "transformers_version": "4.53.2"
+}

checkpoint-100/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a05d23ce874658c1e8eebb645e5e029df8ba5eee6a6989b0d6b0f653331cc0cb
+size 286554448

checkpoint-100/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39ddf059f82120d3ecc641318db9e5c48c859646db0f54b583adc3412836a941
+size 573165963

checkpoint-100/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f196323d7423b60f8e4ceb7dbf8715ee326c0d068e5ff164f13c63b279b9f1a0
+size 14645

checkpoint-100/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d15f80eff8e7aa3a49c0408f3d31fa2e6e9e75a2fbae063b0cc83bfa545cc3d7
+size 1383

checkpoint-100/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:535bdf7803edb387e75540cf936103a855c5145cd52a07a93a879538888a2420
+size 1465

checkpoint-100/special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": true,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-100/tokenizer_config.json

@@ -0,0 +1,23 @@
+{
+  "add_bos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "50256": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|endoftext|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 1024,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}

checkpoint-100/trainer_state.json

@@ -0,0 +1,151 @@
+{
+  "best_global_step": 100,
+  "best_metric": 7.308048725128174,
+  "best_model_checkpoint": "./qwen3-50m-c4-checkpoints_test_H200/checkpoint-100",
+  "epoch": 1.0526315789473684,
+  "eval_steps": 20,
+  "global_step": 100,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.010526315789473684,
+      "grad_norm": Infinity,
+      "learning_rate": 0.0,
+      "loss": 12.1602,
+      "step": 1
+    },
+    {
+      "epoch": 0.10526315789473684,
+      "grad_norm": 33.84176254272461,
+      "learning_rate": 1.8421052631578947e-05,
+      "loss": 11.2069,
+      "step": 10
+    },
+    {
+      "epoch": 0.21052631578947367,
+      "grad_norm": 6.943706035614014,
+      "learning_rate": 4.473684210526316e-05,
+      "loss": 9.0111,
+      "step": 20
+    },
+    {
+      "epoch": 0.21052631578947367,
+      "eval_loss": 8.518446922302246,
+      "eval_runtime": 2.4413,
+      "eval_samples_per_second": 650.051,
+      "eval_steps_per_second": 4.096,
+      "step": 20
+    },
+    {
+      "epoch": 0.3157894736842105,
+      "grad_norm": 3.6014673709869385,
+      "learning_rate": 4.7660818713450294e-05,
+      "loss": 8.3116,
+      "step": 30
+    },
+    {
+      "epoch": 0.42105263157894735,
+      "grad_norm": 1.7579938173294067,
+      "learning_rate": 4.473684210526316e-05,
+      "loss": 7.9996,
+      "step": 40
+    },
+    {
+      "epoch": 0.42105263157894735,
+      "eval_loss": 7.889881134033203,
+      "eval_runtime": 2.6339,
+      "eval_samples_per_second": 602.522,
+      "eval_steps_per_second": 3.797,
+      "step": 40
+    },
+    {
+      "epoch": 0.5263157894736842,
+      "grad_norm": 1.8037464618682861,
+      "learning_rate": 4.1812865497076025e-05,
+      "loss": 7.7626,
+      "step": 50
+    },
+    {
+      "epoch": 0.631578947368421,
+      "grad_norm": 2.642207145690918,
+      "learning_rate": 3.888888888888889e-05,
+      "loss": 7.5759,
+      "step": 60
+    },
+    {
+      "epoch": 0.631578947368421,
+      "eval_loss": 7.545220851898193,
+      "eval_runtime": 2.3517,
+      "eval_samples_per_second": 674.838,
+      "eval_steps_per_second": 4.252,
+      "step": 60
+    },
+    {
+      "epoch": 0.7368421052631579,
+      "grad_norm": 1.9747369289398193,
+      "learning_rate": 3.5964912280701756e-05,
+      "loss": 7.4782,
+      "step": 70
+    },
+    {
+      "epoch": 0.8421052631578947,
+      "grad_norm": 1.6699209213256836,
+      "learning_rate": 3.304093567251462e-05,
+      "loss": 7.3998,
+      "step": 80
+    },
+    {
+      "epoch": 0.8421052631578947,
+      "eval_loss": 7.401547431945801,
+      "eval_runtime": 2.3491,
+      "eval_samples_per_second": 675.572,
+      "eval_steps_per_second": 4.257,
+      "step": 80
+    },
+    {
+      "epoch": 0.9473684210526315,
+      "grad_norm": 1.9321849346160889,
+      "learning_rate": 3.0116959064327488e-05,
+      "loss": 7.344,
+      "step": 90
+    },
+    {
+      "epoch": 1.0526315789473684,
+      "grad_norm": 1.2896867990493774,
+      "learning_rate": 2.7192982456140354e-05,
+      "loss": 7.3003,
+      "step": 100
+    },
+    {
+      "epoch": 1.0526315789473684,
+      "eval_loss": 7.308048725128174,
+      "eval_runtime": 2.4689,
+      "eval_samples_per_second": 642.789,
+      "eval_steps_per_second": 4.05,
+      "step": 100
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 190,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 20,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2571041253359616.0,
+  "train_batch_size": 160,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-100/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:887cf7c787f240f5c8de4cd5b10cb35ae6bb8a820351b89fcd085e5bcee8717f
+size 5777