Hugging Face's logo

AdityaNarayan
/
KAT-Dev-72B-Exp-CPT-LoRA-Adapter-HyperSwitch

Text Generation
Safetensors
Rust
English
Hyperswitch
LoRA
CPT
Fine-Tuned
Causal-LM
conversational
Model card Files
xet
Community
KAT-Dev-72B-Exp-CPT-LoRA-Adapter-HyperSwitch
File size: 2,158 Bytes 
3dcceba
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
 
{
  "training_metadata": {
    "timestamp": "20251031_111636",
    "training_date": "2025-11-01",
    "training_time": "00:28:49",
    "final_epoch": 3.0,
    "total_steps": 711,
    "status": "completed"
  },
  "model_config": {
    "base_model": "Kwaipilot/KAT-Dev-72B-Exp",
    "model_type": "causal_lm",
    "architecture": "Qwen2ForCausalLM"
  },
  "lora_config": {
    "r": 64,
    "lora_alpha": 128,
    "lora_dropout": 0.05,
    "target_modules": [
      "q_proj",
      "k_proj",
      "v_proj",
      "o_proj",
      "gate_proj",
      "up_proj",
      "down_proj"
    ]
  },
  "training_config": {
    "num_epochs": 3,
    "per_device_train_batch_size": 1,
    "per_device_eval_batch_size": 1,
    "gradient_accumulation_steps": 16,
    "effective_batch_size": 64,
    "learning_rate": 5e-05,
    "lr_scheduler_type": "cosine",
    "warmup_ratio": 0.03,
    "weight_decay": 0.1,
    "max_grad_norm": 0.5,
    "bf16": true,
    "gradient_checkpointing": true,
    "optim": "adamw_torch",
    "logging_steps": 10,
    "save_steps": 50,
    "eval_steps": 25
  },
  "dataset_info": {
    "train_samples": 15129,
    "eval_samples": 1682,
    "max_seq_length": 8192,
    "sample_packing": false
  },
  "hardware_config": {
    "num_gpus": 4,
    "gpu_model": "NVIDIA H200",
    "distributed_strategy": "DeepSpeed ZeRO-2",
    "flash_attention": "2.8.3"
  },
  "performance_metrics": {
    "final_train_loss": 0.2641,
    "final_eval_loss": 0.37574875354766846,
    "final_train_perplexity": 1.3022584156313823,
    "final_eval_perplexity": 1.4560812525608204,
    "final_token_accuracy": 0.9259863365441561,
    "initial_loss": 1.6648,
    "initial_perplexity": 5.284616220817229,
    "initial_accuracy": 0.6015806214883923
  },
  "framework_versions": {
    "torch": "2.9.0+cu128",
    "transformers": "4.57.1",
    "peft": "0.17.1",
    "trl": "0.24.0",
    "deepspeed": "0.18.1",
    "flash_attn": "2.8.3",
    "python": "3.12.3"
  },
  "special_features": {
    "flash_attention_2": true,
    "gradient_checkpointing": true,
    "bf16_training": true,
    "sample_packing": false,
    "deepspeed_zero2": true,
    "distributed_training": true
  }
}