{
    "distributed": {
        "model_dtype": "bf16"
    },
    "model": {
        "dim": 3072,
        "n_layers": 28,
        "n_heads": 24,
        "n_kv_heads": 8,
        "vocab_size": 128256,
        "ffn_dim_multiplier": 1.0,
        "multiple_of": 256,
        "norm_eps": 1e-05,
        "rope_theta": 500000.0,
        "use_scaled_rope": true,
        "rope_scale_factor": 32,
        "high_freq_factor": 4,
        "max_seqlen": 11520,
        "pooling_ratio": 2,
        "weight_tying": true,
        "vision_model": {
            "image_size": 448,
            "patch_size": 14,
            "width": 1024,
            "layers": 23,
            "heads": 16,
            "use_cls_token": true,
            "use_abs_posemb": true,
            "mlp_ratio": 4.0,
            "ls_init_value": 0.1,
            "drop_path": 0.1,
            "use_ln_post": false,
            "pool_type": "none"
        }
    },
    "data": {
        "tokenizer_name": "plmchat",
        "tokenizer_path": "tokenizer.model",
        "vision_input_type": "thumb+tile",
        "max_num_tiles": 36,
        "max_video_frames": 32
    }
}