Update config.json

config.json

@@ -3,7 +3,7 @@
   "architectures": [
     "NomicBertModel"
   ],
-  "attn_pdrop": 0.0,
+  "attn_pdrop": 0.1,
   "auto_map": {
     "AutoConfig": "configuration_hf_nomic_bert.NomicBertConfig",
     "AutoModel": "modeling_hf_nomic_bert.NomicBertModel",
@@ -32,7 +32,7 @@
   "prenorm": false,
   "qkv_proj_bias": false,
   "reorder_and_upcast_attn": false,
-  "resid_pdrop": 0.0,
+  "resid_pdrop": 0.1,
   "rotary_emb_base": 1000,
   "rotary_emb_fraction": 1.0,
   "rotary_emb_interleaved": false,
@@ -41,7 +41,7 @@
   "scale_attn_by_inverse_layer_idx": false,
   "scale_attn_weights": true,
   "summary_activation": null,
-  "summary_first_dropout": 0.0,
+  "summary_first_dropout": 0.1,
   "summary_proj_to_labels": true,
   "summary_type": "cls_index",
   "summary_use_proj": true,