Uploading patch

configuration_gpt_bert.py

@@ -23,8 +23,8 @@ class ModelConfig(PretrainedConfig):
             self.position_bucket_size = 32
             self.num_attention_heads = 12
             self.num_layers = 12
-            self.vocab_size = 8192
-            self.layer_norm_eps = 1e-5
+            self.vocab_size = 16384
+            self.layer_norm_eps = 1e-7
         else:
             if config_file == "str":
                 config_file = pathlib.Path(config_file)