Uploading patch

modeling_gpt_bert.py

@@ -310,6 +310,7 @@ class Embedding(nn.Module):
 class GPTBERTPreTrainedModel(PreTrainedModel):
     config_class = ModelConfig
     supports_gradient_checkpointing = False
+    base_model_prefix = "model"
 
     def _set_gradient_checkpointing(self, module, value=False):
         raise NotImplementedError("Gradient checkpointing is not supported by this model")
@@ -330,7 +331,7 @@ class GPTBERTPreTrainedModel(PreTrainedModel):
 
 class GPTBERT(GPTBERTPreTrainedModel):
 
-    def __init__(self, config: ModelConfig, is_causal: bool, **kwargs):
+    def __init__(self, config: ModelConfig, is_causal: bool = False, **kwargs):
         super().__init__(config, **kwargs)
         self.config = config
         self.hidden_size = config.hidden_size
@@ -380,7 +381,7 @@ class GPTBERT(GPTBERTPreTrainedModel):
             attention, layer_attention_probs = attention_layer(contextualized_embeddings[-1], attention_mask, relative_embeddings)
             layer_embeddings = contextualized_embeddings[-1] + attention
             layer_embeddings = self.dwa_modules(layer_embeddings, block_idx=i * 2)
-            layer_embeddings += mlp_layer(layer_embeddings)
+            layer_embeddings = layer_embeddings + mlp_layer(layer_embeddings)
             layer_embeddings = self.dwa_modules(layer_embeddings, block_idx=i * 2 + 1)
             contextualized_embeddings.append(layer_embeddings)
             attention_probs.append(layer_attention_probs)