Update configuration_gpt_bert.py

configuration_gpt_bert.py

@@ -17,13 +17,13 @@ class ModelConfig(PretrainedConfig):
         if config_file is None:
             self.attention_probs_dropout_prob: float = 0.1
             self.hidden_dropout_prob = 0.1
-            self.hidden_size = 768
-            self.intermediate_size = 2560
+            self.hidden_size = 384
+            self.intermediate_size = 1280
             self.max_sequence_length = 512
             self.position_bucket_size = 32
-            self.num_attention_heads = 12
+            self.num_attention_heads = 6
             self.num_layers = 12
-            self.vocab_size = 8192
+            self.vocab_size = 4096
             self.layer_norm_eps = 1e-5
         else:
             if config_file == "str":