





'''simple docstring'''






import argparse
import os
import re
import zipfile

import torch

from transformers import MegatronBertConfig
def      UpperCamelCase__   (	lowerCAmelCase							,			lowerCAmelCase							,			lowerCAmelCase=0						):





						"""simple docstring"""


						if name is None:
												_lowerCAmelCase							  =							None
						else:
												_lowerCAmelCase							  =							""".""" * max(0							,			spaces - 2						) + """# {:""" + str(50 - spaces						) + """s}"""
												_lowerCAmelCase							  =							fmt.format(lowerCAmelCase						)

						# Print and recurse (if needed).
						if isinstance(lowerCAmelCase							,			lowerCAmelCase						):
												if msg is not None:
																		print(lowerCAmelCase						)
												for k in val.keys():
																		recursive_print(lowerCAmelCase							,			val[k]							,			spaces + 2						)
						elif isinstance(lowerCAmelCase							,			torch.Tensor						):
												print(lowerCAmelCase							,			""":"""							,			val.size()						)
						else:
												print(lowerCAmelCase							,			""":"""							,			lowerCAmelCase						)
def      UpperCamelCase__   (	lowerCAmelCase							,			lowerCAmelCase							,			lowerCAmelCase							,			lowerCAmelCase							,			lowerCAmelCase						):





						"""simple docstring"""


						_lowerCAmelCase							  =							param.size()
						if checkpoint_version == 1.0:
												# version 1.0 stores [num_heads * hidden_size * num_splits, :]
												_lowerCAmelCase							  =							(num_heads, hidden_size, num_splits) + input_shape[1:]
												_lowerCAmelCase							  =							param.view(*lowerCAmelCase						)
												_lowerCAmelCase							  =							param.transpose(0							,			2						)
												_lowerCAmelCase							  =							param.transpose(1							,			2						).contiguous()
						elif checkpoint_version >= 2.0:
												# other versions store [num_heads * num_splits * hidden_size, :]
												_lowerCAmelCase							  =							(num_heads, num_splits, hidden_size) + input_shape[1:]
												_lowerCAmelCase							  =							param.view(*lowerCAmelCase						)
												_lowerCAmelCase							  =							param.transpose(0							,			1						).contiguous()
						_lowerCAmelCase							  =							param.view(*lowerCAmelCase						)
						return param
def      UpperCamelCase__   (	lowerCAmelCase							,			lowerCAmelCase							,			lowerCAmelCase						):





						"""simple docstring"""


						_lowerCAmelCase							  =							{}

						# old versions did not store training args
						_lowerCAmelCase							  =							input_state_dict.get("""args"""							,			lowerCAmelCase						)
						if ds_args is not None:
												# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
												# from pprint import pprint
												# pprint(vars(ds_args))

												_lowerCAmelCase							  =							ds_args.tokenizer_type
												_lowerCAmelCase							  =							ds_args.padded_vocab_size
												_lowerCAmelCase							  =							ds_args.max_position_embeddings
												_lowerCAmelCase							  =							ds_args.hidden_size
												_lowerCAmelCase							  =							ds_args.num_layers
												_lowerCAmelCase							  =							ds_args.num_attention_heads
												_lowerCAmelCase							  =							ds_args.ffn_hidden_size if """ffn_hidden_size""" in ds_args else 4 * ds_args.hidden_size
												# pprint(config)

						# The number of heads.
						_lowerCAmelCase							  =							config.num_attention_heads
						# The hidden_size per head.
						_lowerCAmelCase							  =							config.hidden_size // heads
						# Megatron-LM checkpoint version
						if "checkpoint_version" in input_state_dict.keys():
												_lowerCAmelCase							  =							input_state_dict["""checkpoint_version"""]
						else:
												_lowerCAmelCase							  =							0.0

						# The model.
						_lowerCAmelCase							  =							input_state_dict["""model"""]
						# The language model.
						_lowerCAmelCase							  =							model["""language_model"""]
						# The embeddings.
						_lowerCAmelCase							  =							lm["""embedding"""]

						# The word embeddings.
						_lowerCAmelCase							  =							embeddings["""word_embeddings"""]["""weight"""]
						# Truncate the embedding table to vocab_size rows.
						_lowerCAmelCase							  =							word_embeddings[: config.vocab_size, :]
						# Store the word embeddings.
						_lowerCAmelCase							  =							word_embeddings

						# The position embeddings.
						_lowerCAmelCase							  =							embeddings["""position_embeddings"""]["""weight"""]
						assert pos_embeddings.size(0						) == config.max_position_embeddings and pos_embeddings.size(1						) == config.hidden_size
						# Store the position embeddings.
						_lowerCAmelCase							  =							pos_embeddings

						# The token-type embeddings.
						_lowerCAmelCase							  =							embeddings["""tokentype_embeddings"""]["""weight"""]
						# Store the position embeddings.
						_lowerCAmelCase							  =							tokentype_embeddings

						# The transformer.
						_lowerCAmelCase							  =							lm["""transformer"""] if """transformer""" in lm.keys() else lm["""encoder"""]

						# The regex to extract layer names.
						_lowerCAmelCase							  =							re.compile(r"""layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"""						)

						# The simple map of names for "automated" rules.
						_lowerCAmelCase							  =							{
						    """attention.dense""": """.attention.output.dense.""",
						    """self_attention.dense""": """.attention.output.dense.""",
						    """mlp.dense_h_to_4h""": """.intermediate.dense.""",
						    """mlp.dense_4h_to_h""": """.output.dense.""",
						}

						# Keep track of the attention/query/value tensor.
						_lowerCAmelCase							  =							None

						# Extract the layers.
						for key, val in transformer.items():
												# Match the name.
												_lowerCAmelCase							  =							layer_re.match(lowerCAmelCase						)

												# Stop if that's not a layer
												if m is None:
																		break

												# The index of the layer.
												_lowerCAmelCase							  =							int(m.group(1						)						)
												# The name of the operation.
												_lowerCAmelCase							  =							m.group(2						)
												# Is it a weight or a bias?
												_lowerCAmelCase							  =							m.group(3						)

												# The name of the layer.
												_lowerCAmelCase							  =							f"bert.encoder.layer.{layer_idx}"

												# For layernorm(s), simply store the layer norm.
												if op_name.endswith("""layernorm"""						):
																		_lowerCAmelCase							  =							"""attention.ln""" if op_name.startswith("""input"""						) else """ln"""
																		_lowerCAmelCase							  =							val

												# Transpose the QKV matrix.
												elif (
												    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
												) and weight_or_bias == "weight":
																		# Make sure the QKV pointer is nil.
																		assert attention_qkv_weight is None, ""

																		_lowerCAmelCase							  =							fix_query_key_value_ordering(lowerCAmelCase							,			lowerCAmelCase							,			3							,			lowerCAmelCase							,			lowerCAmelCase						)
																		# Store the tensor as we need the bias as well to interleave QKV and biases.
																		_lowerCAmelCase							  =							out_val

												# Transpose the bias.
												elif (
												    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
												) and weight_or_bias == "bias":
																		# Make sure we read the weight tensor.
																		assert attention_qkv_weight is not None, ""

																		# Split the QKV matrix into Q, K and V. Megatron stores Q,K,V interleaved.
																		_lowerCAmelCase							  =							attention_qkv_weight[0 * config.hidden_size : 1 * config.hidden_size, :]
																		_lowerCAmelCase							  =							attention_qkv_weight[1 * config.hidden_size : 2 * config.hidden_size, :]
																		_lowerCAmelCase							  =							attention_qkv_weight[2 * config.hidden_size : 3 * config.hidden_size, :]

																		_lowerCAmelCase							  =							fix_query_key_value_ordering(lowerCAmelCase							,			lowerCAmelCase							,			3							,			lowerCAmelCase							,			lowerCAmelCase						)
																		# Split the bias.
																		_lowerCAmelCase							  =							out_val[0 * config.hidden_size : 1 * config.hidden_size]
																		_lowerCAmelCase							  =							out_val[1 * config.hidden_size : 2 * config.hidden_size]
																		_lowerCAmelCase							  =							out_val[2 * config.hidden_size : 3 * config.hidden_size]

																		# Store.
																		_lowerCAmelCase							  =							q
																		_lowerCAmelCase							  =							q_bias
																		_lowerCAmelCase							  =							k
																		_lowerCAmelCase							  =							k_bias
																		_lowerCAmelCase							  =							v
																		_lowerCAmelCase							  =							v_bias

																		# Clear the stored tensor.
																		_lowerCAmelCase							  =							None

												# Copy weights and biases as is.
												elif weight_or_bias in ["weight", "bias"]:
																		_lowerCAmelCase							  =							megatron_to_transformers[op_name]
																		_lowerCAmelCase							  =							val

    # The final layernorm.
						_lowerCAmelCase							  =							transformer["""final_layernorm.weight"""]
						_lowerCAmelCase							  =							transformer["""final_layernorm.bias"""]

						# The pooler.
						_lowerCAmelCase							  =							lm["""pooler"""]

						# Store the matrix and the bias.
						_lowerCAmelCase							  =							pooler["""dense.weight"""]
						_lowerCAmelCase							  =							pooler["""dense.bias"""]

						# The LM head from Megatron (for RACE).
						_lowerCAmelCase							  =							model["""lm_head"""]

						# The transform matrix.
						_lowerCAmelCase							  =							lm_head["""dense.weight"""]
						_lowerCAmelCase							  =							lm_head["""dense.bias"""]

						# The transform LN.
						_lowerCAmelCase							  =							lm_head["""layernorm.weight"""]
						_lowerCAmelCase							  =							lm_head["""layernorm.bias"""]

						# For the decoder, we replicate the weights.
						_lowerCAmelCase							  =							word_embeddings
						_lowerCAmelCase							  =							lm_head["""bias"""]

						# The classifier from Megatron (for MLNI).
						_lowerCAmelCase							  =							model["""binary_head"""]

						# Store the classifier.
						_lowerCAmelCase							  =							binary_head["""weight"""]
						_lowerCAmelCase							  =							binary_head["""bias"""]

						# It should be done!
						return output_state_dict
def      UpperCamelCase__   (	):





						"""simple docstring"""


						_lowerCAmelCase							  =							argparse.ArgumentParser()
						parser.add_argument("""--print-checkpoint-structure"""							,			action="""store_true"""						)
						parser.add_argument("""path_to_checkpoint"""							,			type=lowerCAmelCase							,			help="""Path to the ZIP file containing the checkpoint"""						)
						parser.add_argument(
						    """--config_file"""							,			default=""""""							,			type=lowerCAmelCase							,			help="""An optional config json file describing the pre-trained model."""							,			)
						_lowerCAmelCase							  =							parser.parse_args()

						# Extract the basename.
						_lowerCAmelCase							  =							os.path.dirname(args.path_to_checkpoint						)

						# Load the model.
						# the .zip is very optional, let's keep it for backward compatibility
						print(f"Extracting PyTorch state dictionary from \"{args.path_to_checkpoint}\""						)
						if args.path_to_checkpoint.endswith(""".zip"""						):
												with zipfile.ZipFile(args.path_to_checkpoint							,			"""r"""						) as checkpoint:
																		with checkpoint.open("""release/mp_rank_00/model_optim_rng.pt"""						) as pytorch_dict:
																								_lowerCAmelCase							  =							torch.load(lowerCAmelCase							,			map_location="""cpu"""						)
						else:
												_lowerCAmelCase							  =							torch.load(args.path_to_checkpoint							,			map_location="""cpu"""						)

						if args.config_file == "":
												# Default config of megatron-bert 345m
												_lowerCAmelCase							  =							MegatronBertConfig()

												# different megatron-bert-*-345m models have different vocab sizes, so override the default
												# config (which is for megatron-bert-cased-345m) with the actual vocab dimension
												_lowerCAmelCase							  =							input_state_dict["""model"""]["""lm_head"""]["""bias"""].numel()
						else:
												_lowerCAmelCase							  =							MegatronBertConfig.from_json_file(args.config_file						)

						# Convert.
						print("""Converting"""						)
						_lowerCAmelCase							  =							convert_megatron_checkpoint(lowerCAmelCase							,			lowerCAmelCase							,			lowerCAmelCase						)

						# Print the structure of converted state dict.
						if args.print_checkpoint_structure:
												recursive_print(lowerCAmelCase							,			lowerCAmelCase						)

						# Store the config to file.
						print("""Saving config"""						)
						config.save_pretrained(lowerCAmelCase						)

						# Store the state_dict to file.
						_lowerCAmelCase							  =							os.path.join(lowerCAmelCase							,			"""pytorch_model.bin"""						)
						print(f"Saving checkpoint to \"{output_checkpoint_file}\""						)
						torch.save(lowerCAmelCase							,			lowerCAmelCase						)


####################################################################################################

if __name__ == "__main__":
						main()

####################################################################################################













'''simple docstring'''






import unittest

import numpy as np

from transformers import ElectraConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
      from transformers.models.electra.modeling_flax_electra import (
          FlaxElectraForCausalLM,
          FlaxElectraForMaskedLM,
          FlaxElectraForMultipleChoice,
          FlaxElectraForPreTraining,
          FlaxElectraForQuestionAnswering,
          FlaxElectraForSequenceClassification,
          FlaxElectraForTokenClassification,
          FlaxElectraModel,
      )




class 				UpperCAmelCase  (    unittest.TestCase     ):




      def __init__(    self    :				str    ,    __snake_case    :				str    ,    __snake_case    :				str=13    ,    __snake_case    :				Optional[Any]=7    ,    __snake_case    :				str=True    ,    __snake_case    :				Any=True    ,    __snake_case    :				List[str]=True    ,    __snake_case    :				Any=True    ,    __snake_case    :				List[str]=99    ,    __snake_case    :				Dict=24    ,    __snake_case    :				Any=32    ,    __snake_case    :				List[str]=5    ,    __snake_case    :				Any=4    ,    __snake_case    :				Optional[Any]=37    ,    __snake_case    :				Union[str, Any]="gelu"    ,    __snake_case    :				Dict=0.1    ,    __snake_case    :				int=0.1    ,    __snake_case    :				Dict=5_12    ,    __snake_case    :				List[Any]=16    ,    __snake_case    :				Dict=2    ,    __snake_case    :				str=0.02    ,    __snake_case    :				Any=4    ,    )       ->   Union[str, Any]:
            _lowerCAmelCase							  =							parent
            _lowerCAmelCase							  =							batch_size
            _lowerCAmelCase							  =							seq_length
            _lowerCAmelCase							  =							is_training
            _lowerCAmelCase							  =							use_attention_mask
            _lowerCAmelCase							  =							use_token_type_ids
            _lowerCAmelCase							  =							use_labels
            _lowerCAmelCase							  =							vocab_size
            _lowerCAmelCase							  =							hidden_size
            _lowerCAmelCase							  =							embedding_size
            _lowerCAmelCase							  =							num_hidden_layers
            _lowerCAmelCase							  =							num_attention_heads
            _lowerCAmelCase							  =							intermediate_size
            _lowerCAmelCase							  =							hidden_act
            _lowerCAmelCase							  =							hidden_dropout_prob
            _lowerCAmelCase							  =							attention_probs_dropout_prob
            _lowerCAmelCase							  =							max_position_embeddings
            _lowerCAmelCase							  =							type_vocab_size
            _lowerCAmelCase							  =							type_sequence_label_size
            _lowerCAmelCase							  =							initializer_range
            _lowerCAmelCase							  =							num_choices




      def        lowercase__					(    self    :				Optional[Any]						)       ->   Optional[Any]:
            _lowerCAmelCase							  =							ids_tensor([self.batch_size, self.seq_length]    ,    self.vocab_size						)

            _lowerCAmelCase							  =							None
            if self.use_attention_mask:
                  _lowerCAmelCase							  =							random_attention_mask([self.batch_size, self.seq_length]						)

            _lowerCAmelCase							  =							None
            if self.use_token_type_ids:
                  _lowerCAmelCase							  =							ids_tensor([self.batch_size, self.seq_length]    ,    self.type_vocab_size						)

            _lowerCAmelCase							  =							ElectraConfig(
                vocab_size=self.vocab_size    ,    hidden_size=self.hidden_size    ,    embedding_size=self.embedding_size    ,    num_hidden_layers=self.num_hidden_layers    ,    num_attention_heads=self.num_attention_heads    ,    intermediate_size=self.intermediate_size    ,    hidden_act=self.hidden_act    ,    hidden_dropout_prob=self.hidden_dropout_prob    ,    attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,    max_position_embeddings=self.max_position_embeddings    ,    type_vocab_size=self.type_vocab_size    ,    initializer_range=self.initializer_range    ,    )

            return config, input_ids, token_type_ids, attention_mask




      def        lowercase__					(    self    :				Any						)       ->   Optional[int]:
            _lowerCAmelCase							  =							self.prepare_config_and_inputs()
            _lowerCAmelCase  ,							_lowerCAmelCase  ,							_lowerCAmelCase  ,							_lowerCAmelCase							  =							config_and_inputs
            _lowerCAmelCase							  =							{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": attention_mask}
            return config, inputs_dict




@require_flax
class 				UpperCAmelCase  (    snake_case_							,	unittest.TestCase     ):
      _lowercase:    str		  =     True

      _lowercase:    Optional[Any]		  =     (
          (
              FlaxElectraModel,
              FlaxElectraForCausalLM,
              FlaxElectraForMaskedLM,
              FlaxElectraForPreTraining,
              FlaxElectraForTokenClassification,
              FlaxElectraForQuestionAnswering,
              FlaxElectraForMultipleChoice,
              FlaxElectraForSequenceClassification,
          )
          if is_flax_available()
          else ()
      )




      def        lowercase__					(    self    :				Tuple						)       ->   Optional[int]:
            _lowerCAmelCase							  =							FlaxElectraModelTester(self						)




      @slow
      def        lowercase__					(    self    :				List[Any]						)       ->   str:
            for model_class_name in self.all_model_classes:
                  if model_class_name == FlaxElectraForMaskedLM:
                        _lowerCAmelCase							  =							model_class_name.from_pretrained("""google/electra-small-generator"""						)
                  else:
                        _lowerCAmelCase							  =							model_class_name.from_pretrained("""google/electra-small-discriminator"""						)
                  _lowerCAmelCase							  =							model(np.ones((1, 1)						)						)
                  self.assertIsNotNone(__snake_case						)






