






import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig






def        lowerCamelCase__   (					__lowerCamelCase    :  Union[str, Any]						,	__lowerCamelCase    :  int						,	__lowerCamelCase    :  List[Any]=0	):
  # Format the message.
  if name is None:
    __UpperCAmelCase							:			Union[str, Any]     =						None
  else:
    __UpperCAmelCase							:			str     =						""".""" * max(0						,	spaces - 2	) + """# {:""" + str(50 - spaces	) + """s}"""
    __UpperCAmelCase							:			Union[str, Any]     =						fmt.format(__lowerCamelCase	)

  # Print and recurse (if needed).
  if isinstance(__lowerCamelCase						,	__lowerCamelCase	):
    if msg is not None:
      print(__lowerCamelCase	)
    for k in val.keys():
      recursive_print(__lowerCamelCase						,	val[k]						,	spaces + 2	)
  elif isinstance(__lowerCamelCase						,	torch.Tensor	):
    print(__lowerCamelCase						,	""":"""						,	val.size()	)
  else:
    print(__lowerCamelCase						,	""":"""						,	__lowerCamelCase	)






def        lowerCamelCase__   (					__lowerCamelCase    :  Union[str, Any]						,	__lowerCamelCase    :  int						,	__lowerCamelCase    :  List[str]						,	__lowerCamelCase    :  Optional[int]	):
  # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
  # for compatibility with later versions of NVIDIA Megatron-LM.
  # The inverse operation is performed inside Megatron-LM to read checkpoints:
  # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
  # If param is the weight tensor of the self-attention block, the returned tensor
  # will have to be transposed one more time to be read by HuggingFace GPT2.
  __UpperCAmelCase							:			Optional[int]     =						param.size()
  # other versions store [num_heads * num_splits * hidden_size, :]
  __UpperCAmelCase							:			str     =						(num_heads, num_splits, hidden_size) + input_shape[1:]
  __UpperCAmelCase							:			List[str]     =						param.view(*__lowerCamelCase	)
  __UpperCAmelCase							:			Optional[Any]     =						param.transpose(0						,	1	).contiguous()
  __UpperCAmelCase							:			str     =						param.view(*__lowerCamelCase	)
  return param






def        lowerCamelCase__   (					__lowerCamelCase    :  List[Any]						,	__lowerCamelCase    :  int	):
  __UpperCAmelCase							:			List[str]     =						config.n_positions
  __UpperCAmelCase							:			str     =						config.n_layer
  __UpperCAmelCase							:			List[Any]     =						config.vocab_size
  __UpperCAmelCase							:			Optional[int]     =						config.n_head
  __UpperCAmelCase							:			Optional[Any]     =						config.n_embd // config.n_head

  __UpperCAmelCase							:			Optional[int]     =						sd_megatron["""model.language_model.embedding.word_embeddings.weight"""][:vocab_size, :]
  __UpperCAmelCase							:			Dict     =						{
      """transformer.wte.weight""": word_embeddings,
      """transformer.wpe.weight""": sd_megatron["""model.language_model.embedding.position_embeddings.weight"""],
      """transformer.ln_f.weight""": sd_megatron["""model.language_model.encoder.final_layernorm.weight"""],
      """transformer.ln_f.bias""": sd_megatron["""model.language_model.encoder.final_layernorm.bias"""],
  }

  __UpperCAmelCase							:			List[str]     =						"""model.language_model.encoder.layers."""
  for i in range(__lowerCamelCase	):
    __UpperCAmelCase							:			Union[str, Any]     =						torch.tril(torch.ones((n_positions, n_positions)						,	dtype=torch.bool	)	)
    __UpperCAmelCase							:			Optional[Any]     =						causal_mask.view(1						,	1						,	__lowerCamelCase						,	__lowerCamelCase	)
    __UpperCAmelCase							:			Tuple     =						causal_mask
    __UpperCAmelCase							:			Optional[Any]     =						torch.tensor(-1E4						,	dtype=torch.bfloataa	)

    __UpperCAmelCase							:			int     =						sd_megatron[f"""{pf}{i}.input_layernorm.weight"""]
    __UpperCAmelCase							:			List[Any]     =						sd_megatron[f"""{pf}{i}.input_layernorm.bias"""]

    __UpperCAmelCase							:			Dict     =						sd_megatron[f"""{pf}{i}.self_attention.query_key_value.weight"""]
    __UpperCAmelCase							:			Optional[Any]     =						fix_query_key_value_ordering(__lowerCamelCase						,	3						,	__lowerCamelCase						,	__lowerCamelCase	)
    __UpperCAmelCase							:			Optional[Any]     =						vala.transpose(0						,	1	).contiguous()

    __UpperCAmelCase							:			Dict     =						sd_megatron[f"""{pf}{i}.self_attention.query_key_value.bias"""]
    __UpperCAmelCase							:			Dict     =						fix_query_key_value_ordering(__lowerCamelCase						,	3						,	__lowerCamelCase						,	__lowerCamelCase	)
    __UpperCAmelCase							:			str     =						vala

    __UpperCAmelCase							:			Dict     =						sd_megatron[f"""{pf}{i}.self_attention.dense.weight"""].transpose(
        0						,	1	)
    __UpperCAmelCase							:			Optional[int]     =						sd_megatron[f"""{pf}{i}.self_attention.dense.bias"""]
    __UpperCAmelCase							:			Tuple     =						sd_megatron[f"""{pf}{i}.post_attention_layernorm.weight"""]
    __UpperCAmelCase							:			List[Any]     =						sd_megatron[f"""{pf}{i}.post_attention_layernorm.bias"""]
    __UpperCAmelCase							:			List[Any]     =						sd_megatron[f"""{pf}{i}.mlp.dense_h_to_4h.weight"""].transpose(0						,	1	)
    __UpperCAmelCase							:			str     =						sd_megatron[f"""{pf}{i}.mlp.dense_h_to_4h.bias"""]
    __UpperCAmelCase							:			Optional[int]     =						sd_megatron[f"""{pf}{i}.mlp.dense_4h_to_h.weight"""].transpose(
        0						,	1	)
    __UpperCAmelCase							:			str     =						sd_megatron[f"""{pf}{i}.mlp.dense_4h_to_h.bias"""]

  # For LM head, transformers' wants the matrix to weight embeddings.
  __UpperCAmelCase							:			str     =						word_embeddings

  return sd_hf






def        lowerCamelCase__   (					__lowerCamelCase    :  Dict						,	__lowerCamelCase    :  Dict	):
  __UpperCAmelCase							:			List[Any]     =						64000
  __UpperCAmelCase							:			Optional[Any]     =						config_megatron["""encoder_seq_length"""]
  __UpperCAmelCase							:			List[str]     =						config_megatron["""hidden_size"""]
  __UpperCAmelCase							:			Any     =						config_megatron["""num_layers"""]
  __UpperCAmelCase							:			Any     =						config_megatron["""num_attention_heads"""]
  __UpperCAmelCase							:			List[Any]     =						config_megatron["""ffn_hidden_size"""]
  __UpperCAmelCase							:			str     =						"""gelu"""
  __UpperCAmelCase							:			Tuple     =						0.1
  __UpperCAmelCase							:			str     =						0.1
  __UpperCAmelCase							:			List[str]     =						0.1
  __UpperCAmelCase							:			Optional[int]     =						config_megatron["""layernorm_epsilon"""]  # 1e-5
  __UpperCAmelCase							:			List[Any]     =						config_megatron["""init_method_std"""]  # 0.02
  __UpperCAmelCase							:			Optional[int]     =						config_megatron["""apply_query_key_layer_scaling"""]  # True
  __UpperCAmelCase							:			List[Any]     =						True
  __UpperCAmelCase							:			Any     =						True

  # This identifies the 6.7B (7B) model which uses a different tokenizer
  if config_megatron["hidden_size"] == 4096:
    __UpperCAmelCase							:			Union[str, Any]     =						1  # <|endoftext|>
    __UpperCAmelCase							:			int     =						1  # <|endoftext|>
    __UpperCAmelCase							:			List[str]     =						0  # <unk>
  else:
    __UpperCAmelCase							:			List[Any]     =						2  # <s>
    __UpperCAmelCase							:			str     =						3  # <|endoftext|>
    __UpperCAmelCase							:			Tuple     =						0  # <pad>

  return config_hf






def        lowerCamelCase__   (					__lowerCamelCase    :  Optional[int]	):
  print(__lowerCamelCase	)

  __UpperCAmelCase							:			Optional[Any]     =						args.checkpoint_path
  __UpperCAmelCase							:			str     =						args.save_path
  if isfile(__lowerCamelCase	):
    raise FileNotFoundError(f"""ERROR! could not find file {checkpoint_path}"""	)

  # Load the model.
  __UpperCAmelCase							:			Any     =						torch.load(__lowerCamelCase						,	map_location="""cpu"""	)

  # Load the config.
  __UpperCAmelCase							:			Optional[int]     =						checkpoint["""hyper_parameters"""]["""cfg"""]
  __UpperCAmelCase							:			Tuple     =						GPTaConfig()
  __UpperCAmelCase							:			Tuple     =						copy_config(config_hf=__lowerCamelCase						,	config_megatron=__lowerCamelCase	)
  __UpperCAmelCase							:			Any     =						["""GPT2LMHeadModel"""]

  __UpperCAmelCase							:			List[Any]     =						checkpoint["""state_dict"""]

  # Convert.
  print("""Converting"""	)
  __UpperCAmelCase							:			Any     =						convert_megatron_checkpoint(__lowerCamelCase						,	__lowerCamelCase	)

  # Print the structure of converted state dict.
  if args.print_checkpoint_structure:
    recursive_print(__lowerCamelCase						,	__lowerCamelCase	)

  __UpperCAmelCase							:			Tuple     =						"""GPTSw3Tokenizer"""

  # Store the config to file.
  print("""Saving config"""	)
  config_hf.save_pretrained(__lowerCamelCase	)

  # Store the state_dict to file.
  __UpperCAmelCase							:			Any     =						os.path.join(__lowerCamelCase						,	"""pytorch_model.bin"""	)
  print(f"""Saving checkpoint to \"{output_checkpoint_file}\""""	)
  torch.save(__lowerCamelCase						,	__lowerCamelCase	)


if __name__ == "__main__":
       a	: Optional[int]      =				argparse.ArgumentParser()
       parser.add_argument(
           "--checkpoint_path",
           type=str,
           required=True,
           help="e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000",
       )
       parser.add_argument("--save_path", type=str, required=True, help="e.g. /home/user/gpt-sw3/hf")
       parser.add_argument("--print-checkpoint-structure", action="store_true")
       a	: List[Any]      =				parser.parse_args()
       main(_args)










from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available, is_vision_available


a	: Optional[int]      =				{
    "configuration_deformable_detr": ["DEFORMABLE_DETR_PRETRAINED_CONFIG_ARCHIVE_MAP", "DeformableDetrConfig"],
}

try:
       if not is_vision_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       a	: Optional[int]      =				["DeformableDetrFeatureExtractor"]
       a	: List[str]      =				["DeformableDetrImageProcessor"]

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       a	: List[Any]      =				[
           "DEFORMABLE_DETR_PRETRAINED_MODEL_ARCHIVE_LIST",
           "DeformableDetrForObjectDetection",
           "DeformableDetrModel",
           "DeformableDetrPreTrainedModel",
       ]


if TYPE_CHECKING:
       from .configuration_deformable_detr import DEFORMABLE_DETR_PRETRAINED_CONFIG_ARCHIVE_MAP, DeformableDetrConfig

       try:
              if not is_vision_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .feature_extraction_deformable_detr import DeformableDetrFeatureExtractor
              from .image_processing_deformable_detr import DeformableDetrImageProcessor

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_deformable_detr import (
                  DEFORMABLE_DETR_PRETRAINED_MODEL_ARCHIVE_LIST,
                  DeformableDetrForObjectDetection,
                  DeformableDetrModel,
                  DeformableDetrPreTrainedModel,
              )

else:
       import sys

       a	: List[Any]      =				_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)


