





import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig





def 							__lowerCAmelCase(					_SCREAMING_SNAKE_CASE       ,    _SCREAMING_SNAKE_CASE       ,    _SCREAMING_SNAKE_CASE=0							)       ->   Union[str, Any]:




    """simple docstring"""




    if name is None:
        _A    =       None
    else:
        _A    =       '.' * max(0       ,    spaces - 2							) + '# {:' + str(50 - spaces							) + 's}'
        _A    =       fmt.format(_SCREAMING_SNAKE_CASE							)

    # Print and recurse (if needed).
    if isinstance(_SCREAMING_SNAKE_CASE       ,    _SCREAMING_SNAKE_CASE							):
        if msg is not None:
            print(_SCREAMING_SNAKE_CASE							)
        for k in val.keys():
            recursive_print(_SCREAMING_SNAKE_CASE       ,    val[k]       ,    spaces + 2							)
    elif isinstance(_SCREAMING_SNAKE_CASE       ,    torch.Tensor							):
        print(_SCREAMING_SNAKE_CASE       ,    ':'       ,    val.size()							)
    else:
        print(_SCREAMING_SNAKE_CASE       ,    ':'       ,    _SCREAMING_SNAKE_CASE							)





def 							__lowerCAmelCase(					_SCREAMING_SNAKE_CASE       ,    _SCREAMING_SNAKE_CASE       ,    _SCREAMING_SNAKE_CASE       ,    _SCREAMING_SNAKE_CASE							)       ->   int:




    """simple docstring"""




    _A    =       param.size()
    # other versions store [num_heads * num_splits * hidden_size, :]
    _A    =       (num_heads, num_splits, hidden_size) + input_shape[1:]
    _A    =       param.view(*_SCREAMING_SNAKE_CASE							)
    _A    =       param.transpose(0       ,    1							).contiguous()
    _A    =       param.view(*_SCREAMING_SNAKE_CASE							)
    return param





def 							__lowerCAmelCase(					_SCREAMING_SNAKE_CASE       ,    _SCREAMING_SNAKE_CASE							)       ->   Optional[int]:




    """simple docstring"""




    _A    =       config.n_positions
    _A    =       config.n_layer
    _A    =       config.vocab_size
    _A    =       config.n_head
    _A    =       config.n_embd // config.n_head

    _A    =       sd_megatron['model.language_model.embedding.word_embeddings.weight'][:vocab_size, :]
    _A    =       {
        'transformer.wte.weight': word_embeddings,
        'transformer.wpe.weight': sd_megatron['model.language_model.embedding.position_embeddings.weight'],
        'transformer.ln_f.weight': sd_megatron['model.language_model.encoder.final_layernorm.weight'],
        'transformer.ln_f.bias': sd_megatron['model.language_model.encoder.final_layernorm.bias'],
    }

    _A    =       'model.language_model.encoder.layers.'
    for i in range(_SCREAMING_SNAKE_CASE							):
        _A    =       torch.tril(torch.ones((n_positions, n_positions)       ,    dtype=torch.bool							)							)
        _A    =       causal_mask.view(1       ,    1       ,    _SCREAMING_SNAKE_CASE       ,    _SCREAMING_SNAKE_CASE							)
        _A    =       causal_mask
        _A    =       torch.tensor(-1e4       ,    dtype=torch.bfloataa							)

        _A    =       sd_megatron[F"{pf}{i}.input_layernorm.weight"]
        _A    =       sd_megatron[F"{pf}{i}.input_layernorm.bias"]

        _A    =       sd_megatron[F"{pf}{i}.self_attention.query_key_value.weight"]
        _A    =       fix_query_key_value_ordering(_SCREAMING_SNAKE_CASE       ,    3       ,    _SCREAMING_SNAKE_CASE       ,    _SCREAMING_SNAKE_CASE							)
        _A    =       vala.transpose(0       ,    1							).contiguous()

        _A    =       sd_megatron[F"{pf}{i}.self_attention.query_key_value.bias"]
        _A    =       fix_query_key_value_ordering(_SCREAMING_SNAKE_CASE       ,    3       ,    _SCREAMING_SNAKE_CASE       ,    _SCREAMING_SNAKE_CASE							)
        _A    =       vala

        _A    =       sd_megatron[F"{pf}{i}.self_attention.dense.weight"].transpose(
            0       ,    1							)
        _A    =       sd_megatron[F"{pf}{i}.self_attention.dense.bias"]
        _A    =       sd_megatron[F"{pf}{i}.post_attention_layernorm.weight"]
        _A    =       sd_megatron[F"{pf}{i}.post_attention_layernorm.bias"]
        _A    =       sd_megatron[F"{pf}{i}.mlp.dense_h_to_4h.weight"].transpose(0       ,    1							)
        _A    =       sd_megatron[F"{pf}{i}.mlp.dense_h_to_4h.bias"]
        _A    =       sd_megatron[F"{pf}{i}.mlp.dense_4h_to_h.weight"].transpose(
            0       ,    1							)
        _A    =       sd_megatron[F"{pf}{i}.mlp.dense_4h_to_h.bias"]

    # For LM head, transformers' wants the matrix to weight embeddings.
    _A    =       word_embeddings

    return sd_hf





def 							__lowerCAmelCase(					_SCREAMING_SNAKE_CASE       ,    _SCREAMING_SNAKE_CASE							)       ->   int:




    """simple docstring"""




    _A    =       64_000
    _A    =       config_megatron['encoder_seq_length']
    _A    =       config_megatron['hidden_size']
    _A    =       config_megatron['num_layers']
    _A    =       config_megatron['num_attention_heads']
    _A    =       config_megatron['ffn_hidden_size']
    _A    =       'gelu'
    _A    =       0.1
    _A    =       0.1
    _A    =       0.1
    _A    =       config_megatron['layernorm_epsilon']  # 1e-5
    _A    =       config_megatron['init_method_std']  # 0.02
    _A    =       config_megatron['apply_query_key_layer_scaling']  # True
    _A    =       True
    _A    =       True

    # This identifies the 6.7B (7B) model which uses a different tokenizer
    if config_megatron["hidden_size"] == 4_096:
        _A    =       1  # <|endoftext|>
        _A    =       1  # <|endoftext|>
        _A    =       0  # <unk>
    else:
        _A    =       2  # <s>
        _A    =       3  # <|endoftext|>
        _A    =       0  # <pad>

    return config_hf





def 							__lowerCAmelCase(					_SCREAMING_SNAKE_CASE							)       ->   int:




    """simple docstring"""




    print(_SCREAMING_SNAKE_CASE							)

    _A    =       args.checkpoint_path
    _A    =       args.save_path
    if isfile(_SCREAMING_SNAKE_CASE							):
        raise FileNotFoundError(F"ERROR! could not find file {checkpoint_path}"							)

    # Load the model.
    _A    =       torch.load(_SCREAMING_SNAKE_CASE       ,    map_location='cpu'							)

    # Load the config.
    _A    =       checkpoint['hyper_parameters']['cfg']
    _A    =       GPTaConfig()
    _A    =       copy_config(config_hf=_SCREAMING_SNAKE_CASE       ,    config_megatron=_SCREAMING_SNAKE_CASE							)
    _A    =       ['GPT2LMHeadModel']

    _A    =       checkpoint['state_dict']

    # Convert.
    print('Converting'							)
    _A    =       convert_megatron_checkpoint(_SCREAMING_SNAKE_CASE       ,    _SCREAMING_SNAKE_CASE							)

    # Print the structure of converted state dict.
    if args.print_checkpoint_structure:
        recursive_print(_SCREAMING_SNAKE_CASE       ,    _SCREAMING_SNAKE_CASE							)

    _A    =       'GPTSw3Tokenizer'

    # Store the config to file.
    print('Saving config'							)
    config_hf.save_pretrained(_SCREAMING_SNAKE_CASE							)

    # Store the state_dict to file.
    _A    =       os.path.join(_SCREAMING_SNAKE_CASE       ,    'pytorch_model.bin'							)
    print(F"Saving checkpoint to \"{output_checkpoint_file}\""							)
    torch.save(_SCREAMING_SNAKE_CASE       ,    _SCREAMING_SNAKE_CASE							)


if __name__ == "__main__":
 __A       : Tuple						    = argparse.ArgumentParser()
 parser.add_argument(
     "--checkpoint_path",
     type=str,
     required=True,
     help="e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000",
 )
 parser.add_argument("--save_path", type=str, required=True, help="e.g. /home/user/gpt-sw3/hf")
 parser.add_argument("--print-checkpoint-structure", action="store_true")
 __A       : Optional[Any]						    = parser.parse_args()
 main(_args)












from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


__A       : Dict						    = {"configuration_hubert": ["HUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "HubertConfig"]}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __A       : List[str]						    = [
     "HUBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
     "HubertForCTC",
     "HubertForSequenceClassification",
     "HubertModel",
     "HubertPreTrainedModel",
 ]


try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __A       : Tuple						    = [
     "TF_HUBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
     "TFHubertForCTC",
     "TFHubertModel",
     "TFHubertPreTrainedModel",
 ]

if TYPE_CHECKING:
 from .configuration_hubert import HUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, HubertConfig

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_hubert import (
      HUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      HubertForCTC,
      HubertForSequenceClassification,
      HubertModel,
      HubertPreTrainedModel,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_hubert import (
      TF_HUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFHubertForCTC,
      TFHubertModel,
      TFHubertPreTrainedModel,
  )


else:
 import sys

 __A       : Dict						    = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





