



import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig





def 						__lowerCamelCase       (      __magic_name__	:  Optional[Any]     , __magic_name__	:  str     , __magic_name__	:  Optional[Any]=0						):
							# Format the message.
							if name is None:
														a__:       Optional[int]											=None
							else:
														a__:       Union[str, Any]											="." * max(0     , spaces - 2						) + "# {:" + str(50 - spaces						) + "s}"
														a__:       Optional[int]											=fmt.format(__magic_name__						)

							# Print and recurse (if needed).
							if isinstance(__magic_name__     , __magic_name__						):
														if msg is not None:
																					print(__magic_name__						)
														for k in val.keys():
																					recursive_print(__magic_name__     , val[k]     , spaces + 2						)
							elif isinstance(__magic_name__     , torch.Tensor						):
														print(__magic_name__     , ":"     , val.size()						)
							else:
														print(__magic_name__     , ":"     , __magic_name__						)





def 						__lowerCamelCase       (      __magic_name__	:  int     , __magic_name__	:  str     , __magic_name__	:  Tuple     , __magic_name__	:  List[str]						):
							a__:       Optional[int]											=param.size()
							# other versions store [num_heads * num_splits * hidden_size, :]
							a__:       Optional[int]											=(num_heads, num_splits, hidden_size) + input_shape[1:]
							a__:       Tuple											=param.view(*__magic_name__						)
							a__:       Dict											=param.transpose(0     , 1						).contiguous()
							a__:       List[Any]											=param.view(*__magic_name__						)
							return param





def 						__lowerCamelCase       (      __magic_name__	:  Union[str, Any]     , __magic_name__	:  str						):
							a__:       Tuple											=config.n_positions
							a__:       List[str]											=config.n_layer
							a__:       int											=config.vocab_size
							a__:       Union[str, Any]											=config.n_head
							a__:       List[Any]											=config.n_embd // config.n_head

							a__:       Tuple											=sd_megatron["model.language_model.embedding.word_embeddings.weight"][:vocab_size, :]
							a__:       Union[str, Any]											={
							    "transformer.wte.weight": word_embeddings,
							    "transformer.wpe.weight": sd_megatron["model.language_model.embedding.position_embeddings.weight"],
							    "transformer.ln_f.weight": sd_megatron["model.language_model.encoder.final_layernorm.weight"],
							    "transformer.ln_f.bias": sd_megatron["model.language_model.encoder.final_layernorm.bias"],
							}

							a__:       Dict											="model.language_model.encoder.layers."
							for i in range(__magic_name__						):
														a__:       List[Any]											=torch.tril(torch.ones((n_positions, n_positions)     , dtype=torch.bool						)						)
														a__:       List[Any]											=causal_mask.view(1     , 1     , __magic_name__     , __magic_name__						)
														a__:       List[Any]											=causal_mask
														a__:       Optional[int]											=torch.tensor(-1e4     , dtype=torch.bfloataa						)

														a__:       Any											=sd_megatron[F"{pf}{i}.input_layernorm.weight"]
														a__:       List[Any]											=sd_megatron[F"{pf}{i}.input_layernorm.bias"]

														a__:       int											=sd_megatron[F"{pf}{i}.self_attention.query_key_value.weight"]
														a__:       List[Any]											=fix_query_key_value_ordering(__magic_name__     , 3     , __magic_name__     , __magic_name__						)
														a__:       Tuple											=vala.transpose(0     , 1						).contiguous()

														a__:       Union[str, Any]											=sd_megatron[F"{pf}{i}.self_attention.query_key_value.bias"]
														a__:       str											=fix_query_key_value_ordering(__magic_name__     , 3     , __magic_name__     , __magic_name__						)
														a__:       Union[str, Any]											=vala

														a__:       Tuple											=sd_megatron[F"{pf}{i}.self_attention.dense.weight"].transpose(
														    0     , 1						)
														a__:       List[str]											=sd_megatron[F"{pf}{i}.self_attention.dense.bias"]
														a__:       Optional[Any]											=sd_megatron[F"{pf}{i}.post_attention_layernorm.weight"]
														a__:       str											=sd_megatron[F"{pf}{i}.post_attention_layernorm.bias"]
														a__:       List[str]											=sd_megatron[F"{pf}{i}.mlp.dense_h_to_4h.weight"].transpose(0     , 1						)
														a__:       Dict											=sd_megatron[F"{pf}{i}.mlp.dense_h_to_4h.bias"]
														a__:       Dict											=sd_megatron[F"{pf}{i}.mlp.dense_4h_to_h.weight"].transpose(
														    0     , 1						)
														a__:       int											=sd_megatron[F"{pf}{i}.mlp.dense_4h_to_h.bias"]

							# For LM head, transformers' wants the matrix to weight embeddings.
							a__:       Tuple											=word_embeddings

							return sd_hf





def 						__lowerCamelCase       (      __magic_name__	:  Any     , __magic_name__	:  Union[str, Any]						):
							a__:       Optional[int]											=64_000
							a__:       str											=config_megatron["encoder_seq_length"]
							a__:       Tuple											=config_megatron["hidden_size"]
							a__:       Optional[int]											=config_megatron["num_layers"]
							a__:       Tuple											=config_megatron["num_attention_heads"]
							a__:       str											=config_megatron["ffn_hidden_size"]
							a__:       Union[str, Any]											="gelu"
							a__:       Optional[Any]											=0.1
							a__:       Union[str, Any]											=0.1
							a__:       Optional[int]											=0.1
							a__:       Tuple											=config_megatron["layernorm_epsilon"]  # 1e-5
							a__:       List[str]											=config_megatron["init_method_std"]  # 0.02
							a__:       Union[str, Any]											=config_megatron["apply_query_key_layer_scaling"]  # True
							a__:       List[Any]											=True
							a__:       Tuple											=True

							# This identifies the 6.7B (7B) model which uses a different tokenizer
							if config_megatron["hidden_size"] == 4_096:
														a__:       Optional[Any]											=1  # <|endoftext|>
														a__:       Any											=1  # <|endoftext|>
														a__:       Optional[int]											=0  # <unk>
							else:
														a__:       str											=2  # <s>
														a__:       Optional[Any]											=3  # <|endoftext|>
														a__:       int											=0  # <pad>

							return config_hf





def 						__lowerCamelCase       (      __magic_name__	:  List[Any]						):
							print(__magic_name__						)

							a__:       int											=args.checkpoint_path
							a__:       int											=args.save_path
							if isfile(__magic_name__						):
														raise FileNotFoundError(F"ERROR! could not find file {checkpoint_path}"						)

							# Load the model.
							a__:       Union[str, Any]											=torch.load(__magic_name__     , map_location="cpu"						)

							# Load the config.
							a__:       Optional[Any]											=checkpoint["hyper_parameters"]["cfg"]
							a__:       Optional[int]											=GPTaConfig()
							a__:       Optional[int]											=copy_config(config_hf=__magic_name__     , config_megatron=__magic_name__						)
							a__:       Union[str, Any]											=["GPT2LMHeadModel"]

							a__:       str											=checkpoint["state_dict"]

							# Convert.
							print("Converting"						)
							a__:       str											=convert_megatron_checkpoint(__magic_name__     , __magic_name__						)

							# Print the structure of converted state dict.
							if args.print_checkpoint_structure:
														recursive_print(__magic_name__     , __magic_name__						)

							a__:       int											="GPTSw3Tokenizer"

							# Store the config to file.
							print("Saving config"						)
							config_hf.save_pretrained(__magic_name__						)

							# Store the state_dict to file.
							a__:       Dict											=os.path.join(__magic_name__     , "pytorch_model.bin"						)
							print(F"Saving checkpoint to \"{output_checkpoint_file}\""						)
							torch.save(__magic_name__     , __magic_name__						)


if __name__ == "__main__":
		__UpperCAmelCase						  =     argparse.ArgumentParser()
		parser.add_argument(
		    '''--checkpoint_path''',
		    type=str,
		    required=True,
		    help='''e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000''',
		)
		parser.add_argument('''--save_path''', type=str, required=True, help='''e.g. /home/user/gpt-sw3/hf''')
		parser.add_argument('''--print-checkpoint-structure''', action='''store_true''')
		__UpperCAmelCase						  =     parser.parse_args()
		main(_args)










# This test is meant to be run in on an instance with TPUs like this:
#
#   python examples/pytorch/xla_spawn.py --num_cores=8 tests/test_trainer_tpu.py
#
# Replace 8 with the number of TPU cores you have.
#

import sys
from typing import Dict

from transformers import EvalPrediction, HfArgumentParser, TrainingArguments, is_torch_available
from transformers.utils import logging


__UpperCAmelCase						  =     logging.get_logger(__name__)


if is_torch_available():
		import torch
		from torch import nn
		from torch.utils.data import Dataset

		from transformers import Trainer


		class 	lowerCamelCase__ (		_a   ):


								def __init__( self						:  Any      ,       _a						:  int = 1_0_1			):
															a__:       Dict											=length


								def __len__( self						:  Optional[Any]			):
															return self.length







								def __getitem__( self						:  Dict      ,       _a						:  int			):
															return i


		class 	lowerCamelCase__ :


								def __call__( self						:  Any      ,       _a						:  List[str]			):
															return {"input_ids": torch.tensor(_a			), "labels": torch.tensor(_a			)}


		class 	lowerCamelCase__ (		nn.Module   ):


								def __init__( self						:  Tuple			):
															super().__init__()
															# Add some (unused) params otherwise DDP will complain.
															a__:       int											=nn.Linear(1_2_0      ,       8_0			)







								def   _lowerCamelCase			( self						:  str      ,       _a						:  List[Any]      ,       _a						:  List[str]=None			):
															if labels is not None:
																						return torch.tensor(0.0      ,       device=input_ids.device			), input_ids
															else:
																						return input_ids





def 						__lowerCamelCase       (      ):
							a__:       List[str]											=HfArgumentParser((TrainingArguments,)						)
							sys.argv += ["--output_dir", "./examples"]
							a__:       int											=parser.parse_args_into_dataclasses()[0]

							logger.warning(
							    F"Process rank: {training_args.local_rank}, device: {training_args.device}, "
							    F"tpu_num_cores: {training_args.tpu_num_cores}"     , )

							# Essentially, what we want to verify in the distributed case is
							# that we get all samples back, in the right order.
							# (this is crucial for prediction for instance)
							for dataset_length in [1_001, 256, 15]:
														a__:       Optional[Any]											=DummyDataset(__magic_name__						)

														def compute_metrics(__magic_name__	:  EvalPrediction						) -> Dict:
																					a__:       List[str]											=list(range(len(__magic_name__						)						)						)
																					a__:       Union[str, Any]											=p.predictions.tolist() == sequential and p.label_ids.tolist() == sequential
																					return {"success": success}

														a__:       Union[str, Any]											=Trainer(
														    model=DummyModel()     , args=__magic_name__     , data_collator=DummyDataCollator()     , eval_dataset=__magic_name__     , compute_metrics=__magic_name__     , )
														a__:       str											=trainer.evaluate()
														logger.info(__magic_name__						)
														if metrics["eval_success"] is not True:
																					logger.error(__magic_name__						)
																					exit(1						)

														a__:       List[str]											=trainer.predict(__magic_name__						)
														logger.info(p.metrics						)
														if p.metrics["test_success"] is not True:
																					logger.error(p.metrics						)
																					exit(1						)

														a__:       Tuple											=2

														a__:       Optional[int]											=trainer.evaluate()
														logger.info(__magic_name__						)
														if metrics["eval_success"] is not True:
																					logger.error(__magic_name__						)
																					exit(1						)

														a__:       List[str]											=trainer.predict(__magic_name__						)
														logger.info(p.metrics						)
														if p.metrics["test_success"] is not True:
																					logger.error(p.metrics						)
																					exit(1						)

														a__:       List[str]											=None

							logger.info("🔥 All distributed tests successful"						)





def 						__lowerCamelCase       (      __magic_name__	:  Optional[Any]						):
							# For xla_spawn (TPUs)
							main()


if __name__ == "__main__":
		main()





