

"""simple docstring"""


import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig






def 					a__					(					__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE=0				)			->			Optional[Any]:
		# Format the message.
		if name is None:
				__lowerCAmelCase: Any					   =    None
		else:
				__lowerCAmelCase: Tuple					   =    "." * max(0      ,				spaces - 2				) + "# {:" + str(5_0 - spaces				) + "s}"
				__lowerCAmelCase: Tuple					   =    fmt.format(__SCREAMING_SNAKE_CASE				)

		# Print and recurse (if needed).
		if isinstance(__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE				):
				if msg is not None:
						print(__SCREAMING_SNAKE_CASE				)
				for k in val.keys():
						recursive_print(__SCREAMING_SNAKE_CASE      ,				val[k]      ,				spaces + 2				)
		elif isinstance(__SCREAMING_SNAKE_CASE      ,				torch.Tensor				):
				print(__SCREAMING_SNAKE_CASE      ,				":"      ,				val.size()				)
		else:
				print(__SCREAMING_SNAKE_CASE      ,				":"      ,				__SCREAMING_SNAKE_CASE				)






def 					a__					(					__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE				)			->			Optional[int]:
		# Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
		# for compatibility with later versions of NVIDIA Megatron-LM.
		# The inverse operation is performed inside Megatron-LM to read checkpoints:
		# https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
		# If param is the weight tensor of the self-attention block, the returned tensor
		# will have to be transposed one more time to be read by HuggingFace GPT2.
		__lowerCAmelCase: Union[str, Any]					   =    param.size()
		# other versions store [num_heads * num_splits * hidden_size, :]
		__lowerCAmelCase: Dict					   =    (num_heads, num_splits, hidden_size) + input_shape[1:]
		__lowerCAmelCase: str					   =    param.view(*__SCREAMING_SNAKE_CASE				)
		__lowerCAmelCase: Any					   =    param.transpose(0      ,				1				).contiguous()
		__lowerCAmelCase: str					   =    param.view(*__SCREAMING_SNAKE_CASE				)
		return param






def 					a__					(					__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE				)			->			Dict:
		__lowerCAmelCase: Tuple					   =    config.n_positions
		__lowerCAmelCase: Tuple					   =    config.n_layer
		__lowerCAmelCase: Tuple					   =    config.vocab_size
		__lowerCAmelCase: int					   =    config.n_head
		__lowerCAmelCase: Union[str, Any]					   =    config.n_embd // config.n_head

		__lowerCAmelCase: str					   =    sd_megatron["model.language_model.embedding.word_embeddings.weight"][:vocab_size, :]
		__lowerCAmelCase: Any					   =    {
		    "transformer.wte.weight": word_embeddings,
		    "transformer.wpe.weight": sd_megatron["model.language_model.embedding.position_embeddings.weight"],
		    "transformer.ln_f.weight": sd_megatron["model.language_model.encoder.final_layernorm.weight"],
		    "transformer.ln_f.bias": sd_megatron["model.language_model.encoder.final_layernorm.bias"],
		}

		__lowerCAmelCase: int					   =    "model.language_model.encoder.layers."
		for i in range(__SCREAMING_SNAKE_CASE				):
				__lowerCAmelCase: Optional[Any]					   =    torch.tril(torch.ones((n_positions, n_positions)      ,				dtype=torch.bool				)				)
				__lowerCAmelCase: Dict					   =    causal_mask.view(1      ,				1      ,				__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE				)
				__lowerCAmelCase: Dict					   =    causal_mask
				__lowerCAmelCase: str					   =    torch.tensor(-1E4      ,				dtype=torch.bfloataa				)

				__lowerCAmelCase: List[Any]					   =    sd_megatron[F"{pf}{i}.input_layernorm.weight"]
				__lowerCAmelCase: Dict					   =    sd_megatron[F"{pf}{i}.input_layernorm.bias"]

				__lowerCAmelCase: Optional[int]					   =    sd_megatron[F"{pf}{i}.self_attention.query_key_value.weight"]
				__lowerCAmelCase: Tuple					   =    fix_query_key_value_ordering(__SCREAMING_SNAKE_CASE      ,				3      ,				__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE				)
				__lowerCAmelCase: List[str]					   =    vala.transpose(0      ,				1				).contiguous()

				__lowerCAmelCase: Tuple					   =    sd_megatron[F"{pf}{i}.self_attention.query_key_value.bias"]
				__lowerCAmelCase: Any					   =    fix_query_key_value_ordering(__SCREAMING_SNAKE_CASE      ,				3      ,				__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE				)
				__lowerCAmelCase: List[Any]					   =    vala

				__lowerCAmelCase: Any					   =    sd_megatron[F"{pf}{i}.self_attention.dense.weight"].transpose(
				    0      ,				1				)
				__lowerCAmelCase: str					   =    sd_megatron[F"{pf}{i}.self_attention.dense.bias"]
				__lowerCAmelCase: Optional[int]					   =    sd_megatron[F"{pf}{i}.post_attention_layernorm.weight"]
				__lowerCAmelCase: Union[str, Any]					   =    sd_megatron[F"{pf}{i}.post_attention_layernorm.bias"]
				__lowerCAmelCase: List[Any]					   =    sd_megatron[F"{pf}{i}.mlp.dense_h_to_4h.weight"].transpose(0      ,				1				)
				__lowerCAmelCase: Tuple					   =    sd_megatron[F"{pf}{i}.mlp.dense_h_to_4h.bias"]
				__lowerCAmelCase: Any					   =    sd_megatron[F"{pf}{i}.mlp.dense_4h_to_h.weight"].transpose(
				    0      ,				1				)
				__lowerCAmelCase: Optional[int]					   =    sd_megatron[F"{pf}{i}.mlp.dense_4h_to_h.bias"]

		# For LM head, transformers' wants the matrix to weight embeddings.
		__lowerCAmelCase: List[Any]					   =    word_embeddings

		return sd_hf






def 					a__					(					__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE				)			->			Dict:
		__lowerCAmelCase: List[str]					   =    6_4_0_0_0
		__lowerCAmelCase: str					   =    config_megatron["encoder_seq_length"]
		__lowerCAmelCase: str					   =    config_megatron["hidden_size"]
		__lowerCAmelCase: Optional[int]					   =    config_megatron["num_layers"]
		__lowerCAmelCase: Optional[int]					   =    config_megatron["num_attention_heads"]
		__lowerCAmelCase: Optional[int]					   =    config_megatron["ffn_hidden_size"]
		__lowerCAmelCase: Dict					   =    "gelu"
		__lowerCAmelCase: str					   =    0.1
		__lowerCAmelCase: Optional[Any]					   =    0.1
		__lowerCAmelCase: str					   =    0.1
		__lowerCAmelCase: Optional[int]					   =    config_megatron["layernorm_epsilon"]  # 1e-5
		__lowerCAmelCase: str					   =    config_megatron["init_method_std"]  # 0.02
		__lowerCAmelCase: Any					   =    config_megatron["apply_query_key_layer_scaling"]  # True
		__lowerCAmelCase: str					   =    True
		__lowerCAmelCase: Optional[Any]					   =    True

		# This identifies the 6.7B (7B) model which uses a different tokenizer
		if config_megatron["hidden_size"] == 4_0_9_6:
				__lowerCAmelCase: int					   =    1  # <|endoftext|>
				__lowerCAmelCase: Tuple					   =    1  # <|endoftext|>
				__lowerCAmelCase: int					   =    0  # <unk>
		else:
				__lowerCAmelCase: List[Any]					   =    2  # <s>
				__lowerCAmelCase: Optional[Any]					   =    3  # <|endoftext|>
				__lowerCAmelCase: Dict					   =    0  # <pad>

		return config_hf






def 					a__					(					__SCREAMING_SNAKE_CASE				)			->			Tuple:
		print(__SCREAMING_SNAKE_CASE				)

		__lowerCAmelCase: Tuple					   =    args.checkpoint_path
		__lowerCAmelCase: List[Any]					   =    args.save_path
		if isfile(__SCREAMING_SNAKE_CASE				):
				raise FileNotFoundError(F"ERROR! could not find file {checkpoint_path}"				)

		# Load the model.
		__lowerCAmelCase: List[Any]					   =    torch.load(__SCREAMING_SNAKE_CASE      ,				map_location="cpu"				)

		# Load the config.
		__lowerCAmelCase: Union[str, Any]					   =    checkpoint["hyper_parameters"]["cfg"]
		__lowerCAmelCase: Dict					   =    GPTaConfig()
		__lowerCAmelCase: Any					   =    copy_config(config_hf=__SCREAMING_SNAKE_CASE      ,				config_megatron=__SCREAMING_SNAKE_CASE				)
		__lowerCAmelCase: int					   =    ["GPT2LMHeadModel"]

		__lowerCAmelCase: Any					   =    checkpoint["state_dict"]

		# Convert.
		print("Converting"				)
		__lowerCAmelCase: List[str]					   =    convert_megatron_checkpoint(__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE				)

		# Print the structure of converted state dict.
		if args.print_checkpoint_structure:
				recursive_print(__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE				)

		__lowerCAmelCase: int					   =    "GPTSw3Tokenizer"

		# Store the config to file.
		print("Saving config"				)
		config_hf.save_pretrained(__SCREAMING_SNAKE_CASE				)

		# Store the state_dict to file.
		__lowerCAmelCase: Tuple					   =    os.path.join(__SCREAMING_SNAKE_CASE      ,				"pytorch_model.bin"				)
		print(F"Saving checkpoint to \"{output_checkpoint_file}\""				)
		torch.save(__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE				)


if __name__ == "__main__":
			__A		     =    argparse.ArgumentParser()
			parser.add_argument(
			    "--checkpoint_path",
			    type=str,
			    required=True,
			    help="e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000",
			)
			parser.add_argument("--save_path", type=str, required=True, help="e.g. /home/user/gpt-sw3/hf")
			parser.add_argument("--print-checkpoint-structure", action="store_true")
			__A		     =    parser.parse_args()
			main(_args)





"""simple docstring"""


import random
import sys

__A		     =    "ABCDEFGHIJKLMNOPQRSTUVWXYZ"






def 					a__					(					)			->			None:
		__lowerCAmelCase: int					   =    input("Enter message: "				)
		__lowerCAmelCase: Any					   =    "LFWOAYUISVKMNXPBDCRJTQEGHZ"
		__lowerCAmelCase: int					   =    input("Encrypt/Decrypt [e/d]: "				)

		check_valid_key(__SCREAMING_SNAKE_CASE				)

		if resp.lower().startswith("e"				):
				__lowerCAmelCase: Tuple					   =    "encrypt"
				__lowerCAmelCase: int					   =    encrypt_message(__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE				)
		elif resp.lower().startswith("d"				):
				__lowerCAmelCase: str					   =    "decrypt"
				__lowerCAmelCase: List[str]					   =    decrypt_message(__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE				)

		print(F"\n{mode.title()}ion: \n{translated}"				)






def 					a__					(					__SCREAMING_SNAKE_CASE				)			->			None:
		__lowerCAmelCase: List[Any]					   =    list(__SCREAMING_SNAKE_CASE				)
		__lowerCAmelCase: Optional[int]					   =    list(__SCREAMING_SNAKE_CASE				)
		key_list.sort()
		letters_list.sort()

		if key_list != letters_list:
				sys.exit("Error in the key or symbol set."				)






def 					a__					(					__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE				)			->			str:
		return translate_message(__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE      ,				"encrypt"				)






def 					a__					(					__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE				)			->			str:
		return translate_message(__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE      ,				"decrypt"				)






def 					a__					(					__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE      ,				__SCREAMING_SNAKE_CASE				)			->			str:
		__lowerCAmelCase: Union[str, Any]					   =    ""
		__lowerCAmelCase: List[str]					   =    LETTERS
		__lowerCAmelCase: Optional[int]					   =    key

		if mode == "decrypt":
				__lowerCAmelCase   ,				__lowerCAmelCase: Optional[int]					   =    chars_b, chars_a

		for symbol in message:
				if symbol.upper() in chars_a:
						__lowerCAmelCase: Union[str, Any]					   =    chars_a.find(symbol.upper()				)
						if symbol.isupper():
								translated += chars_b[sym_index].upper()
						else:
								translated += chars_b[sym_index].lower()
				else:
						translated += symbol

		return translated






def 					a__					(					)			->			str:
		__lowerCAmelCase: int					   =    list(__SCREAMING_SNAKE_CASE				)
		random.shuffle(__SCREAMING_SNAKE_CASE				)
		return "".join(__SCREAMING_SNAKE_CASE				)


if __name__ == "__main__":
			main()


