



import argparse

# utilities to import the model weights and config file
import os
import pickle as pkl

# PyTorch + new model classes
import torch
from torch import nn

from transformers import AutoTokenizer, MegaConfig, MegaForMaskedLM


# import the EncoderLayer class used to pretrain
# !! NOTE !! this requires the version of fairseq that is built when you install the Mega source
try:
		from fairseq.modules.mega_layer import MegaEncoderLayer
except ImportError:
		raise ImportError('''You need to install the version of fairseq from the Mega repo!''')


class 	lowerCamelCase__ (		nn.Module   ):


						def __init__( self						:  str      ,       _a						:  Optional[Any]      ,       _a						:  Tuple      ,       _a						:  Optional[Any]			):
													super().__init__()
													a__:       int											=mega_args
													a__:       Any											=nn.Embedding(a_      ,       self.mega_args.encoder_embed_dim			)
													a__:       Union[str, Any]											=nn.ModuleList([MegaEncoderLayer(self.mega_args			) for _ in range(a_			)]			)
													a__:       Any											=depth







						def   _lowerCamelCase			( self						:  Optional[Any]      ,       _a						:  List[str]      ,       _a						:  Any      ,       _a						:  Any=True      ,       _a						:  int=0			):



													if batch_first:
																				a__:       Dict											=input_ids.T

													# to make things more confusing, Mega expects the attention mask to
													# be (batch, time), but with values of 0 (normal token) and 1 (ignore token)
													# which is the opposite of what HF returns
													if ignore_mask_value == 0:
																				a__:       Union[str, Any]											=1 - attention_mask

													# get token embeddings from IDs
													a__:       str											=self.embedding_layer(a_			)

													# pass through the Mega layers
													# input is (time, batch, encoder dim) and output is the same
													for encoder in self.encoders:
																				a__:       Optional[int]											=encoder(a_      ,       a_			)

													# return according to the shape specified
													if batch_first:
																				# (T, B, H) --> (B, T, H)
																				return torch.transpose(a_      ,       0      ,       1			)
													else:
																				return embeds


class 	lowerCamelCase__ (		nn.Module   ):


						def __init__( self						:  str      ,       _a						:  List[Any]      ,       _a						:  Any      ,       _a						:  str			):
													super().__init__()
													a__:       Tuple											=MegaLM(a_      ,       a_      ,       a_			)
													a__:       List[Any]											=nn.Linear(mega_args.encoder_embed_dim      ,       a_			)
													a__:       Optional[Any]											=nn.Dropout(p=0.1			)







						def   _lowerCamelCase			( self						:  Dict      ,       _a						:  Tuple      ,       _a						:  str      ,       _a						:  int=True      ,       _a						:  int=0			):
													a__:       Optional[Any]											=self.mega(a_      ,       a_      ,       a_      ,       a_			)
													return self.mlm_head(self.dropout(a_			)			)





def 						__lowerCamelCase       (      __magic_name__	:  Optional[int]     , __magic_name__	:  List[str]     , __magic_name__	:  List[Any]						):







							with open(os.path.join(_UpperCamelCase     , "model_args.pkl"						)     , "rb"						) as f:
														a__:       Union[str, Any]											=pkl.load(_UpperCamelCase						)

							# load the original encoder
							a__:       Optional[int]											=OriginalMegaForMaskedLM(**_UpperCamelCase						).eval()

							# load its weights
							print(
							    "Original Mega encoder:"     , original_mlm.mega.load_state_dict(
							        torch.load(os.path.join(_UpperCamelCase     , "encoder_weights.pt"						)     , map_location="cpu"						)						)     , )
							print(
							    "Original Mega MLM layer:"     , original_mlm.mlm_head.load_state_dict(
							        torch.load(os.path.join(_UpperCamelCase     , "mlm_head_weights.pt"						)     , map_location="cpu"						)						)     , )

							# create a new config from the old one
							a__:       List[Any]											=MegaConfig(
							    num_hidden_layers=mega_original_args["depth"]     , vocab_size=mega_original_args["vocab_size"]     , hidden_size=mega_original_args["mega_args"].encoder_embed_dim     , shared_representation_size=mega_original_args["mega_args"].encoder_z_dim     , intermediate_size=mega_original_args["mega_args"].encoder_hidden_dim     , ema_projection_size=mega_original_args["mega_args"].encoder_n_dim     , dropout_prob=mega_original_args["mega_args"].dropout     , attention_probs_dropout_prob=mega_original_args["mega_args"].attention_dropout     , hidden_dropout_prob=mega_original_args["mega_args"].hidden_dropout     , activation=mega_original_args["mega_args"].activation_fn     , attention_activation=mega_original_args["mega_args"].attention_activation_fn     , bidirectional=mega_original_args["mega_args"].bidirectional     , use_chunking=mega_original_args["mega_args"].encoder_chunk_size > 0     , chunk_size=mega_original_args["mega_args"].encoder_chunk_size     , truncation=mega_original_args["mega_args"].truncation_length     , normalization_type=mega_original_args["mega_args"].normalization_type     , normalize_before_mega=_UpperCamelCase     , norm_affine=_UpperCamelCase     , use_feature_dropout=mega_original_args["mega_args"].feature_dropout     , relative_positional_bias=mega_original_args["mega_args"].rel_pos_bias     , max_positions=mega_original_args["mega_args"].max_source_positions     , nffn_hidden_size=mega_original_args["mega_args"].encoder_ffn_embed_dim     , normalize_before_ffn=mega_original_args["mega_args"].normalize_before     , nffn_activation_dropout_prob=0.0     , add_token_type_embeddings=_UpperCamelCase     , add_lm_hidden_dense_layer=_UpperCamelCase     , )

							a__:       str											=MegaForMaskedLM(_UpperCamelCase						).eval()

							# the originl checkpoint just uses nn.Embedding for the word embeddings
							# we use a wrapper module for embeddings to add support for positional embeddings
							a__:       int											=original_mlm.mega.embedding_layer.weight

							# modify the state dictionary of the original checkpoint to account for naming issues in the Hugging Face
							# ecosystem -- any names containing "beta" or "gamma" aren't safe to use and are renamed upon _load_pretrained,
							# also renaming previously confusing parameter names
							a__:       List[Any]											=original_mlm.mega.encoders.state_dict()
							a__:       List[str]											={}
							for module_name in original_state_dict.keys():
														a__:       Dict											=None
														# have to handle gamma, beta, and alpha differently due to their use
														# in multiple modules within the original repository;
														# beta is used in EMA, MovingAverageGatedAttention, and RotaryRelativePositionalBias, and must be renamed due to flax/tf weights
														# the EMA sublayer was renamed from "move" to "ema_gate" for readability, so that is also done here
														if "beta" in module_name:
																					# EMA sub-layers were always called "move" in the original repo
																					if "move.beta" in module_name:
																												a__:       Any											=module_name.replace("move.beta"     , "ema_gate.ema_expansion_matrix"						)
																					elif "mega_layer.beta" in module_name:
																												a__:       Tuple											=module_name.replace("beta"     , "qk_bias"						)
																					else:
																												a__:       Optional[Any]											=module_name.replace("beta"     , "b_param"						)
        # beta is used in EMA and MovingAverageGatedAttention, and must be renamed due to flax/tf weights
														elif "gamma" in module_name:
																					if "move.gamma" in module_name:
																												a__:       int											=module_name.replace("move.gamma"     , "ema_gate.kernel_projection_matrix"						)
																					elif "mega_layer.gamma" in module_name:
																												a__:       Tuple											=module_name.replace("gamma"     , "qk_weight"						)
																					else:
																												a__:       List[str]											=module_name.replace("gamma"     , "g_param"						)
        # alpha is used in EMA and positional bias; renaming to improve readability
														elif "move.alpha" in module_name:
																					a__:       Optional[int]											=module_name.replace("move.alpha"     , "ema_gate.decay_factor"						)
														# delta is only used in EMA; renaming to improve readability
														elif "move.delta" in module_name:
																					a__:       Optional[int]											=module_name.replace("move.delta"     , "ema_gate.damping_factor"						)
														# omega is only used in EMA; renaming to improve readability
														elif "omega" in module_name:
																					a__:       Tuple											=module_name.replace("move.omega"     , "ema_gate.residual_weight"						)

														if new_module_name:
																					a__:       Tuple											=new_module_name

							if len(_UpperCamelCase						) != 0:
														print(F"Renaming these keys: {updated_keys.keys()}"						)
							else:
														print("No need to rename state dict entries"						)
							for old, new in updated_keys.items():
														a__:       Optional[Any]											=original_state_dict.pop(_UpperCamelCase						)

							# now attempt to load the state dictionary with updated names
							# note that we now call it `mega.layers` instead of `mega.encoders` due to hugging face style
							print("HF Mega encoder:"     , hf_mlm.mega.layers.load_state_dict(_UpperCamelCase						)						)

							# load the MLM head weights directly
							print(
							    "HF Mega MLM layer:"     , hf_mlm.mlm_head.load_state_dict(
							        torch.load(os.path.join(_UpperCamelCase     , "mlm_head_weights.pt"						)     , map_location="cpu"						)						)     , )

							# test on a randomly generated input sequence
							a__:       str											=torch.randint(0     , hf_config.vocab_size     , size=(4, 256)						)
							a__:       Tuple											=torch.ones_like(_UpperCamelCase						)
							# mask a few tokens to make sure masking is applied appropriately :)
							a__:       Any											=0

							# run forward passes
							a__:       Optional[int]											=original_mlm(_UpperCamelCase     , _UpperCamelCase     , batch_first=_UpperCamelCase     , ignore_mask_value=0						)
							a__:       Optional[Any]											=hf_mlm(_UpperCamelCase     , _UpperCamelCase						)[0]

							# print shapes and diff
							print(F"original output {original_output.shape}"						)
							print(F"hf output {hf_output.shape}"						)
							print(F"max diff: {(original_output - hf_output).max()}"						)  # 0.0
							a__:       int											=torch.allclose(_UpperCamelCase     , _UpperCamelCase     , atol=1e-3						)

							if success:
														print("Yay!"						)
														hf_mlm.save_pretrained(_UpperCamelCase						)
							else:
														raise RuntimeError(F"Something\'s broken :(\nOriginal:\n{original_output}\n\nHF\n{hf_output}\n{hf_mlm}"						)

							if includes_tokenizer:
														print("Transferring tokenizer"						)
														a__:       List[str]											=AutoTokenizer.from_pretrained(_UpperCamelCase						)
														tokenizer.save_pretrained(_UpperCamelCase						)


if __name__ == "__main__":
		__UpperCAmelCase						  =     argparse.ArgumentParser()

		parser.add_argument(
		    '''--pretrained_checkpoint_path''',
		    default=None,
		    type=str,
		    required=True,
		    help='''Point to the directory containing your model weights using the official Mega repo''',
		)

		parser.add_argument(
		    '''--output_path''', default=None, type=str, required=True, help='''Location to save the Hugging Face version'''
		)

		parser.add_argument(
		    '''--includes_tokenizer''',
		    action='''store_true''',
		    help='''Use this flag if there is a Hugging Face tokenizer in the original checkpoint repo''',
		)

		__UpperCAmelCase						  =     parser.parse_args()

		convert_checkpoint_to_huggingface(args.pretrained_checkpoint_path, args.output_path, args.includes_tokenizer)










# This test is meant to be run in on an instance with TPUs like this:
#
#   python examples/pytorch/xla_spawn.py --num_cores=8 tests/test_trainer_tpu.py
#
# Replace 8 with the number of TPU cores you have.
#

import sys
from typing import Dict

from transformers import EvalPrediction, HfArgumentParser, TrainingArguments, is_torch_available
from transformers.utils import logging


__UpperCAmelCase						  =     logging.get_logger(__name__)


if is_torch_available():
		import torch
		from torch import nn
		from torch.utils.data import Dataset

		from transformers import Trainer


		class 	lowerCamelCase__ (		_a   ):


								def __init__( self						:  Any      ,       _a						:  int = 1_0_1			):
															a__:       Dict											=length


								def __len__( self						:  Optional[Any]			):
															return self.length







								def __getitem__( self						:  Dict      ,       _a						:  int			):
															return i


		class 	lowerCamelCase__ :


								def __call__( self						:  Any      ,       _a						:  List[str]			):
															return {"input_ids": torch.tensor(_a			), "labels": torch.tensor(_a			)}


		class 	lowerCamelCase__ (		nn.Module   ):


								def __init__( self						:  Tuple			):
															super().__init__()
															# Add some (unused) params otherwise DDP will complain.
															a__:       int											=nn.Linear(1_2_0      ,       8_0			)







								def   _lowerCamelCase			( self						:  str      ,       _a						:  List[Any]      ,       _a						:  List[str]=None			):
															if labels is not None:
																						return torch.tensor(0.0      ,       device=input_ids.device			), input_ids
															else:
																						return input_ids





def 						__lowerCamelCase       (      ):
							a__:       List[str]											=HfArgumentParser((TrainingArguments,)						)
							sys.argv += ["--output_dir", "./examples"]
							a__:       int											=parser.parse_args_into_dataclasses()[0]

							logger.warning(
							    F"Process rank: {training_args.local_rank}, device: {training_args.device}, "
							    F"tpu_num_cores: {training_args.tpu_num_cores}"     , )

							# Essentially, what we want to verify in the distributed case is
							# that we get all samples back, in the right order.
							# (this is crucial for prediction for instance)
							for dataset_length in [1_001, 256, 15]:
														a__:       Optional[Any]											=DummyDataset(__magic_name__						)

														def compute_metrics(__magic_name__	:  EvalPrediction						) -> Dict:
																					a__:       List[str]											=list(range(len(__magic_name__						)						)						)
																					a__:       Union[str, Any]											=p.predictions.tolist() == sequential and p.label_ids.tolist() == sequential
																					return {"success": success}

														a__:       Union[str, Any]											=Trainer(
														    model=DummyModel()     , args=__magic_name__     , data_collator=DummyDataCollator()     , eval_dataset=__magic_name__     , compute_metrics=__magic_name__     , )
														a__:       str											=trainer.evaluate()
														logger.info(__magic_name__						)
														if metrics["eval_success"] is not True:
																					logger.error(__magic_name__						)
																					exit(1						)

														a__:       List[str]											=trainer.predict(__magic_name__						)
														logger.info(p.metrics						)
														if p.metrics["test_success"] is not True:
																					logger.error(p.metrics						)
																					exit(1						)

														a__:       Tuple											=2

														a__:       Optional[int]											=trainer.evaluate()
														logger.info(__magic_name__						)
														if metrics["eval_success"] is not True:
																					logger.error(__magic_name__						)
																					exit(1						)

														a__:       List[str]											=trainer.predict(__magic_name__						)
														logger.info(p.metrics						)
														if p.metrics["test_success"] is not True:
																					logger.error(p.metrics						)
																					exit(1						)

														a__:       List[str]											=None

							logger.info("🔥 All distributed tests successful"						)





def 						__lowerCamelCase       (      __magic_name__	:  Optional[Any]						):
							# For xla_spawn (TPUs)
							main()


if __name__ == "__main__":
		main()





