

# Copyright (c) 2021-, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

####################################################################################################

#
# Note: If when running this conversion script you're getting an exception:
#     ModuleNotFoundError: No module named 'megatron.model.enums'
# you need to tell python where to find the clone of Megatron-LM, e.g.:
#
# cd /tmp
# git clone https://github.com/NVIDIA/Megatron-LM
# PYTHONPATH=/tmp/Megatron-LM python src/transformers/models/megatron_bert/convert_megatron_bert_checkpoint.py ...
#
# if you already have it cloned elsewhere, simply adjust the path to the existing path
#
# If the training was done using a Megatron-LM fork, e.g.,
# https://github.com/microsoft/Megatron-DeepSpeed/ then chances are that you need to have that one
# in your path, i.e., /path/to/Megatron-DeepSpeed/
#

import argparse
import os
import re
import zipfile

import torch

from transformers import MegatronBertConfig







def     A_   (					_UpperCAmelCase       ,			_UpperCAmelCase       ,			_UpperCAmelCase=0					):
						# Format the message.
						if name is None:
												SCREAMING_SNAKE_CASE_:				List[str]							= None
						else:
												SCREAMING_SNAKE_CASE_:				Any							= "." * max(0       ,			spaces - 2					) + "# {:" + str(50 - spaces					) + "s}"
												SCREAMING_SNAKE_CASE_:				Optional[int]							= fmt.format(_UpperCAmelCase					)

						# Print and recurse (if needed).
						if isinstance(_UpperCAmelCase       ,			_UpperCAmelCase					):
												if msg is not None:
																		print(_UpperCAmelCase					)
												for k in val.keys():
																		recursive_print(_UpperCAmelCase       ,			val[k]       ,			spaces + 2					)
						elif isinstance(_UpperCAmelCase       ,			torch.Tensor					):
												print(_UpperCAmelCase       ,			":"       ,			val.size()					)
						else:
												print(_UpperCAmelCase       ,			":"       ,			_UpperCAmelCase					)







def     A_   (					_UpperCAmelCase       ,			_UpperCAmelCase       ,			_UpperCAmelCase       ,			_UpperCAmelCase       ,			_UpperCAmelCase					):
						# Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
						# for compatibility with later versions of NVIDIA Megatron-LM.
						# The inverse operation is performed inside Megatron-LM to read checkpoints:
						# https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
						# If param is the weight tensor of the self-attention block, the returned tensor
						# will have to be transposed one more time to be read by HuggingFace BERT.
						SCREAMING_SNAKE_CASE_:				Any							= param.size()
						if checkpoint_version == 1.0:
												# version 1.0 stores [num_heads * hidden_size * num_splits, :]
												SCREAMING_SNAKE_CASE_:				List[Any]							= (num_heads, hidden_size, num_splits) + input_shape[1:]
												SCREAMING_SNAKE_CASE_:				Tuple							= param.view(*_UpperCAmelCase					)
												SCREAMING_SNAKE_CASE_:				int							= param.transpose(0       ,			2					)
												SCREAMING_SNAKE_CASE_:				Any							= param.transpose(1       ,			2					).contiguous()
						elif checkpoint_version >= 2.0:
												# other versions store [num_heads * num_splits * hidden_size, :]
												SCREAMING_SNAKE_CASE_:				Tuple							= (num_heads, num_splits, hidden_size) + input_shape[1:]
												SCREAMING_SNAKE_CASE_:				Optional[int]							= param.view(*_UpperCAmelCase					)
												SCREAMING_SNAKE_CASE_:				Optional[Any]							= param.transpose(0       ,			1					).contiguous()
						SCREAMING_SNAKE_CASE_:				Union[str, Any]							= param.view(*_UpperCAmelCase					)
						return param







def     A_   (					_UpperCAmelCase       ,			_UpperCAmelCase       ,			_UpperCAmelCase					):
						# The converted output model.
						SCREAMING_SNAKE_CASE_:				List[Any]							= {}

						# old versions did not store training args
						SCREAMING_SNAKE_CASE_:				List[str]							= input_state_dict.get("args"       ,			_UpperCAmelCase					)
						if ds_args is not None:
												# do not make the user write a config file when the exact dimensions/sizes are already in the checkpoint
												# from pprint import pprint
												# pprint(vars(ds_args))

												SCREAMING_SNAKE_CASE_:				Optional[Any]							= ds_args.tokenizer_type
												SCREAMING_SNAKE_CASE_:				Union[str, Any]							= ds_args.padded_vocab_size
												SCREAMING_SNAKE_CASE_:				Optional[int]							= ds_args.max_position_embeddings
												SCREAMING_SNAKE_CASE_:				List[str]							= ds_args.hidden_size
												SCREAMING_SNAKE_CASE_:				str							= ds_args.num_layers
												SCREAMING_SNAKE_CASE_:				List[str]							= ds_args.num_attention_heads
												SCREAMING_SNAKE_CASE_:				Tuple							= ds_args.ffn_hidden_size if "ffn_hidden_size" in ds_args else 4 * ds_args.hidden_size
												# pprint(config)

						# The number of heads.
						SCREAMING_SNAKE_CASE_:				Optional[Any]							= config.num_attention_heads
						# The hidden_size per head.
						SCREAMING_SNAKE_CASE_:				str							= config.hidden_size // heads
						# Megatron-LM checkpoint version
						if "checkpoint_version" in input_state_dict.keys():
												SCREAMING_SNAKE_CASE_:				Union[str, Any]							= input_state_dict["checkpoint_version"]
						else:
												SCREAMING_SNAKE_CASE_:				Tuple							= 0.0

						# The model.
						SCREAMING_SNAKE_CASE_:				List[Any]							= input_state_dict["model"]
						# The language model.
						SCREAMING_SNAKE_CASE_:				int							= model["language_model"]
						# The embeddings.
						SCREAMING_SNAKE_CASE_:				Optional[Any]							= lm["embedding"]

						# The word embeddings.
						SCREAMING_SNAKE_CASE_:				Any							= embeddings["word_embeddings"]["weight"]
						# Truncate the embedding table to vocab_size rows.
						SCREAMING_SNAKE_CASE_:				List[str]							= word_embeddings[: config.vocab_size, :]
						# Store the word embeddings.
						SCREAMING_SNAKE_CASE_:				str							= word_embeddings

						# The position embeddings.
						SCREAMING_SNAKE_CASE_:				List[Any]							= embeddings["position_embeddings"]["weight"]
						assert pos_embeddings.size(0					) == config.max_position_embeddings and pos_embeddings.size(1					) == config.hidden_size
						# Store the position embeddings.
						SCREAMING_SNAKE_CASE_:				int							= pos_embeddings

						# The token-type embeddings.
						SCREAMING_SNAKE_CASE_:				Union[str, Any]							= embeddings["tokentype_embeddings"]["weight"]
						# Store the position embeddings.
						SCREAMING_SNAKE_CASE_:				Optional[Any]							= tokentype_embeddings

						# The transformer.
						SCREAMING_SNAKE_CASE_:				int							= lm["transformer"] if "transformer" in lm.keys() else lm["encoder"]

						# The regex to extract layer names.
						SCREAMING_SNAKE_CASE_:				Any							= re.compile(R"layers\.(\d+)\.([a-z0-9_.]+)\.([a-z]+)"					)

						# The simple map of names for "automated" rules.
						SCREAMING_SNAKE_CASE_:				List[Any]							= {
						    "attention.dense": ".attention.output.dense.",
						    "self_attention.dense": ".attention.output.dense.",
						    "mlp.dense_h_to_4h": ".intermediate.dense.",
						    "mlp.dense_4h_to_h": ".output.dense.",
						}

						# Keep track of the attention/query/value tensor.
						SCREAMING_SNAKE_CASE_:				Dict							= None

						# Extract the layers.
						for key, val in transformer.items():
												# Match the name.
												SCREAMING_SNAKE_CASE_:				Optional[int]							= layer_re.match(_UpperCAmelCase					)

												# Stop if that's not a layer
												if m is None:
																		break

												# The index of the layer.
												SCREAMING_SNAKE_CASE_:				Dict							= int(m.group(1					)					)
												# The name of the operation.
												SCREAMING_SNAKE_CASE_:				Optional[int]							= m.group(2					)
												# Is it a weight or a bias?
												SCREAMING_SNAKE_CASE_:				Any							= m.group(3					)

												# The name of the layer.
												SCREAMING_SNAKE_CASE_:				Dict							= f"bert.encoder.layer.{layer_idx}"

												# For layernorm(s), simply store the layer norm.
												if op_name.endswith("layernorm"					):
																		SCREAMING_SNAKE_CASE_:				Tuple							= "attention.ln" if op_name.startswith("input"					) else "ln"
																		SCREAMING_SNAKE_CASE_:				Optional[Any]							= val

												# Transpose the QKV matrix.
												elif (
												    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
												) and weight_or_bias == "weight":
																		# Make sure the QKV pointer is nil.
																		assert attention_qkv_weight is None, ""

																		SCREAMING_SNAKE_CASE_:				List[str]							= fix_query_key_value_ordering(_UpperCAmelCase       ,			_UpperCAmelCase       ,			3       ,			_UpperCAmelCase       ,			_UpperCAmelCase					)
																		# Store the tensor as we need the bias as well to interleave QKV and biases.
																		SCREAMING_SNAKE_CASE_:				Dict							= out_val

												# Transpose the bias.
												elif (
												    op_name == "attention.query_key_value" or op_name == "self_attention.query_key_value"
												) and weight_or_bias == "bias":
																		# Make sure we read the weight tensor.
																		assert attention_qkv_weight is not None, ""

																		# Split the QKV matrix into Q, K and V. Megatron stores Q,K,V interleaved.
																		SCREAMING_SNAKE_CASE_:				Optional[Any]							= attention_qkv_weight[0 * config.hidden_size : 1 * config.hidden_size, :]
																		SCREAMING_SNAKE_CASE_:				int							= attention_qkv_weight[1 * config.hidden_size : 2 * config.hidden_size, :]
																		SCREAMING_SNAKE_CASE_:				Tuple							= attention_qkv_weight[2 * config.hidden_size : 3 * config.hidden_size, :]

																		SCREAMING_SNAKE_CASE_:				int							= fix_query_key_value_ordering(_UpperCAmelCase       ,			_UpperCAmelCase       ,			3       ,			_UpperCAmelCase       ,			_UpperCAmelCase					)
																		# Split the bias.
																		SCREAMING_SNAKE_CASE_:				Union[str, Any]							= out_val[0 * config.hidden_size : 1 * config.hidden_size]
																		SCREAMING_SNAKE_CASE_:				List[str]							= out_val[1 * config.hidden_size : 2 * config.hidden_size]
																		SCREAMING_SNAKE_CASE_:				Tuple							= out_val[2 * config.hidden_size : 3 * config.hidden_size]

																		# Store.
																		SCREAMING_SNAKE_CASE_:				Optional[Any]							= q
																		SCREAMING_SNAKE_CASE_:				Union[str, Any]							= q_bias
																		SCREAMING_SNAKE_CASE_:				str							= k
																		SCREAMING_SNAKE_CASE_:				Optional[Any]							= k_bias
																		SCREAMING_SNAKE_CASE_:				Union[str, Any]							= v
																		SCREAMING_SNAKE_CASE_:				Optional[int]							= v_bias

																		# Clear the stored tensor.
																		SCREAMING_SNAKE_CASE_:				Dict							= None

												# Copy weights and biases as is.
												elif weight_or_bias in ["weight", "bias"]:
																		SCREAMING_SNAKE_CASE_:				Dict							= megatron_to_transformers[op_name]
																		SCREAMING_SNAKE_CASE_:				Tuple							= val

    # The final layernorm.
						SCREAMING_SNAKE_CASE_:				List[Any]							= transformer["final_layernorm.weight"]
						SCREAMING_SNAKE_CASE_:				Union[str, Any]							= transformer["final_layernorm.bias"]

						# The pooler.
						SCREAMING_SNAKE_CASE_:				Optional[Any]							= lm["pooler"]

						# Store the matrix and the bias.
						SCREAMING_SNAKE_CASE_:				str							= pooler["dense.weight"]
						SCREAMING_SNAKE_CASE_:				Any							= pooler["dense.bias"]

						# The LM head from Megatron (for RACE).
						SCREAMING_SNAKE_CASE_:				Any							= model["lm_head"]

						# The transform matrix.
						SCREAMING_SNAKE_CASE_:				Union[str, Any]							= lm_head["dense.weight"]
						SCREAMING_SNAKE_CASE_:				str							= lm_head["dense.bias"]

						# The transform LN.
						SCREAMING_SNAKE_CASE_:				List[Any]							= lm_head["layernorm.weight"]
						SCREAMING_SNAKE_CASE_:				Union[str, Any]							= lm_head["layernorm.bias"]

						# For the decoder, we replicate the weights.
						SCREAMING_SNAKE_CASE_:				Tuple							= word_embeddings
						SCREAMING_SNAKE_CASE_:				Union[str, Any]							= lm_head["bias"]

						# The classifier from Megatron (for MLNI).
						SCREAMING_SNAKE_CASE_:				Any							= model["binary_head"]

						# Store the classifier.
						SCREAMING_SNAKE_CASE_:				int							= binary_head["weight"]
						SCREAMING_SNAKE_CASE_:				str							= binary_head["bias"]

						# It should be done!
						return output_state_dict







def     A_   (					):
						# Create the argument parser.
						SCREAMING_SNAKE_CASE_:				Optional[Any]							= argparse.ArgumentParser()
						parser.add_argument("--print-checkpoint-structure"       ,			action="store_true"					)
						parser.add_argument("path_to_checkpoint"       ,			type=_UpperCAmelCase       ,			help="Path to the ZIP file containing the checkpoint"					)
						parser.add_argument(
						    "--config_file"       ,			default=""       ,			type=_UpperCAmelCase       ,			help="An optional config json file describing the pre-trained model."       ,			)
						SCREAMING_SNAKE_CASE_:				str							= parser.parse_args()

						# Extract the basename.
						SCREAMING_SNAKE_CASE_:				List[Any]							= os.path.dirname(args.path_to_checkpoint					)

						# Load the model.
						# the .zip is very optional, let's keep it for backward compatibility
						print(f"Extracting PyTorch state dictionary from \"{args.path_to_checkpoint}\""					)
						if args.path_to_checkpoint.endswith(".zip"					):
												with zipfile.ZipFile(args.path_to_checkpoint       ,			"r"					) as checkpoint:
																		with checkpoint.open("release/mp_rank_00/model_optim_rng.pt"					) as pytorch_dict:
																								SCREAMING_SNAKE_CASE_:				Dict							= torch.load(_UpperCAmelCase       ,			map_location="cpu"					)
						else:
												SCREAMING_SNAKE_CASE_:				str							= torch.load(args.path_to_checkpoint       ,			map_location="cpu"					)

						if args.config_file == "":
												# Default config of megatron-bert 345m
												SCREAMING_SNAKE_CASE_:				str							= MegatronBertConfig()

												# different megatron-bert-*-345m models have different vocab sizes, so override the default
												# config (which is for megatron-bert-cased-345m) with the actual vocab dimension
												SCREAMING_SNAKE_CASE_:				List[Any]							= input_state_dict["model"]["lm_head"]["bias"].numel()
						else:
												SCREAMING_SNAKE_CASE_:				str							= MegatronBertConfig.from_json_file(args.config_file					)

						# Convert.
						print("Converting"					)
						SCREAMING_SNAKE_CASE_:				List[Any]							= convert_megatron_checkpoint(_UpperCAmelCase       ,			_UpperCAmelCase       ,			_UpperCAmelCase					)

						# Print the structure of converted state dict.
						if args.print_checkpoint_structure:
												recursive_print(_UpperCAmelCase       ,			_UpperCAmelCase					)

						# Store the config to file.
						print("Saving config"					)
						config.save_pretrained(_UpperCAmelCase					)

						# Store the state_dict to file.
						SCREAMING_SNAKE_CASE_:				Tuple							= os.path.join(_UpperCAmelCase       ,			"pytorch_model.bin"					)
						print(f"Saving checkpoint to \"{output_checkpoint_file}\""					)
						torch.save(_UpperCAmelCase       ,			_UpperCAmelCase					)


####################################################################################################

if __name__ == "__main__":
						main()

####################################################################################################









import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


lowerCAmelCase							:						Tuple     			=		datasets.utils.logging.get_logger(__name__)



@dataclass
class 						__lowercase       (				datasets.BuilderConfig			):




			"""simple docstring"""







			_UpperCAmelCase     :							Optional[datasets.Features]    							=  None
			_UpperCAmelCase     :							str    							=  "utf-8"
			_UpperCAmelCase     :							Optional[str]    							=  None
			_UpperCAmelCase     :							int    							=  10 << 20  # 10MB
			_UpperCAmelCase     :							bool    							=  False
			_UpperCAmelCase     :							str    							=  "line"







class 						__lowercase       (				datasets.ArrowBasedBuilder			):




			"""simple docstring"""







			_UpperCAmelCase     :							List[Any]    							=  TextConfig



			def  _SCREAMING_SNAKE_CASE    (				self						:  Optional[Any]):
									return datasets.DatasetInfo(features=self.config.features)



			def  _SCREAMING_SNAKE_CASE    (				self						:  List[Any]     ,		lowerCAmelCase__						:  Tuple):
									if not self.config.data_files:
															raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}")
									SCREAMING_SNAKE_CASE_:				Dict							= dl_manager.download_and_extract(self.config.data_files)
									if isinstance(lowerCAmelCase__     ,		(str, list, tuple)):
															SCREAMING_SNAKE_CASE_:				Dict							= data_files
															if isinstance(lowerCAmelCase__     ,		lowerCAmelCase__):
																					SCREAMING_SNAKE_CASE_:				Dict							= [files]
															SCREAMING_SNAKE_CASE_:				List[str]							= [dl_manager.iter_files(lowerCAmelCase__) for file in files]
															return [datasets.SplitGenerator(name=datasets.Split.TRAIN     ,		gen_kwargs={"files": files})]
									SCREAMING_SNAKE_CASE_:				Any							= []
									for split_name, files in data_files.items():
															if isinstance(lowerCAmelCase__     ,		lowerCAmelCase__):
																					SCREAMING_SNAKE_CASE_:				List[Any]							= [files]
															SCREAMING_SNAKE_CASE_:				Union[str, Any]							= [dl_manager.iter_files(lowerCAmelCase__) for file in files]
															splits.append(datasets.SplitGenerator(name=lowerCAmelCase__     ,		gen_kwargs={"files": files}))
									return splits



			def  _SCREAMING_SNAKE_CASE    (				self						:  Optional[int]     ,		lowerCAmelCase__						:  pa.Table):
									if self.config.features is not None:
															SCREAMING_SNAKE_CASE_:				Optional[int]							= self.config.features.arrow_schema
															if all(not require_storage_cast(lowerCAmelCase__) for feature in self.config.features.values()):
																					# cheaper cast
																					SCREAMING_SNAKE_CASE_:				List[str]							= pa_table.cast(lowerCAmelCase__)
															else:
																					# more expensive cast; allows str <-> int/float or str to Audio for example
																					SCREAMING_SNAKE_CASE_:				List[Any]							= table_cast(lowerCAmelCase__     ,		lowerCAmelCase__)
															return pa_table
									else:
															return pa_table.cast(pa.schema({"text": pa.string()}))




			def  _SCREAMING_SNAKE_CASE    (				self						:  int     ,		lowerCAmelCase__						:  Dict):
									SCREAMING_SNAKE_CASE_:				str							= list(self.config.features) if self.config.features is not None else ["text"]
									for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCAmelCase__)):
															# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
															with open(lowerCAmelCase__     ,		encoding=self.config.encoding     ,		errors=self.config.errors) as f:
																					if self.config.sample_by == "line":
																											SCREAMING_SNAKE_CASE_:				Any							= 0
																											while True:
																																	SCREAMING_SNAKE_CASE_:				str							= f.read(self.config.chunksize)
																																	if not batch:
																																							break
																																	batch += f.readline()  # finish current line
																																	# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																																	SCREAMING_SNAKE_CASE_:				Optional[Any]							= StringIO(lowerCAmelCase__).readlines()
																																	if not self.config.keep_linebreaks:
																																							SCREAMING_SNAKE_CASE_:				Optional[Any]							= [line.rstrip("\n") for line in batch]
																																	SCREAMING_SNAKE_CASE_:				Any							= pa.Table.from_arrays([pa.array(lowerCAmelCase__)]     ,		names=lowerCAmelCase__)
																																	# Uncomment for debugging (will print the Arrow table size and elements)
																																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																	yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase__)
																																	batch_idx += 1
																					elif self.config.sample_by == "paragraph":
																											SCREAMING_SNAKE_CASE_:				List[str]							= 0
																											SCREAMING_SNAKE_CASE_:				Tuple							= ""
																											while True:
																																	SCREAMING_SNAKE_CASE_:				Dict							= f.read(self.config.chunksize)
																																	if not new_batch:
																																							break
																																	batch += new_batch
																																	batch += f.readline()  # finish current line
																																	SCREAMING_SNAKE_CASE_:				List[str]							= batch.split("\n\n")
																																	SCREAMING_SNAKE_CASE_:				Optional[int]							= pa.Table.from_arrays(
																																	    [pa.array([example for example in batch[:-1] if example])]     ,		names=lowerCAmelCase__)
																																	# Uncomment for debugging (will print the Arrow table size and elements)
																																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																	yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase__)
																																	batch_idx += 1
																																	SCREAMING_SNAKE_CASE_:				Optional[Any]							= batch[-1]
																											if batch:
																																	SCREAMING_SNAKE_CASE_:				int							= pa.Table.from_arrays([pa.array([batch])]     ,		names=lowerCAmelCase__)
																																	yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase__)
																					elif self.config.sample_by == "document":
																											SCREAMING_SNAKE_CASE_:				List[Any]							= f.read()
																											SCREAMING_SNAKE_CASE_:				Optional[int]							= pa.Table.from_arrays([pa.array([text])]     ,		names=lowerCAmelCase__)
																											yield file_idx, self._cast_table(lowerCAmelCase__)






