import argparse
import os
from os.path import isfile

import torch

from transformers import GPTaConfig






def 					SCREAMING_SNAKE_CASE_			(		lowerCAmelCase_: Any ,	lowerCAmelCase_: Tuple ,	lowerCAmelCase_: List[str]=0     ):
      # Format the message.
      if name is None:
            snake_case_     :			Union[str, Any]       =       None
      else:
            snake_case_     :			List[Any]       =       "." * max(0 ,	spaces - 2     ) + "# {:" + str(5_0 - spaces     ) + "s}"
            snake_case_     :			Optional[int]       =       fmt.format(lowerCAmelCase_     )

      # Print and recurse (if needed).
      if isinstance(lowerCAmelCase_ ,	lowerCAmelCase_     ):
            if msg is not None:
                  print(lowerCAmelCase_     )
            for k in val.keys():
                  recursive_print(lowerCAmelCase_ ,	val[k] ,	spaces + 2     )
      elif isinstance(lowerCAmelCase_ ,	torch.Tensor     ):
            print(lowerCAmelCase_ ,	":" ,	val.size()     )
      else:
            print(lowerCAmelCase_ ,	":" ,	lowerCAmelCase_     )






def 					SCREAMING_SNAKE_CASE_			(		lowerCAmelCase_: int ,	lowerCAmelCase_: Dict ,	lowerCAmelCase_: Tuple ,	lowerCAmelCase_: List[Any]     ):
      # Permutes layout of param tensor to [num_splits * num_heads * hidden_size, :]
      # for compatibility with later versions of NVIDIA Megatron-LM.
      # The inverse operation is performed inside Megatron-LM to read checkpoints:
      # https://github.com/NVIDIA/Megatron-LM/blob/v2.4/megatron/checkpointing.py#L209
      # If param is the weight tensor of the self-attention block, the returned tensor
      # will have to be transposed one more time to be read by HuggingFace GPT2.
      snake_case_     :			Tuple       =       param.size()
      # other versions store [num_heads * num_splits * hidden_size, :]
      snake_case_     :			Tuple       =       (num_heads, num_splits, hidden_size) + input_shape[1:]
      snake_case_     :			int       =       param.view(*lowerCAmelCase_     )
      snake_case_     :			Optional[Any]       =       param.transpose(0 ,	1     ).contiguous()
      snake_case_     :			str       =       param.view(*lowerCAmelCase_     )
      return param






def 					SCREAMING_SNAKE_CASE_			(		lowerCAmelCase_: Dict ,	lowerCAmelCase_: Optional[int]     ):
      snake_case_     :			List[str]       =       config.n_positions
      snake_case_     :			Dict       =       config.n_layer
      snake_case_     :			Optional[Any]       =       config.vocab_size
      snake_case_     :			Optional[int]       =       config.n_head
      snake_case_     :			Any       =       config.n_embd // config.n_head

      snake_case_     :			List[Any]       =       sd_megatron["model.language_model.embedding.word_embeddings.weight"][:vocab_size, :]
      snake_case_     :			Optional[Any]       =       {
          "transformer.wte.weight": word_embeddings,
          "transformer.wpe.weight": sd_megatron["model.language_model.embedding.position_embeddings.weight"],
          "transformer.ln_f.weight": sd_megatron["model.language_model.encoder.final_layernorm.weight"],
          "transformer.ln_f.bias": sd_megatron["model.language_model.encoder.final_layernorm.bias"],
      }

      snake_case_     :			List[Any]       =       "model.language_model.encoder.layers."
      for i in range(lowerCAmelCase_     ):
            snake_case_     :			int       =       torch.tril(torch.ones((n_positions, n_positions) ,	dtype=torch.bool     )     )
            snake_case_     :			Tuple       =       causal_mask.view(1 ,	1 ,	lowerCAmelCase_ ,	lowerCAmelCase_     )
            snake_case_     :			List[Any]       =       causal_mask
            snake_case_     :			str       =       torch.tensor(-1e4 ,	dtype=torch.bfloataa     )

            snake_case_     :			List[str]       =       sd_megatron[f"{pf}{i}.input_layernorm.weight"]
            snake_case_     :			List[Any]       =       sd_megatron[f"{pf}{i}.input_layernorm.bias"]

            snake_case_     :			Tuple       =       sd_megatron[f"{pf}{i}.self_attention.query_key_value.weight"]
            snake_case_     :			Union[str, Any]       =       fix_query_key_value_ordering(lowerCAmelCase_ ,	3 ,	lowerCAmelCase_ ,	lowerCAmelCase_     )
            snake_case_     :			Any       =       vala.transpose(0 ,	1     ).contiguous()

            snake_case_     :			Optional[int]       =       sd_megatron[f"{pf}{i}.self_attention.query_key_value.bias"]
            snake_case_     :			Optional[int]       =       fix_query_key_value_ordering(lowerCAmelCase_ ,	3 ,	lowerCAmelCase_ ,	lowerCAmelCase_     )
            snake_case_     :			Tuple       =       vala

            snake_case_     :			List[Any]       =       sd_megatron[f"{pf}{i}.self_attention.dense.weight"].transpose(
                0 ,	1     )
            snake_case_     :			str       =       sd_megatron[f"{pf}{i}.self_attention.dense.bias"]
            snake_case_     :			Union[str, Any]       =       sd_megatron[f"{pf}{i}.post_attention_layernorm.weight"]
            snake_case_     :			Optional[Any]       =       sd_megatron[f"{pf}{i}.post_attention_layernorm.bias"]
            snake_case_     :			List[str]       =       sd_megatron[f"{pf}{i}.mlp.dense_h_to_4h.weight"].transpose(0 ,	1     )
            snake_case_     :			str       =       sd_megatron[f"{pf}{i}.mlp.dense_h_to_4h.bias"]
            snake_case_     :			List[str]       =       sd_megatron[f"{pf}{i}.mlp.dense_4h_to_h.weight"].transpose(
                0 ,	1     )
            snake_case_     :			List[Any]       =       sd_megatron[f"{pf}{i}.mlp.dense_4h_to_h.bias"]

      # For LM head, transformers' wants the matrix to weight embeddings.
      snake_case_     :			int       =       word_embeddings

      return sd_hf






def 					SCREAMING_SNAKE_CASE_			(		lowerCAmelCase_: str ,	lowerCAmelCase_: Optional[int]     ):
      snake_case_     :			int       =       6_4_0_0_0
      snake_case_     :			List[Any]       =       config_megatron["encoder_seq_length"]
      snake_case_     :			Any       =       config_megatron["hidden_size"]
      snake_case_     :			str       =       config_megatron["num_layers"]
      snake_case_     :			Union[str, Any]       =       config_megatron["num_attention_heads"]
      snake_case_     :			Union[str, Any]       =       config_megatron["ffn_hidden_size"]
      snake_case_     :			Dict       =       "gelu"
      snake_case_     :			List[Any]       =       0.1
      snake_case_     :			str       =       0.1
      snake_case_     :			List[Any]       =       0.1
      snake_case_     :			List[Any]       =       config_megatron["layernorm_epsilon"]  # 1e-5
      snake_case_     :			Dict       =       config_megatron["init_method_std"]  # 0.02
      snake_case_     :			Tuple       =       config_megatron["apply_query_key_layer_scaling"]  # True
      snake_case_     :			Tuple       =       True
      snake_case_     :			Union[str, Any]       =       True

      # This identifies the 6.7B (7B) model which uses a different tokenizer
      if config_megatron["hidden_size"] == 4_0_9_6:
            snake_case_     :			Any       =       1  # <|endoftext|>
            snake_case_     :			List[Any]       =       1  # <|endoftext|>
            snake_case_     :			Dict       =       0  # <unk>
      else:
            snake_case_     :			Union[str, Any]       =       2  # <s>
            snake_case_     :			List[Any]       =       3  # <|endoftext|>
            snake_case_     :			List[str]       =       0  # <pad>

      return config_hf






def 					SCREAMING_SNAKE_CASE_			(		lowerCAmelCase_: Union[str, Any]     ):
      print(lowerCAmelCase_     )

      snake_case_     :			List[Any]       =       args.checkpoint_path
      snake_case_     :			str       =       args.save_path
      if isfile(lowerCAmelCase_     ):
            raise FileNotFoundError(f"ERROR! could not find file {checkpoint_path}"     )

      # Load the model.
      snake_case_     :			List[Any]       =       torch.load(lowerCAmelCase_ ,	map_location="cpu"     )

      # Load the config.
      snake_case_     :			List[Any]       =       checkpoint["hyper_parameters"]["cfg"]
      snake_case_     :			List[Any]       =       GPTaConfig()
      snake_case_     :			List[str]       =       copy_config(config_hf=lowerCAmelCase_ ,	config_megatron=lowerCAmelCase_     )
      snake_case_     :			Union[str, Any]       =       ["GPT2LMHeadModel"]

      snake_case_     :			Dict       =       checkpoint["state_dict"]

      # Convert.
      print("Converting"     )
      snake_case_     :			List[str]       =       convert_megatron_checkpoint(lowerCAmelCase_ ,	lowerCAmelCase_     )

      # Print the structure of converted state dict.
      if args.print_checkpoint_structure:
            recursive_print(lowerCAmelCase_ ,	lowerCAmelCase_     )

      snake_case_     :			str       =       "GPTSw3Tokenizer"

      # Store the config to file.
      print("Saving config"     )
      config_hf.save_pretrained(lowerCAmelCase_     )

      # Store the state_dict to file.
      snake_case_     :			str       =       os.path.join(lowerCAmelCase_ ,	"pytorch_model.bin"     )
      print(f"Saving checkpoint to \"{output_checkpoint_file}\""     )
      torch.save(lowerCAmelCase_ ,	lowerCAmelCase_     )


if __name__ == "__main__":
     UpperCAmelCase							=       argparse.ArgumentParser()
     parser.add_argument(
         "--checkpoint_path",
         type=str,
         required=True,
         help="e.g. megatron_gpt--val_loss=2.42-step=38000-consumed_samples=54720000",
     )
     parser.add_argument("--save_path", type=str, required=True, help="e.g. /home/user/gpt-sw3/hf")
     parser.add_argument("--print-checkpoint-structure", action="store_true")
     UpperCAmelCase							=       parser.parse_args()
     main(_args)




import requests
from bsa import BeautifulSoup






def 					SCREAMING_SNAKE_CASE_			(		lowerCAmelCase_: int ,	lowerCAmelCase_: str     ):
      snake_case_     :			Optional[int]       =       (
          "https://www.horoscope.com/us/horoscopes/general/"
          f"horoscope-general-daily-{day}.aspx?sign={zodiac_sign}"
      )
      snake_case_     :			int       =       BeautifulSoup(requests.get(lowerCAmelCase_     ).content ,	"html.parser"     )
      return soup.find("div" ,	class_="main-horoscope"     ).p.text


if __name__ == "__main__":
     print("Daily Horoscope. \n")
     print(
         "enter your Zodiac sign number:\n",
         "1. Aries\n",
         "2. Taurus\n",
         "3. Gemini\n",
         "4. Cancer\n",
         "5. Leo\n",
         "6. Virgo\n",
         "7. Libra\n",
         "8. Scorpio\n",
         "9. Sagittarius\n",
         "10. Capricorn\n",
         "11. Aquarius\n",
         "12. Pisces\n",
     )
     UpperCAmelCase							=       int(input("number> ").strip())
     print("choose some day:\n", "yesterday\n", "today\n", "tomorrow\n")
     UpperCAmelCase							=       input("enter the day> ")
     UpperCAmelCase							=       horoscope(zodiac_sign, day)
     print(horoscope_text)



