






'''simple docstring'''







import torch.nn as nn

from .imports import is_fpa_available


if is_fpa_available():
       import transformer_engine.pytorch as te


def 						_UpperCamelCase							(     snake_case__							,			snake_case__=True							,			snake_case__=True							,			snake_case__=True							):
      if not is_fpa_available():
            raise ImportError("Using `convert_model` requires transformer_engine to be installed."							)
      for name, module in model.named_children():
            if isinstance(__lowerCamelCase							,			nn.Linear							) and to_transformer_engine and _convert_linear:
                  # Return early if the linear layer weights are not multiples of 16
                  if any(p % 16 != 0 for p in module.weight.shape							):
                        return
                  __A		   =      module.bias is not None
                  __A		   =      te.Linear(
                      module.in_features							,			module.out_features							,			bias=__lowerCamelCase							,			params_dtype=module.weight.dtype							)
                  __A		   =      module.weight.data.clone()
                  if has_bias:
                        __A		   =      module.bias.data.clone()

                  setattr(__lowerCamelCase							,			__lowerCamelCase							,			__lowerCamelCase							)
            elif isinstance(__lowerCamelCase							,			nn.LayerNorm							) and to_transformer_engine and _convert_ln:
                  __A		   =      te.LayerNorm(module.normalized_shape[0]							,			eps=module.eps							,			params_dtype=module.weight.dtype							)
                  __A		   =      module.weight.data.clone()
                  __A		   =      module.bias.data.clone()

                  setattr(__lowerCamelCase							,			__lowerCamelCase							,			__lowerCamelCase							)
            elif isinstance(__lowerCamelCase							,			te.Linear							) and not to_transformer_engine and _convert_linear:
                  __A		   =      module.bias is not None
                  __A		   =      nn.Linear(
                      module.in_features							,			module.out_features							,			bias=__lowerCamelCase							,			params_dtype=module.weight.dtype							)
                  __A		   =      module.weight.data.clone()
                  if has_bias:
                        __A		   =      module.bias.data.clone()

                  setattr(__lowerCamelCase							,			__lowerCamelCase							,			__lowerCamelCase							)
            elif isinstance(__lowerCamelCase							,			te.LayerNorm							) and not to_transformer_engine and _convert_ln:
                  __A		   =      nn.LayerNorm(module.normalized_shape[0]							,			eps=module.eps							,			params_dtype=module.weight.dtype							)
                  __A		   =      module.weight.data.clone()
                  __A		   =      module.bias.data.clone()

                  setattr(__lowerCamelCase							,			__lowerCamelCase							,			__lowerCamelCase							)
            else:
                  convert_model(
                      __lowerCamelCase							,			to_transformer_engine=__lowerCamelCase							,			_convert_linear=_convert_linear							,			_convert_ln=_convert_ln							,			)





def 						_UpperCamelCase							(     snake_case__							):
      if not is_fpa_available():
            raise ImportError("Using `has_transformer_engine_layers` requires transformer_engine to be installed."							)
      for m in model.modules():
            if isinstance(__lowerCamelCase							,			(te.LayerNorm, te.Linear)							):
                  return True
      return False









'''simple docstring'''







import importlib
import sys
from argparse import REMAINDER, ArgumentParser
from pathlib import Path

import torch_xla.distributed.xla_multiprocessing as xmp


def 						_UpperCamelCase							(     ):
      __A		   =      ArgumentParser(
          description=(
              "PyTorch TPU distributed training launch helper utility that will spawn up multiple distributed processes"
          )							)

      # Optional arguments for the launch helper
      parser.add_argument("--num_cores"							,			type=snake_case__							,			default=1							,			help="Number of TPU cores to use (1 or 8)."							)

      # positional
      parser.add_argument(
          "training_script"							,			type=snake_case__							,			help=(
              "The full path to the single TPU training "
              "program/script to be launched in parallel, "
              "followed by all the arguments for the "
              "training script"
          )							,			)

      # rest from the training program
      parser.add_argument("training_script_args"							,			nargs=snake_case__							)

      return parser.parse_args()


def 						_UpperCamelCase							(     ):
      __A		   =      parse_args()

      # Import training_script as a module.
      __A		   =      Path(args.training_script							)
      sys.path.append(str(script_fpath.parent.resolve()							)							)
      __A		   =      script_fpath.stem
      __A		   =      importlib.import_module(snake_case__							)

      # Patch sys.argv
      __A		   =      [args.training_script] + args.training_script_args + ["--tpu_num_cores", str(args.num_cores							)]

      xmp.spawn(mod._mp_fn							,			args=()							,			nprocs=args.num_cores							)


if __name__ == "__main__":
       main()


