





'''simple docstring'''






import unittest

import numpy as np

from transformers import ElectraConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
      from transformers.models.electra.modeling_flax_electra import (
          FlaxElectraForCausalLM,
          FlaxElectraForMaskedLM,
          FlaxElectraForMultipleChoice,
          FlaxElectraForPreTraining,
          FlaxElectraForQuestionAnswering,
          FlaxElectraForSequenceClassification,
          FlaxElectraForTokenClassification,
          FlaxElectraModel,
      )




class 				UpperCAmelCase  (    unittest.TestCase     ):




      def __init__(    self    :				str    ,    __snake_case    :				str    ,    __snake_case    :				str=13    ,    __snake_case    :				Optional[Any]=7    ,    __snake_case    :				str=True    ,    __snake_case    :				Any=True    ,    __snake_case    :				List[str]=True    ,    __snake_case    :				Any=True    ,    __snake_case    :				List[str]=99    ,    __snake_case    :				Dict=24    ,    __snake_case    :				Any=32    ,    __snake_case    :				List[str]=5    ,    __snake_case    :				Any=4    ,    __snake_case    :				Optional[Any]=37    ,    __snake_case    :				Union[str, Any]="gelu"    ,    __snake_case    :				Dict=0.1    ,    __snake_case    :				int=0.1    ,    __snake_case    :				Dict=5_12    ,    __snake_case    :				List[Any]=16    ,    __snake_case    :				Dict=2    ,    __snake_case    :				str=0.02    ,    __snake_case    :				Any=4    ,    )       ->   Union[str, Any]:
            _lowerCAmelCase							  =							parent
            _lowerCAmelCase							  =							batch_size
            _lowerCAmelCase							  =							seq_length
            _lowerCAmelCase							  =							is_training
            _lowerCAmelCase							  =							use_attention_mask
            _lowerCAmelCase							  =							use_token_type_ids
            _lowerCAmelCase							  =							use_labels
            _lowerCAmelCase							  =							vocab_size
            _lowerCAmelCase							  =							hidden_size
            _lowerCAmelCase							  =							embedding_size
            _lowerCAmelCase							  =							num_hidden_layers
            _lowerCAmelCase							  =							num_attention_heads
            _lowerCAmelCase							  =							intermediate_size
            _lowerCAmelCase							  =							hidden_act
            _lowerCAmelCase							  =							hidden_dropout_prob
            _lowerCAmelCase							  =							attention_probs_dropout_prob
            _lowerCAmelCase							  =							max_position_embeddings
            _lowerCAmelCase							  =							type_vocab_size
            _lowerCAmelCase							  =							type_sequence_label_size
            _lowerCAmelCase							  =							initializer_range
            _lowerCAmelCase							  =							num_choices




      def        lowercase__					(    self    :				Optional[Any]						)       ->   Optional[Any]:
            _lowerCAmelCase							  =							ids_tensor([self.batch_size, self.seq_length]    ,    self.vocab_size						)

            _lowerCAmelCase							  =							None
            if self.use_attention_mask:
                  _lowerCAmelCase							  =							random_attention_mask([self.batch_size, self.seq_length]						)

            _lowerCAmelCase							  =							None
            if self.use_token_type_ids:
                  _lowerCAmelCase							  =							ids_tensor([self.batch_size, self.seq_length]    ,    self.type_vocab_size						)

            _lowerCAmelCase							  =							ElectraConfig(
                vocab_size=self.vocab_size    ,    hidden_size=self.hidden_size    ,    embedding_size=self.embedding_size    ,    num_hidden_layers=self.num_hidden_layers    ,    num_attention_heads=self.num_attention_heads    ,    intermediate_size=self.intermediate_size    ,    hidden_act=self.hidden_act    ,    hidden_dropout_prob=self.hidden_dropout_prob    ,    attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,    max_position_embeddings=self.max_position_embeddings    ,    type_vocab_size=self.type_vocab_size    ,    initializer_range=self.initializer_range    ,    )

            return config, input_ids, token_type_ids, attention_mask




      def        lowercase__					(    self    :				Any						)       ->   Optional[int]:
            _lowerCAmelCase							  =							self.prepare_config_and_inputs()
            _lowerCAmelCase  ,							_lowerCAmelCase  ,							_lowerCAmelCase  ,							_lowerCAmelCase							  =							config_and_inputs
            _lowerCAmelCase							  =							{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": attention_mask}
            return config, inputs_dict




@require_flax
class 				UpperCAmelCase  (    snake_case_							,	unittest.TestCase     ):
      _lowercase:    str		  =     True

      _lowercase:    Optional[Any]		  =     (
          (
              FlaxElectraModel,
              FlaxElectraForCausalLM,
              FlaxElectraForMaskedLM,
              FlaxElectraForPreTraining,
              FlaxElectraForTokenClassification,
              FlaxElectraForQuestionAnswering,
              FlaxElectraForMultipleChoice,
              FlaxElectraForSequenceClassification,
          )
          if is_flax_available()
          else ()
      )




      def        lowercase__					(    self    :				Tuple						)       ->   Optional[int]:
            _lowerCAmelCase							  =							FlaxElectraModelTester(self						)




      @slow
      def        lowercase__					(    self    :				List[Any]						)       ->   str:
            for model_class_name in self.all_model_classes:
                  if model_class_name == FlaxElectraForMaskedLM:
                        _lowerCAmelCase							  =							model_class_name.from_pretrained("""google/electra-small-generator"""						)
                  else:
                        _lowerCAmelCase							  =							model_class_name.from_pretrained("""google/electra-small-discriminator"""						)
                  _lowerCAmelCase							  =							model(np.ones((1, 1)						)						)
                  self.assertIsNotNone(__snake_case						)













'''simple docstring'''






import os
import random
from pathlib import Path
from typing import List

import numpy as np
import torch
from torch.cuda.amp import GradScaler

from .utils import (
    MODEL_NAME,
    OPTIMIZER_NAME,
    RNG_STATE_NAME,
    SCALER_NAME,
    SCHEDULER_NAME,
    get_pretty_name,
    is_tpu_available,
    is_xpu_available,
    save,
)


if is_tpu_available(check_device=False):
      import torch_xla.core.xla_model as xm

from .logging import get_logger
from .state import PartialState


A__       :     Optional[Any]          =get_logger(__name__)
def      UpperCamelCase__   (	lowerCAmelCase							,			lowerCAmelCase							,			lowerCAmelCase							,			lowerCAmelCase							,			lowerCAmelCase							,			lowerCAmelCase = None							,			):





      """simple docstring"""


      for i, state in enumerate(lowerCAmelCase						):
            _lowerCAmelCase							  =							f"{MODEL_NAME}.bin" if i == 0 else f"{MODEL_NAME}_{i}.bin"
            _lowerCAmelCase							  =							os.path.join(lowerCAmelCase							,			lowerCAmelCase						)
            save(lowerCAmelCase							,			lowerCAmelCase						)
            logger.info(f"Model weights saved in {output_model_file}"						)
      # Optimizer states
      for i, opt in enumerate(lowerCAmelCase						):
            _lowerCAmelCase							  =							opt.state_dict()
            _lowerCAmelCase							  =							f"{OPTIMIZER_NAME}.bin" if i == 0 else f"{OPTIMIZER_NAME}_{i}.bin"
            _lowerCAmelCase							  =							os.path.join(lowerCAmelCase							,			lowerCAmelCase						)
            save(lowerCAmelCase							,			lowerCAmelCase						)
            logger.info(f"Optimizer state saved in {output_optimizer_file}"						)
      # Scheduler states
      for i, scheduler in enumerate(lowerCAmelCase						):
            _lowerCAmelCase							  =							scheduler.state_dict()
            _lowerCAmelCase							  =							f"{SCHEDULER_NAME}.bin" if i == 0 else f"{SCHEDULER_NAME}_{i}.bin"
            _lowerCAmelCase							  =							os.path.join(lowerCAmelCase							,			lowerCAmelCase						)
            save(lowerCAmelCase							,			lowerCAmelCase						)
            logger.info(f"Scheduler state saved in {output_scheduler_file}"						)
      # GradScaler state
      if scaler is not None:
            _lowerCAmelCase							  =							scaler.state_dict()
            _lowerCAmelCase							  =							os.path.join(lowerCAmelCase							,			lowerCAmelCase						)
            torch.save(lowerCAmelCase							,			lowerCAmelCase						)
            logger.info(f"Gradient scaler state saved in {output_scaler_file}"						)
      # Random number generator states
      _lowerCAmelCase							  =							{}
      _lowerCAmelCase							  =							f"{RNG_STATE_NAME}_{process_index}.pkl"
      _lowerCAmelCase							  =							random.getstate()
      _lowerCAmelCase							  =							np.random.get_state()
      _lowerCAmelCase							  =							torch.get_rng_state()
      if is_xpu_available():
            _lowerCAmelCase							  =							torch.xpu.get_rng_state_all()
      else:
            _lowerCAmelCase							  =							torch.cuda.get_rng_state_all()
      if is_tpu_available():
            _lowerCAmelCase							  =							xm.get_rng_state()
      _lowerCAmelCase							  =							os.path.join(lowerCAmelCase							,			lowerCAmelCase						)
      torch.save(lowerCAmelCase							,			lowerCAmelCase						)
      logger.info(f"Random states saved in {output_states_file}"						)
      return output_dir
def      UpperCamelCase__   (	lowerCAmelCase							,			lowerCAmelCase							,			lowerCAmelCase							,			lowerCAmelCase							,			lowerCAmelCase							,			lowerCAmelCase=None							,			lowerCAmelCase=None							,			**lowerCAmelCase							,			):





      """simple docstring"""


      if map_location not in [None, "cpu", "on_device"]:
            raise TypeError(
                """Unsupported optimizer map location passed, please choose one of `None`, `'cpu'`, or `'on_device'`"""						)
      if map_location is None:
            _lowerCAmelCase							  =							"""cpu"""
      elif map_location == "on_device":
            _lowerCAmelCase							  =							PartialState().device
      # Model states
      for i, model in enumerate(lowerCAmelCase						):
            _lowerCAmelCase							  =							f"{MODEL_NAME}.bin" if i == 0 else f"{MODEL_NAME}_{i}.bin"
            _lowerCAmelCase							  =							os.path.join(lowerCAmelCase							,			lowerCAmelCase						)
            models[i].load_state_dict(torch.load(lowerCAmelCase							,			map_location=lowerCAmelCase						)							,			**lowerCAmelCase						)
      logger.info("""All model weights loaded successfully"""						)

      # Optimizer states
      for i, opt in enumerate(lowerCAmelCase						):
            _lowerCAmelCase							  =							f"{OPTIMIZER_NAME}.bin" if i == 0 else f"{OPTIMIZER_NAME}_{i}.bin"
            _lowerCAmelCase							  =							os.path.join(lowerCAmelCase							,			lowerCAmelCase						)
            _lowerCAmelCase							  =							torch.load(lowerCAmelCase							,			map_location=lowerCAmelCase						)
            optimizers[i].load_state_dict(lowerCAmelCase						)
      logger.info("""All optimizer states loaded successfully"""						)

      # Scheduler states
      for i, scheduler in enumerate(lowerCAmelCase						):
            _lowerCAmelCase							  =							f"{SCHEDULER_NAME}.bin" if i == 0 else f"{SCHEDULER_NAME}_{i}.bin"
            _lowerCAmelCase							  =							os.path.join(lowerCAmelCase							,			lowerCAmelCase						)
            scheduler.load_state_dict(torch.load(lowerCAmelCase						)						)
      logger.info("""All scheduler states loaded successfully"""						)

      # GradScaler state
      if scaler is not None:
            _lowerCAmelCase							  =							os.path.join(lowerCAmelCase							,			lowerCAmelCase						)
            scaler.load_state_dict(torch.load(lowerCAmelCase						)						)
            logger.info("""GradScaler state loaded successfully"""						)

      # Random states
      try:
            _lowerCAmelCase							  =							torch.load(os.path.join(lowerCAmelCase							,			f"{RNG_STATE_NAME}_{process_index}.pkl"						)						)
            random.setstate(states["""random_state"""]						)
            np.random.set_state(states["""numpy_random_seed"""]						)
            torch.set_rng_state(states["""torch_manual_seed"""]						)
            if is_xpu_available():
                  torch.xpu.set_rng_state_all(states["""torch_xpu_manual_seed"""]						)
            else:
                  torch.cuda.set_rng_state_all(states["""torch_cuda_manual_seed"""]						)
            if is_tpu_available():
                  xm.set_rng_state(states["""xm_seed"""]						)
            logger.info("""All random states loaded successfully"""						)
      except Exception:
            logger.info("""Could not load random states"""						)
def      UpperCamelCase__   (	lowerCAmelCase							,			lowerCAmelCase							,			lowerCAmelCase = 0						):





      """simple docstring"""


      _lowerCAmelCase							  =							Path(lowerCAmelCase						) / f"custom_checkpoint_{index}.pkl"
      logger.info(f"Saving the state of {get_pretty_name(lowerCAmelCase						)} to {save_location}"						)
      torch.save(obj.state_dict()							,			lowerCAmelCase						)





def      UpperCamelCase__   (	lowerCAmelCase							,			lowerCAmelCase							,			lowerCAmelCase = 0						):





      """simple docstring"""


      _lowerCAmelCase							  =							f"{path}/custom_checkpoint_{index}.pkl"
      logger.info(f"Loading the state of {get_pretty_name(lowerCAmelCase						)} from {load_location}"						)
      obj.load_state_dict(torch.load(lowerCAmelCase							,			map_location="""cpu"""						)						)






