






import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
   import torch

a_					      = logging.get_logger(__name__)





def      __lowercase  (		):
 return importlib.util.find_spec('deepspeed'			) is not None


if is_accelerate_available() and is_deepspeed_available():
   from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
   # Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
   # Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
   from builtins import object as DeepSpeedConfig





class 	_lowercase		(      snake_case_ ):

 def __init__(				self			:							Optional[int]     ,   snake_case			:							int    )     ->		Tuple:

  """simple docstring"""





  set_hf_deepspeed_config(self    )
  dep_version_check('accelerate'    )
  dep_version_check('deepspeed'    )
  super().__init__(snake_case    )





class 	_lowercase		(      snake_case_ ):

 def __init__(				self			:							Union[str, Any]     ,   snake_case			:							List[str]    )     ->		int:

  """simple docstring"""





  super().__init__(snake_case    )
  UpperCamelCase_      :		str				=      None
  UpperCamelCase_      :		List[str]				=      []

 def 		SCREAMING_SNAKE_CASE__     (				self			:							Dict    )     ->		Any:

  """simple docstring"""





  if self._dtype is None:
   raise ValueError('trainer_config_process() wasn\'t called yet to tell dtype'    )
  return self._dtype

 def 		SCREAMING_SNAKE_CASE__     (				self			:							str     ,   snake_case			:							int    )     ->		Optional[int]:

  """simple docstring"""





  UpperCamelCase_      :		Dict				=      self.get_value(snake_case    )
  if val is None:
   return False
  else:
   return val == "auto"

 def 		SCREAMING_SNAKE_CASE__     (				self			:							Union[str, Any]     ,   snake_case			:							Any     ,   snake_case			:							Optional[Any]     ,   snake_case			:							Optional[int]=None     ,   snake_case			:							Optional[int]=True    )     ->		Optional[Any]:

  """simple docstring"""





  UpperCamelCase_      :		Optional[Any]				=      self.find_config_node(snake_case    )
  if config is None:
   return

  if config.get(snake_case    ) == "auto":
   UpperCamelCase_      :		Union[str, Any]				=      hf_val
   return

  if not must_match:
   return

  UpperCamelCase_      :		Any				=      config.get(snake_case    )
  if ds_val is not None and ds_val != hf_val:
   self.mismatches.append(f"- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}"    )

 lowercase         =     partialmethod(snake_case_		,   must_match=snake_case_ )

 def 		SCREAMING_SNAKE_CASE__     (				self			:							Dict     ,   snake_case			:							str    )     ->		List[Any]:

  """simple docstring"""





  UpperCamelCase_      :		str				=      args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
  self.fill_match(
      'train_micro_batch_size_per_gpu'     ,   args.per_device_train_batch_size     ,   'per_device_train_batch_size'    )
  self.fill_match('gradient_accumulation_steps'     ,   args.gradient_accumulation_steps     ,   'gradient_accumulation_steps'    )
  self.fill_match('train_batch_size'     ,   snake_case     ,   'train_batch_size (calculated)'    )
  self.fill_match('gradient_clipping'     ,   args.max_grad_norm     ,   'max_grad_norm'    )

  self.fill_match('optimizer.params.lr'     ,   args.learning_rate     ,   'learning_rate'    )
  self.fill_match('optimizer.params.betas'     ,   [args.adam_betaa, args.adam_betaa]     ,   'adam_beta1+adam_beta2'    )
  self.fill_match('optimizer.params.eps'     ,   args.adam_epsilon     ,   'adam_epsilon'    )
  self.fill_match('optimizer.params.weight_decay'     ,   args.weight_decay     ,   'weight_decay'    )

  self.fill_only('scheduler.params.warmup_min_lr'     ,   0    )  # not a trainer arg
  self.fill_match('scheduler.params.warmup_max_lr'     ,   args.learning_rate     ,   'learning_rate'    )
  # total_num_steps - will get set in trainer_config_finalize

  # fp16
  if args.fpaa or args.fpaa_full_eval:
   UpperCamelCase_      :		List[str]				=      'apex' if args.fpaa_backend == 'apex' else 'amp'
  else:
   UpperCamelCase_      :		Optional[int]				=      None

  if args.save_on_each_node:
   # deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
   UpperCamelCase_      :		int				=      self.config.get('checkpoint'     ,   {}    )
   UpperCamelCase_      :		List[Any]				=      args.save_on_each_node

  # amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
  # any here unless the user did the work
  self.fill_match(
      'fp16.enabled'     ,   ((args.fpaa or args.fpaa_full_eval) and fpaa_backend == 'amp')     ,   'fp16|fp16_full_eval+fp16_backend(amp)'     ,   )

  # apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
  # ZeRO features
  self.fill_match('amp.enabled'     ,   fpaa_backend == 'apex'     ,   'fp16+fp16_backend(apex)'    )
  self.fill_match('amp.opt_level'     ,   args.fpaa_opt_level     ,   'fp16_opt_level'    )

  self.fill_match('bf16.enabled'     ,   (args.bfaa or args.bfaa_full_eval)     ,   'bf16|bf16_full_eval'    )

  # deepspeed's default mode is fp16 unless there is a config that says differently
  if self.is_true('bf16.enabled'    ):
   UpperCamelCase_      :		List[Any]				=      torch.bfloataa
  elif self.is_false('fp16.enabled'    ):
   UpperCamelCase_      :		Union[str, Any]				=      torch.floataa
  else:
   UpperCamelCase_      :		Union[str, Any]				=      torch.floataa


 def 		SCREAMING_SNAKE_CASE__     (				self			:							Optional[int]     ,   snake_case			:							Tuple     ,   snake_case			:							int     ,   snake_case			:							Any    )     ->		Tuple:

  """simple docstring"""





  UpperCamelCase_      :		List[Any]				=      [
      'zero_optimization.reduce_bucket_size',
      'zero_optimization.stage3_prefetch_bucket_size',
      'zero_optimization.stage3_param_persistence_threshold',
  ]
  UpperCamelCase_      :		Optional[int]				=      [x for x in hidden_size_based_keys if self.is_auto(snake_case    )]

  if len(snake_case    ) > 0:
   if hasattr(model.config     ,   'hidden_size'    ):
    UpperCamelCase_      :		str				=      model.config.hidden_size
   elif hasattr(model.config     ,   'hidden_sizes'    ):
    # if there are many hidden sizes pick the largest one
    UpperCamelCase_      :		Optional[Any]				=      max(model.config.hidden_sizes    )
   else:
    raise ValueError(
        'The model\'s config file has neither `hidden_size` nor `hidden_sizes` entry, '
        'therefore it\'s not possible to automatically fill out the following `auto` entries '
        f"in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing "
        '`auto` values for these keys with an integer value of your choice.'    )

   self.fill_only('zero_optimization.reduce_bucket_size'     ,   hidden_size * hidden_size    )
   if self.is_zeroa():
    # automatically assign the optimal config values based on model config
    self.fill_only('zero_optimization.stage3_prefetch_bucket_size'     ,   0.9 * hidden_size * hidden_size    )
    self.fill_only('zero_optimization.stage3_param_persistence_threshold'     ,   1_0 * hidden_size    )

        # scheduler
  self.fill_match('scheduler.params.total_num_steps'     ,   snake_case     ,   'num_training_steps (calculated)'    )
  self.fill_match('scheduler.params.warmup_num_steps'     ,   args.get_warmup_steps(snake_case    )     ,   'warmup_steps'    )

  if len(self.mismatches    ) > 0:
   UpperCamelCase_      :		List[str]				=      '\n'.join(self.mismatches    )
   raise ValueError(
       'Please correct the following DeepSpeed config values that mismatch TrainingArguments'
       f" values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to 'auto'."    )
a_					      = None





def      __lowercase  (		lowerCamelCase      :  Union[str, Any]			):
 # this is a special weakref global object to allow us to get to Deepspeed config from APIs
 # that don't have an easy way to get to the Deepspeed config outside of the Trainer domain.
 global _hf_deepspeed_config_weak_ref
 # will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
 UpperCamelCase_      :		Dict				=      weakref.ref(lowerCamelCase			)





def      __lowercase  (		):
 # useful for unit tests to ensure the global state doesn't leak - call from `tearDown` method
 global _hf_deepspeed_config_weak_ref
 UpperCamelCase_      :		Optional[int]				=      None





def      __lowercase  (		):
 if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
  return _hf_deepspeed_config_weak_ref().is_zeroa()
 else:
  return False





def      __lowercase  (		):
 if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
  return _hf_deepspeed_config_weak_ref().config
 else:
  return None





def      __lowercase  (		lowerCamelCase      :  Dict						,						lowerCamelCase      :  Any						,						lowerCamelCase      :  Tuple						,						lowerCamelCase      :  List[str]						,						lowerCamelCase      :  str			):
 from accelerate.utils import DummyOptim, DummyScheduler

 UpperCamelCase_      :		Tuple				=      hf_deepspeed_config.config

 # Optimizer + Scheduler
 # Currently supported combos:
 # 1. DS scheduler + DS optimizer: Yes
 # 2. HF scheduler + HF optimizer: Yes
 # 3. DS scheduler + HF optimizer: Yes
 # 4. HF scheduler + DS optimizer: No
 #
 # Unless Offload is enabled in which case it's:
 # 1. DS scheduler + DS optimizer: Yes
 # 2. HF scheduler + HF optimizer: Mostly*
 # 3. DS scheduler + HF optimizer: Mostly*
 # 4. HF scheduler + DS optimizer: No
 #
 # Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

 UpperCamelCase_      :		Optional[Any]				=      None
 if "optimizer" in config:
  if args.adafactor:
   raise ValueError(
       '--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. '
       'Only one optimizer can be configured.'			)
  UpperCamelCase_      :		Any				=      DummyOptim(params=lowerCamelCase			)
 else:
  if hf_deepspeed_config.is_offload():
   logger.info(
       'Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the'
       ' custom optimizer has both CPU and GPU implementation (except LAMB)'			)

  # ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
  # But trainer uses AdamW by default.
  UpperCamelCase_      :		List[str]				=      trainer.create_optimizer()
  # To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
  UpperCamelCase_      :		Union[str, Any]				=      True

 UpperCamelCase_      :		Optional[Any]				=      None
 if "scheduler" in config:
  UpperCamelCase_      :		Tuple				=      DummyScheduler(lowerCamelCase			)
 else:
  if isinstance(lowerCamelCase						,						lowerCamelCase			):
   raise ValueError(
       'Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. '
       'Please configure a scheduler in the DeepSpeed config.'			)
  UpperCamelCase_      :		Optional[Any]				=      trainer.create_scheduler(num_training_steps=lowerCamelCase						,						optimizer=lowerCamelCase			)

 return optimizer, lr_scheduler





def      __lowercase  (		lowerCamelCase      :  Any						,						lowerCamelCase      :  int						,						lowerCamelCase      :  List[str]=False			):
 from deepspeed.utils import logger as ds_logger

 UpperCamelCase_      :		Tuple				=      trainer.model
 UpperCamelCase_      :		List[Any]				=      trainer.args

 UpperCamelCase_      :		Any				=      trainer.accelerator.state.deepspeed_plugin.hf_ds_config

 # resume config update - some bits like `model` and `num_training_steps` only become available during train
 hf_deepspeed_config.trainer_config_finalize(lowerCamelCase						,						lowerCamelCase						,						lowerCamelCase			)

 # set the Deepspeed log level consistent with the Trainer
 ds_logger.setLevel(args.get_process_log_level()			)

 if inference:
  # only Z3 makes sense for the inference
  if not hf_deepspeed_config.is_zeroa():
   raise ValueError('ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config'			)

  # in case the training config is re-used for inference
  hf_deepspeed_config.del_config_sub_tree('optimizer'			)
  hf_deepspeed_config.del_config_sub_tree('lr_scheduler'			)
  UpperCamelCase_      :		str				=      None, None
  UpperCamelCase_      :		Union[str, Any]				=      None
 else:
  UpperCamelCase_      :		Union[str, Any]				=      None  # important for when deepspeed_init is used as re-init
  UpperCamelCase_      :		str				=      list(filter(lambda lowerCamelCase			: p.requires_grad						,						model.parameters()			)			)
  UpperCamelCase_      :		Optional[Any]				=      deepspeed_optim_sched(
      lowerCamelCase						,						lowerCamelCase						,						lowerCamelCase						,						lowerCamelCase						,						lowerCamelCase			)

 # keep for quick debug:
 # from pprint import pprint; pprint(config)

 return optimizer, lr_scheduler






def      __lowercase  (		lowerCamelCase      :  Dict						,						lowerCamelCase      :  Tuple			):
 # it's possible that the user is trying to resume from model_path, which doesn't necessarily
 # contain a deepspeed checkpoint. e.g. examples just check if the dir exists and assume it's
 # a resume from a checkpoint and not just a local pretrained weight. So we check here if the
 # path contains what looks like a deepspeed checkpoint
 import glob

 UpperCamelCase_      :		Dict				=      sorted(glob.glob(F"{checkpoint_path}/global_step*"			)			)

 if len(lowerCamelCase			) > 0:
  logger.info(F"Attempting to resume from {checkpoint_path}"			)
  # this magically updates self.optimizer and self.lr_scheduler
  UpperCamelCase_      :		int				=      deepspeed_engine.load_checkpoint(
      lowerCamelCase						,						load_optimizer_states=lowerCamelCase						,						load_lr_scheduler_states=lowerCamelCase			)
  if load_path is None:
   raise ValueError(F"[deepspeed] failed to resume from checkpoint {checkpoint_path}"			)
 else:
  raise ValueError(F"Can't find a valid checkpoint at {checkpoint_path}"			)




import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_deberta import DebertaTokenizer


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation


a_					      = logging.get_logger(__name__)

a_					      = {'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

a_					      = {
    'vocab_file': {
        'microsoft/deberta-base': 'https://huggingface.co/microsoft/deberta-base/resolve/main/vocab.json',
        'microsoft/deberta-large': 'https://huggingface.co/microsoft/deberta-large/resolve/main/vocab.json',
        'microsoft/deberta-xlarge': 'https://huggingface.co/microsoft/deberta-xlarge/resolve/main/vocab.json',
        'microsoft/deberta-base-mnli': 'https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/vocab.json',
        'microsoft/deberta-large-mnli': 'https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/vocab.json',
        'microsoft/deberta-xlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'microsoft/deberta-base': 'https://huggingface.co/microsoft/deberta-base/resolve/main/merges.txt',
        'microsoft/deberta-large': 'https://huggingface.co/microsoft/deberta-large/resolve/main/merges.txt',
        'microsoft/deberta-xlarge': 'https://huggingface.co/microsoft/deberta-xlarge/resolve/main/merges.txt',
        'microsoft/deberta-base-mnli': 'https://huggingface.co/microsoft/deberta-base-mnli/resolve/main/merges.txt',
        'microsoft/deberta-large-mnli': 'https://huggingface.co/microsoft/deberta-large-mnli/resolve/main/merges.txt',
        'microsoft/deberta-xlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-xlarge-mnli/resolve/main/merges.txt'
        ),
    },
}

a_					      = {
    'microsoft/deberta-base': 512,
    'microsoft/deberta-large': 512,
    'microsoft/deberta-xlarge': 512,
    'microsoft/deberta-base-mnli': 512,
    'microsoft/deberta-large-mnli': 512,
    'microsoft/deberta-xlarge-mnli': 512,
}

a_					      = {
    'microsoft/deberta-base': {'do_lower_case': False},
    'microsoft/deberta-large': {'do_lower_case': False},
}





class 	_lowercase		(      snake_case_ ):
 lowercase         =     VOCAB_FILES_NAMES
 lowercase         =     PRETRAINED_VOCAB_FILES_MAP
 lowercase         =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowercase         =     ['input_ids', 'attention_mask', 'token_type_ids']
 lowercase         =     DebertaTokenizer

 def __init__(				self			:							Optional[int]     ,   snake_case			:							Any=None     ,   snake_case			:							Union[str, Any]=None     ,   snake_case			:							Optional[Any]=None     ,   snake_case			:							List[str]="replace"     ,   snake_case			:							int="[CLS]"     ,   snake_case			:							Any="[SEP]"     ,   snake_case			:							int="[SEP]"     ,   snake_case			:							Union[str, Any]="[CLS]"     ,   snake_case			:							List[Any]="[UNK]"     ,   snake_case			:							List[str]="[PAD]"     ,   snake_case			:							Tuple="[MASK]"     ,   snake_case			:							int=False     ,   **snake_case			:							List[str]     ,   )     ->		Any:

  """simple docstring"""





  super().__init__(
      snake_case     ,   snake_case     ,   tokenizer_file=snake_case     ,   errors=snake_case     ,   bos_token=snake_case     ,   eos_token=snake_case     ,   unk_token=snake_case     ,   sep_token=snake_case     ,   cls_token=snake_case     ,   pad_token=snake_case     ,   mask_token=snake_case     ,   add_prefix_space=snake_case     ,   **snake_case     ,   )
  UpperCamelCase_      :		str				=      kwargs.pop('add_bos_token'     ,   snake_case    )

  UpperCamelCase_      :		Tuple				=      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()    )
  if pre_tok_state.get('add_prefix_space'     ,   snake_case    ) != add_prefix_space:
   UpperCamelCase_      :		str				=      getattr(snake_case     ,   pre_tok_state.pop('type'    )    )
   UpperCamelCase_      :		List[Any]				=      add_prefix_space
   UpperCamelCase_      :		List[str]				=      pre_tok_class(**snake_case    )

  UpperCamelCase_      :		int				=      add_prefix_space

 @property
 def 		SCREAMING_SNAKE_CASE__     (				self			:							List[str]    )     ->		str:

  """simple docstring"""





  if self._mask_token is None:
   if self.verbose:
    logger.error('Using mask_token, but it is not set yet.'    )
   return None
  return str(self._mask_token    )

 @mask_token.setter
 def 		SCREAMING_SNAKE_CASE__     (				self			:							Any     ,   snake_case			:							Optional[int]    )     ->		str:

  """simple docstring"""





  UpperCamelCase_      :		List[Any]				=      AddedToken(snake_case     ,   lstrip=snake_case     ,   rstrip=snake_case    ) if isinstance(snake_case     ,   snake_case    ) else value
  UpperCamelCase_      :		Optional[int]				=      value

 def 		SCREAMING_SNAKE_CASE__     (				self			:							str     ,   snake_case			:							List[int]     ,   snake_case			:							Optional[List[int]] = None    )     ->		List[int]:

  """simple docstring"""





  if token_ids_a is None:
   return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
  UpperCamelCase_      :		Dict				=      [self.cls_token_id]
  UpperCamelCase_      :		Tuple				=      [self.sep_token_id]
  return cls + token_ids_a + sep + token_ids_a + sep

 def 		SCREAMING_SNAKE_CASE__     (				self			:							str     ,   snake_case			:							List[int]     ,   snake_case			:							Optional[List[int]] = None    )     ->		List[int]:

  """simple docstring"""





  UpperCamelCase_      :		Optional[int]				=      [self.sep_token_id]
  UpperCamelCase_      :		List[str]				=      [self.cls_token_id]

  if token_ids_a is None:
   return len(cls + token_ids_a + sep    ) * [0]
  return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]

 def 		SCREAMING_SNAKE_CASE__     (				self			:							Optional[int]     ,   *snake_case			:							Any     ,   **snake_case			:							Union[str, Any]    )     ->		BatchEncoding:

  """simple docstring"""





  UpperCamelCase_      :		Tuple				=      kwargs.get('is_split_into_words'     ,   snake_case    )
  assert self.add_prefix_space or not is_split_into_words, (
      f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
      "to use it with pretokenized inputs."
  )

  return super()._batch_encode_plus(*snake_case     ,   **snake_case    )

 def 		SCREAMING_SNAKE_CASE__     (				self			:							int     ,   *snake_case			:							List[str]     ,   **snake_case			:							Union[str, Any]    )     ->		BatchEncoding:

  """simple docstring"""





  UpperCamelCase_      :		List[Any]				=      kwargs.get('is_split_into_words'     ,   snake_case    )

  assert self.add_prefix_space or not is_split_into_words, (
      f"You need to instantiate {self.__class__.__name__} with add_prefix_space=True "
      "to use it with pretokenized inputs."
  )

  return super()._encode_plus(*snake_case     ,   **snake_case    )

 def 		SCREAMING_SNAKE_CASE__     (				self			:							Optional[Any]     ,   snake_case			:							str     ,   snake_case			:							Optional[str] = None    )     ->		Tuple[str]:

  """simple docstring"""





  UpperCamelCase_      :		Dict				=      self._tokenizer.model.save(snake_case     ,   name=snake_case    )
  return tuple(snake_case    )


 def 		SCREAMING_SNAKE_CASE__     (				self			:							Any     ,   snake_case			:							"Conversation"    )     ->		List[int]:

  """simple docstring"""





  UpperCamelCase_      :		Tuple				=      []
  for is_user, text in conversation.iter_texts():
   input_ids.extend(self.encode(snake_case     ,   add_special_tokens=snake_case    ) + [self.eos_token_id]    )

  if len(snake_case    ) > self.model_max_length:
   UpperCamelCase_      :		Optional[int]				=      input_ids[-self.model_max_length :]
  return input_ids



