





import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
      import torch

__lowercase													=			logging.get_logger(__name__)


def 					__SCREAMING_SNAKE_CASE  (							):



  '''simple docstring'''






  return importlib.util.find_spec('''deepspeed''') is not None


if is_accelerate_available() and is_deepspeed_available():
      from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
      # Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
      # Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
      from builtins import object as DeepSpeedConfig

class   __snake_case				(			_a   ):






    '''simple docstring'''





    def __init__(   self						:		Optional[int]			,						snake_case						:		Tuple							):

      """simple docstring"""





      set_hf_deepspeed_config(self							)
      dep_version_check('''accelerate'''							)
      dep_version_check('''deepspeed'''							)
      super().__init__(snake_case							)


class   __snake_case				(			_a   ):






    '''simple docstring'''





    def __init__(   self						:		Optional[int]			,						snake_case						:		Dict							):

      """simple docstring"""





      super().__init__(snake_case							)
      __lowerCAmelCase			=				None
      __lowerCAmelCase			=				[]





    def 	A		(   self						:		List[str]							):

      """simple docstring"""





      if self._dtype is None:
        raise ValueError('''trainer_config_process() wasn\'t called yet to tell dtype'''							)
      return self._dtype





    def 	A		(   self						:		Optional[int]			,						snake_case						:		List[Any]							):

      """simple docstring"""





      __lowerCAmelCase			=				self.get_value(snake_case							)
      if val is None:
        return False
      else:
        return val == "auto"





    def 	A		(   self						:		Union[str, Any]			,						snake_case						:		List[str]			,						snake_case						:		Optional[int]			,						snake_case						:		List[str]=None			,						snake_case						:		List[str]=True							):

      """simple docstring"""





      __lowerCAmelCase	,						__lowerCAmelCase			=				self.find_config_node(snake_case							)
      if config is None:
        return

      if config.get(snake_case							) == "auto":
        __lowerCAmelCase			=				hf_val
        return

      if not must_match:
        return

      __lowerCAmelCase			=				config.get(snake_case							)
      if ds_val is not None and ds_val != hf_val:
        self.mismatches.append(f'''- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}'''							)

    _lowerCAmelCase     =	partialmethod(_a       ,				must_match=_a   )





    def 	A		(   self						:		int			,						snake_case						:		Optional[Any]							):

      """simple docstring"""





      __lowerCAmelCase			=				args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
      self.fill_match(
          '''train_micro_batch_size_per_gpu'''			,						args.per_device_train_batch_size			,						'''per_device_train_batch_size'''							)
      self.fill_match('''gradient_accumulation_steps'''			,						args.gradient_accumulation_steps			,						'''gradient_accumulation_steps'''							)
      self.fill_match('''train_batch_size'''			,						snake_case			,						'''train_batch_size (calculated)'''							)
      self.fill_match('''gradient_clipping'''			,						args.max_grad_norm			,						'''max_grad_norm'''							)

      self.fill_match('''optimizer.params.lr'''			,						args.learning_rate			,						'''learning_rate'''							)
      self.fill_match('''optimizer.params.betas'''			,						[args.adam_betaa, args.adam_betaa]			,						'''adam_beta1+adam_beta2'''							)
      self.fill_match('''optimizer.params.eps'''			,						args.adam_epsilon			,						'''adam_epsilon'''							)
      self.fill_match('''optimizer.params.weight_decay'''			,						args.weight_decay			,						'''weight_decay'''							)

      self.fill_only('''scheduler.params.warmup_min_lr'''			,						0							)  # not a trainer arg
      self.fill_match('''scheduler.params.warmup_max_lr'''			,						args.learning_rate			,						'''learning_rate'''							)
      # total_num_steps - will get set in trainer_config_finalize

      # fp16
      if args.fpaa or args.fpaa_full_eval:
        __lowerCAmelCase			=				'''apex''' if args.fpaa_backend == '''apex''' else '''amp'''
      else:
        __lowerCAmelCase			=				None

      if args.save_on_each_node:
        # deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
        __lowerCAmelCase			=				self.config.get('''checkpoint'''			,						{}							)
        __lowerCAmelCase			=				args.save_on_each_node

      # amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
      # any here unless the user did the work
      self.fill_match(
          '''fp16.enabled'''			,						((args.fpaa or args.fpaa_full_eval) and fpaa_backend == '''amp''')			,						'''fp16|fp16_full_eval+fp16_backend(amp)'''			,						)

      # apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
      # ZeRO features
      self.fill_match('''amp.enabled'''			,						fpaa_backend == '''apex'''			,						'''fp16+fp16_backend(apex)'''							)
      self.fill_match('''amp.opt_level'''			,						args.fpaa_opt_level			,						'''fp16_opt_level'''							)

      self.fill_match('''bf16.enabled'''			,						(args.bfaa or args.bfaa_full_eval)			,						'''bf16|bf16_full_eval'''							)

      # deepspeed's default mode is fp16 unless there is a config that says differently
      if self.is_true('''bf16.enabled'''							):
        __lowerCAmelCase			=				torch.bfloataa
      elif self.is_false('''fp16.enabled'''							):
        __lowerCAmelCase			=				torch.floataa
      else:
        __lowerCAmelCase			=				torch.floataa





    def 	A		(   self						:		Tuple			,						snake_case						:		int			,						snake_case						:		List[Any]			,						snake_case						:		List[str]							):

      """simple docstring"""





      __lowerCAmelCase			=				[
          '''zero_optimization.reduce_bucket_size''',
          '''zero_optimization.stage3_prefetch_bucket_size''',
          '''zero_optimization.stage3_param_persistence_threshold''',
      ]
      __lowerCAmelCase			=				[x for x in hidden_size_based_keys if self.is_auto(snake_case							)]

      if len(snake_case							) > 0:
        if hasattr(model.config			,						'''hidden_size'''							):
          __lowerCAmelCase			=				model.config.hidden_size
        elif hasattr(model.config			,						'''hidden_sizes'''							):
          # if there are many hidden sizes pick the largest one
          __lowerCAmelCase			=				max(model.config.hidden_sizes							)
        else:
          raise ValueError(
              '''The model\'s config file has neither `hidden_size` nor `hidden_sizes` entry, '''
              '''therefore it\'s not possible to automatically fill out the following `auto` entries '''
              f'''in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing '''
              '''`auto` values for these keys with an integer value of your choice.'''							)

        self.fill_only('''zero_optimization.reduce_bucket_size'''			,						hidden_size * hidden_size							)
        if self.is_zeroa():
          # automatically assign the optimal config values based on model config
          self.fill_only('''zero_optimization.stage3_prefetch_bucket_size'''			,						0.9 * hidden_size * hidden_size							)
          self.fill_only('''zero_optimization.stage3_param_persistence_threshold'''			,						10 * hidden_size							)

        # scheduler
      self.fill_match('''scheduler.params.total_num_steps'''			,						snake_case			,						'''num_training_steps (calculated)'''							)
      self.fill_match('''scheduler.params.warmup_num_steps'''			,						args.get_warmup_steps(snake_case							)			,						'''warmup_steps'''							)

      if len(self.mismatches							) > 0:
        __lowerCAmelCase			=				'''\n'''.join(self.mismatches							)
        raise ValueError(
            '''Please correct the following DeepSpeed config values that mismatch TrainingArguments'''
            f''' values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to \'auto\'.'''							)


__lowercase													=			None


def 					__SCREAMING_SNAKE_CASE  (							__lowercase:		List[str]):



  '''simple docstring'''






  global _hf_deepspeed_config_weak_ref
  # will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
  __lowerCAmelCase			=				weakref.ref(__lowercase)


def 					__SCREAMING_SNAKE_CASE  (							):



  '''simple docstring'''






  global _hf_deepspeed_config_weak_ref
  __lowerCAmelCase			=				None


def 					__SCREAMING_SNAKE_CASE  (							):



  '''simple docstring'''






  if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
    return _hf_deepspeed_config_weak_ref().is_zeroa()
  else:
    return False


def 					__SCREAMING_SNAKE_CASE  (							):



  '''simple docstring'''






  if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
    return _hf_deepspeed_config_weak_ref().config
  else:
    return None


def 					__SCREAMING_SNAKE_CASE  (							__lowercase:		Optional[Any]     ,							__lowercase:		Optional[Any]     ,							__lowercase:		Union[str, Any]     ,							__lowercase:		Dict     ,							__lowercase:		str):



  '''simple docstring'''






  from accelerate.utils import DummyOptim, DummyScheduler

  __lowerCAmelCase			=				hf_deepspeed_config.config

  # Optimizer + Scheduler
  # Currently supported combos:
  # 1. DS scheduler + DS optimizer: Yes
  # 2. HF scheduler + HF optimizer: Yes
  # 3. DS scheduler + HF optimizer: Yes
  # 4. HF scheduler + DS optimizer: No
  #
  # Unless Offload is enabled in which case it's:
  # 1. DS scheduler + DS optimizer: Yes
  # 2. HF scheduler + HF optimizer: Mostly*
  # 3. DS scheduler + HF optimizer: Mostly*
  # 4. HF scheduler + DS optimizer: No
  #
  # Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

  __lowerCAmelCase			=				None
  if "optimizer" in config:
    if args.adafactor:
      raise ValueError(
          '''--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. '''
          '''Only one optimizer can be configured.''')
    __lowerCAmelCase			=				DummyOptim(params=__lowercase)
  else:
    if hf_deepspeed_config.is_offload():
      logger.info(
          '''Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the'''
          ''' custom optimizer has both CPU and GPU implementation (except LAMB)''')

    # ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
    # But trainer uses AdamW by default.
    __lowerCAmelCase			=				trainer.create_optimizer()
    # To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
    __lowerCAmelCase			=				True

  __lowerCAmelCase			=				None
  if "scheduler" in config:
    __lowerCAmelCase			=				DummyScheduler(__lowercase)
  else:
    if isinstance(__lowercase     ,							__lowercase):
      raise ValueError(
          '''Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. '''
          '''Please configure a scheduler in the DeepSpeed config.''')
    __lowerCAmelCase			=				trainer.create_scheduler(num_training_steps=__lowercase     ,							optimizer=__lowercase)

  return optimizer, lr_scheduler


def 					__SCREAMING_SNAKE_CASE  (							__lowercase:		int     ,							__lowercase:		Any     ,							__lowercase:		List[Any]=False):



  '''simple docstring'''






  from deepspeed.utils import logger as ds_logger

  __lowerCAmelCase			=				trainer.model
  __lowerCAmelCase			=				trainer.args

  __lowerCAmelCase			=				trainer.accelerator.state.deepspeed_plugin.hf_ds_config

  # resume config update - some bits like `model` and `num_training_steps` only become available during train
  hf_deepspeed_config.trainer_config_finalize(__lowercase     ,							__lowercase     ,							__lowercase)

  # set the Deepspeed log level consistent with the Trainer
  ds_logger.setLevel(args.get_process_log_level())

  if inference:
    # only Z3 makes sense for the inference
    if not hf_deepspeed_config.is_zeroa():
      raise ValueError('''ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config''')

    # in case the training config is re-used for inference
    hf_deepspeed_config.del_config_sub_tree('''optimizer''')
    hf_deepspeed_config.del_config_sub_tree('''lr_scheduler''')
    __lowerCAmelCase	,						__lowerCAmelCase			=				None, None
    __lowerCAmelCase			=				None
  else:
    __lowerCAmelCase			=				None  # important for when deepspeed_init is used as re-init
    __lowerCAmelCase			=				list(filter(lambda __lowercase: p.requires_grad     ,							model.parameters()))
    __lowerCAmelCase	,						__lowerCAmelCase			=				deepspeed_optim_sched(
        __lowercase     ,							__lowercase     ,							__lowercase     ,							__lowercase     ,							__lowercase)

  # keep for quick debug:
  # from pprint import pprint; pprint(config)

  return optimizer, lr_scheduler




def 					__SCREAMING_SNAKE_CASE  (							__lowercase:		Any     ,							__lowercase:		str):



  '''simple docstring'''






  import glob

  __lowerCAmelCase			=				sorted(glob.glob(F'''{checkpoint_path}/global_step*'''))

  if len(__lowercase) > 0:
    logger.info(F'''Attempting to resume from {checkpoint_path}''')
    # this magically updates self.optimizer and self.lr_scheduler
    __lowerCAmelCase	,						__lowerCAmelCase			=				deepspeed_engine.load_checkpoint(
        __lowercase     ,							load_optimizer_states=__lowercase     ,							load_lr_scheduler_states=__lowercase)
    if load_path is None:
      raise ValueError(F'''[deepspeed] failed to resume from checkpoint {checkpoint_path}''')
  else:
    raise ValueError(F'''Can\'t find a valid checkpoint at {checkpoint_path}''')







import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
      import torch

if is_vision_available():
      from PIL import Image

      from transformers import ViltImageProcessor

class   __snake_case				(			unittest.TestCase   ):






    '''simple docstring'''





    def __init__(   self						:		Tuple			,						snake_case						:		List[str]			,						snake_case						:		Dict=7			,						snake_case						:		str=3			,						snake_case						:		Union[str, Any]=18			,						snake_case						:		str=30			,						snake_case						:		str=400			,						snake_case						:		Optional[Any]=True			,						snake_case						:		List[str]=None			,						snake_case						:		Tuple=2			,						snake_case						:		Dict=True			,						snake_case						:		Dict=[0.5, 0.5, 0.5]			,						snake_case						:		Tuple=[0.5, 0.5, 0.5]			,						):

      """simple docstring"""





      __lowerCAmelCase			=				size if size is not None else {'''shortest_edge''': 30}
      __lowerCAmelCase			=				parent
      __lowerCAmelCase			=				batch_size
      __lowerCAmelCase			=				num_channels
      __lowerCAmelCase			=				image_size
      __lowerCAmelCase			=				min_resolution
      __lowerCAmelCase			=				max_resolution
      __lowerCAmelCase			=				do_resize
      __lowerCAmelCase			=				size
      __lowerCAmelCase			=				size_divisor
      __lowerCAmelCase			=				do_normalize
      __lowerCAmelCase			=				image_mean
      __lowerCAmelCase			=				image_std





    def 	A		(   self						:		int							):

      """simple docstring"""





      return {
          "image_mean": self.image_mean,
          "image_std": self.image_std,
          "do_normalize": self.do_normalize,
          "do_resize": self.do_resize,
          "size": self.size,
          "size_divisor": self.size_divisor,
      }





    def 	A		(   self						:		Dict			,						snake_case						:		List[Any]			,						snake_case						:		Any=False							):

      """simple docstring"""





      if not batched:
        __lowerCAmelCase			=				self.size['''shortest_edge''']
        __lowerCAmelCase			=				image_inputs[0]
        if isinstance(snake_case			,						Image.Image							):
          __lowerCAmelCase	,						__lowerCAmelCase			=				image.size
        else:
          __lowerCAmelCase	,						__lowerCAmelCase			=				image.shape[1], image.shape[2]
        __lowerCAmelCase			=				size / min(snake_case			,						snake_case							)
        if h < w:
          __lowerCAmelCase	,						__lowerCAmelCase			=				size, scale * w
        else:
          __lowerCAmelCase	,						__lowerCAmelCase			=				scale * h, size

        __lowerCAmelCase			=				int((1_333 / 800) * size							)
        if max(snake_case			,						snake_case							) > max_size:
          __lowerCAmelCase			=				max_size / max(snake_case			,						snake_case							)
          __lowerCAmelCase			=				newh * scale
          __lowerCAmelCase			=				neww * scale

        __lowerCAmelCase	,						__lowerCAmelCase			=				int(newh + 0.5							), int(neww + 0.5							)
        __lowerCAmelCase	,						__lowerCAmelCase			=				(
            newh // self.size_divisor * self.size_divisor,
            neww // self.size_divisor * self.size_divisor,
        )

      else:
        __lowerCAmelCase			=				[]
        for image in image_inputs:
          __lowerCAmelCase	,						__lowerCAmelCase			=				self.get_expected_values([image]							)
          expected_values.append((expected_height, expected_width)							)
        __lowerCAmelCase			=				max(snake_case			,						key=lambda snake_case							: item[0]							)[0]
        __lowerCAmelCase			=				max(snake_case			,						key=lambda snake_case							: item[1]							)[1]

      return expected_height, expected_width


@require_torch
@require_vision
class   __snake_case				(			_a       ,				unittest.TestCase   ):






    '''simple docstring'''
    _lowerCAmelCase     =	ViltImageProcessor if is_vision_available() else None





    def 	A		(   self						:		Union[str, Any]							):

      """simple docstring"""





      __lowerCAmelCase			=				ViltImageProcessingTester(self							)





    @property
    def 	A		(   self						:		Any							):

      """simple docstring"""





      return self.image_processor_tester.prepare_image_processor_dict()





    def 	A		(   self						:		List[Any]							):

      """simple docstring"""





      __lowerCAmelCase			=				self.image_processing_class(**self.image_processor_dict							)
      self.assertTrue(hasattr(snake_case			,						'''image_mean'''							)							)
      self.assertTrue(hasattr(snake_case			,						'''image_std'''							)							)
      self.assertTrue(hasattr(snake_case			,						'''do_normalize'''							)							)
      self.assertTrue(hasattr(snake_case			,						'''do_resize'''							)							)
      self.assertTrue(hasattr(snake_case			,						'''size'''							)							)
      self.assertTrue(hasattr(snake_case			,						'''size_divisor'''							)							)





    def 	A		(   self						:		Optional[Any]							):

      """simple docstring"""





      __lowerCAmelCase			=				self.image_processing_class.from_dict(self.image_processor_dict							)
      self.assertEqual(image_processor.size			,						{'''shortest_edge''': 30}							)

      __lowerCAmelCase			=				self.image_processing_class.from_dict(self.image_processor_dict			,						size=42							)
      self.assertEqual(image_processor.size			,						{'''shortest_edge''': 42}							)





    def 	A		(   self						:		Union[str, Any]							):

      """simple docstring"""





      pass





    def 	A		(   self						:		Union[str, Any]							):

      """simple docstring"""





      __lowerCAmelCase			=				self.image_processing_class(**self.image_processor_dict							)
      # create random PIL images
      __lowerCAmelCase			=				prepare_image_inputs(self.image_processor_tester			,						equal_resolution=snake_case							)
      for image in image_inputs:
        self.assertIsInstance(snake_case			,						Image.Image							)

      # Test not batched input
      __lowerCAmelCase			=				image_processing(image_inputs[0]			,						return_tensors='''pt'''							).pixel_values

      __lowerCAmelCase	,						__lowerCAmelCase			=				self.image_processor_tester.get_expected_values(snake_case							)
      self.assertEqual(
          encoded_images.shape			,						(1, self.image_processor_tester.num_channels, expected_height, expected_width)			,						)

      # Test batched
      __lowerCAmelCase			=				image_processing(snake_case			,						return_tensors='''pt'''							).pixel_values

      __lowerCAmelCase	,						__lowerCAmelCase			=				self.image_processor_tester.get_expected_values(snake_case			,						batched=snake_case							)
      self.assertEqual(
          encoded_images.shape			,						(
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              expected_height,
              expected_width,
          )			,						)





    def 	A		(   self						:		Tuple							):

      """simple docstring"""





      __lowerCAmelCase			=				self.image_processing_class(**self.image_processor_dict							)
      # create random numpy tensors
      __lowerCAmelCase			=				prepare_image_inputs(self.image_processor_tester			,						equal_resolution=snake_case			,						numpify=snake_case							)
      for image in image_inputs:
        self.assertIsInstance(snake_case			,						np.ndarray							)

      # Test not batched input
      __lowerCAmelCase			=				image_processing(image_inputs[0]			,						return_tensors='''pt'''							).pixel_values

      __lowerCAmelCase	,						__lowerCAmelCase			=				self.image_processor_tester.get_expected_values(snake_case							)
      self.assertEqual(
          encoded_images.shape			,						(1, self.image_processor_tester.num_channels, expected_height, expected_width)			,						)

      # Test batched
      __lowerCAmelCase			=				image_processing(snake_case			,						return_tensors='''pt'''							).pixel_values

      __lowerCAmelCase	,						__lowerCAmelCase			=				self.image_processor_tester.get_expected_values(snake_case			,						batched=snake_case							)
      self.assertEqual(
          encoded_images.shape			,						(
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              expected_height,
              expected_width,
          )			,						)





    def 	A		(   self						:		List[str]							):

      """simple docstring"""





      __lowerCAmelCase			=				self.image_processing_class(**self.image_processor_dict							)
      # create random PyTorch tensors
      __lowerCAmelCase			=				prepare_image_inputs(self.image_processor_tester			,						equal_resolution=snake_case			,						torchify=snake_case							)
      for image in image_inputs:
        self.assertIsInstance(snake_case			,						torch.Tensor							)

      # Test not batched input
      __lowerCAmelCase			=				image_processing(image_inputs[0]			,						return_tensors='''pt'''							).pixel_values

      __lowerCAmelCase	,						__lowerCAmelCase			=				self.image_processor_tester.get_expected_values(snake_case							)
      self.assertEqual(
          encoded_images.shape			,						(1, self.image_processor_tester.num_channels, expected_height, expected_width)			,						)

      # Test batched
      __lowerCAmelCase			=				image_processing(snake_case			,						return_tensors='''pt'''							).pixel_values

      __lowerCAmelCase	,						__lowerCAmelCase			=				self.image_processor_tester.get_expected_values(snake_case			,						batched=snake_case							)
      self.assertEqual(
          encoded_images.shape			,						(
              self.image_processor_tester.batch_size,
              self.image_processor_tester.num_channels,
              expected_height,
              expected_width,
          )			,						)
