

"""simple docstring"""





import importlib.util
import weakref
from functools import partialmethod

from .dependency_versions_check import dep_version_check
from .utils import is_accelerate_available, is_torch_available, logging


if is_torch_available():
   import torch

_a				   =							logging.get_logger(__name__)




def 						_A						(						)			->		List[Any]:
   '''simple docstring'''


   return importlib.util.find_spec("deepspeed") is not None


if is_accelerate_available() and is_deepspeed_available():
   from accelerate.utils.deepspeed import HfDeepSpeedConfig as DeepSpeedConfig
else:
   # Inherits from a dummy `object` if accelerate is not available, so that python succeeds to import this file.
   # Deepspeed glue code will never inherit this dummy object as it checks if accelerate is available.
   from builtins import object as DeepSpeedConfig




class       _lowerCAmelCase						(			__lowerCamelCase     ):







 """simple docstring"""







 def __init__(    self							:    str,      UpperCAmelCase__							:    Union[str, Any]			):
    set_hf_deepspeed_config(self			)
    dep_version_check("accelerate"			)
    dep_version_check("deepspeed"			)
    super().__init__(UpperCAmelCase__			)







class       _lowerCAmelCase						(			__lowerCamelCase     ):







 """simple docstring"""







 def __init__(    self							:    str,      UpperCAmelCase__							:    List[Any]			):
    super().__init__(UpperCAmelCase__			)
    __lowercase			=		None
    __lowercase			=		[]


 def       _lowercase				(    self							:    Dict			):






    if self._dtype is None:
       raise ValueError("trainer_config_process() wasn\'t called yet to tell dtype"			)
    return self._dtype


 def       _lowercase				(    self							:    List[Any],      UpperCAmelCase__							:    str			):
    __lowercase			=		self.get_value(UpperCAmelCase__			)
    if val is None:
       return False
    else:
       return val == "auto"


 def       _lowercase				(    self							:    int,      UpperCAmelCase__							:    List[str],      UpperCAmelCase__							:    str,      UpperCAmelCase__							:    Any=None,      UpperCAmelCase__							:    Optional[Any]=True			):
    __lowercase	,__lowercase			=		self.find_config_node(UpperCAmelCase__			)
    if config is None:
       return

    if config.get(UpperCAmelCase__			) == "auto":
       __lowercase			=		hf_val
       return

    if not must_match:
       return

    __lowercase			=		config.get(UpperCAmelCase__			)
    if ds_val is not None and ds_val != hf_val:
       self.mismatches.append(F"""- ds {ds_key_long}={ds_val} vs hf {hf_key}={hf_val}"""			)

 __UpperCAmelCase			:      Union[str, Any]     	=			partialmethod(__lowerCamelCase  ,must_match=__lowerCamelCase     )


 def       _lowercase				(    self							:    List[str],      UpperCAmelCase__							:    Any			):
    __lowercase			=		args.world_size * args.per_device_train_batch_size * args.gradient_accumulation_steps
    self.fill_match(
        "train_micro_batch_size_per_gpu",      args.per_device_train_batch_size,      "per_device_train_batch_size"			)
    self.fill_match("gradient_accumulation_steps",      args.gradient_accumulation_steps,      "gradient_accumulation_steps"			)
    self.fill_match("train_batch_size",      UpperCAmelCase__,      "train_batch_size (calculated)"			)
    self.fill_match("gradient_clipping",      args.max_grad_norm,      "max_grad_norm"			)

    self.fill_match("optimizer.params.lr",      args.learning_rate,      "learning_rate"			)
    self.fill_match("optimizer.params.betas",      [args.adam_betaa, args.adam_betaa],      "adam_beta1+adam_beta2"			)
    self.fill_match("optimizer.params.eps",      args.adam_epsilon,      "adam_epsilon"			)
    self.fill_match("optimizer.params.weight_decay",      args.weight_decay,      "weight_decay"			)

    self.fill_only("scheduler.params.warmup_min_lr",      0			)  # not a trainer arg
    self.fill_match("scheduler.params.warmup_max_lr",      args.learning_rate,      "learning_rate"			)
    # total_num_steps - will get set in trainer_config_finalize

    # fp16
    if args.fpaa or args.fpaa_full_eval:
       __lowercase			=		"apex" if args.fpaa_backend == "apex" else "amp"
    else:
       __lowercase			=		None

    if args.save_on_each_node:
       # deepspeed uses shared storage by default. Let's override this setting if save_on_each_node == True
       __lowercase			=		self.config.get("checkpoint",      {}			)
       __lowercase			=		args.save_on_each_node

    # amp: similar to the pytorch native amp - it has a bunch of optional params but we won't set
    # any here unless the user did the work
    self.fill_match(
        "fp16.enabled",      ((args.fpaa or args.fpaa_full_eval) and fpaa_backend == "amp"),      "fp16|fp16_full_eval+fp16_backend(amp)",      )

    # apex: delegates amp work to apex (which needs to be available), but it cannot be used with any
    # ZeRO features
    self.fill_match("amp.enabled",      fpaa_backend == "apex",      "fp16+fp16_backend(apex)"			)
    self.fill_match("amp.opt_level",      args.fpaa_opt_level,      "fp16_opt_level"			)

    self.fill_match("bf16.enabled",      (args.bfaa or args.bfaa_full_eval),      "bf16|bf16_full_eval"			)

    # deepspeed's default mode is fp16 unless there is a config that says differently
    if self.is_true("bf16.enabled"			):
       __lowercase			=		torch.bfloataa
    elif self.is_false("fp16.enabled"			):
       __lowercase			=		torch.floataa
    else:
       __lowercase			=		torch.floataa



 def       _lowercase				(    self							:    Any,      UpperCAmelCase__							:    Any,      UpperCAmelCase__							:    str,      UpperCAmelCase__							:    Any			):
    __lowercase			=		[
        "zero_optimization.reduce_bucket_size",
        "zero_optimization.stage3_prefetch_bucket_size",
        "zero_optimization.stage3_param_persistence_threshold",
    ]
    __lowercase			=		[x for x in hidden_size_based_keys if self.is_auto(UpperCAmelCase__			)]

    if len(UpperCAmelCase__			) > 0:
       if hasattr(model.config,      "hidden_size"			):
          __lowercase			=		model.config.hidden_size
       elif hasattr(model.config,      "hidden_sizes"			):
          # if there are many hidden sizes pick the largest one
          __lowercase			=		max(model.config.hidden_sizes			)
       else:
          raise ValueError(
              "The model\'s config file has neither `hidden_size` nor `hidden_sizes` entry, "
              "therefore it\'s not possible to automatically fill out the following `auto` entries "
              F"""in the DeepSpeed config file: {hidden_size_auto_keys}. You can fix that by replacing """
              "`auto` values for these keys with an integer value of your choice."			)

       self.fill_only("zero_optimization.reduce_bucket_size",      hidden_size * hidden_size			)
       if self.is_zeroa():
          # automatically assign the optimal config values based on model config
          self.fill_only("zero_optimization.stage3_prefetch_bucket_size",      0.9 * hidden_size * hidden_size			)
          self.fill_only("zero_optimization.stage3_param_persistence_threshold",      1_0 * hidden_size			)

        # scheduler
    self.fill_match("scheduler.params.total_num_steps",      UpperCAmelCase__,      "num_training_steps (calculated)"			)
    self.fill_match("scheduler.params.warmup_num_steps",      args.get_warmup_steps(UpperCAmelCase__			),      "warmup_steps"			)

    if len(self.mismatches			) > 0:
       __lowercase			=		"\n".join(self.mismatches			)
       raise ValueError(
           "Please correct the following DeepSpeed config values that mismatch TrainingArguments"
           F""" values:\n{mismatches}\nThe easiest method is to set these DeepSpeed config values to 'auto'."""			)







_a				   =							None




def 						_A						(						UpperCamelCase_  : Union[str, Any])			->		List[Any]:
   '''simple docstring'''


   global _hf_deepspeed_config_weak_ref
   # will go away automatically when HfDeepSpeedConfig is destroyed (when TrainingArguments is destroyed)
   __lowercase			=		weakref.ref(UpperCamelCase_)




def 						_A						(						)			->		Optional[int]:
   '''simple docstring'''


   global _hf_deepspeed_config_weak_ref
   __lowercase			=		None




def 						_A						(						)			->		Dict:
   '''simple docstring'''


   if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
      return _hf_deepspeed_config_weak_ref().is_zeroa()
   else:
      return False




def 						_A						(						)			->		Union[str, Any]:
   '''simple docstring'''


   if _hf_deepspeed_config_weak_ref is not None and _hf_deepspeed_config_weak_ref() is not None:
      return _hf_deepspeed_config_weak_ref().config
   else:
      return None




def 						_A						(						UpperCamelCase_  : Optional[Any],    UpperCamelCase_  : str,    UpperCamelCase_  : Any,    UpperCamelCase_  : List[str],    UpperCamelCase_  : str)			->		int:
   '''simple docstring'''


   from accelerate.utils import DummyOptim, DummyScheduler

   __lowercase			=		hf_deepspeed_config.config

   # Optimizer + Scheduler
   # Currently supported combos:
   # 1. DS scheduler + DS optimizer: Yes
   # 2. HF scheduler + HF optimizer: Yes
   # 3. DS scheduler + HF optimizer: Yes
   # 4. HF scheduler + DS optimizer: No
   #
   # Unless Offload is enabled in which case it's:
   # 1. DS scheduler + DS optimizer: Yes
   # 2. HF scheduler + HF optimizer: Mostly*
   # 3. DS scheduler + HF optimizer: Mostly*
   # 4. HF scheduler + DS optimizer: No
   #
   # Mostly*: All non-native DeepSpeed optimizers that have both CPU and GPU implementation should work (except LAMB)

   __lowercase			=		None
   if "optimizer" in config:
      if args.adafactor:
         raise ValueError(
             "--adafactor was passed, but also found `optimizer` configured in the DeepSpeed config. "
             "Only one optimizer can be configured.")
      __lowercase			=		DummyOptim(params=UpperCamelCase_)
   else:
      if hf_deepspeed_config.is_offload():
         logger.info(
             "Detected ZeRO Offload and non-DeepSpeed optimizers: This combination should work as long as the"
             " custom optimizer has both CPU and GPU implementation (except LAMB)")

      # ds supports Adam, OneBitAdam, and Lamb optimizers and can import other optimizers from torch.
      # But trainer uses AdamW by default.
      __lowercase			=		trainer.create_optimizer()
      # To use other optimizers requires voiding warranty with: `zero_allow_untested_optimizer`
      __lowercase			=		True

   __lowercase			=		None
   if "scheduler" in config:
      __lowercase			=		DummyScheduler(UpperCamelCase_)
   else:
      if isinstance(UpperCamelCase_,    UpperCamelCase_):
         raise ValueError(
             "Found `optimizer` configured in the DeepSpeed config, but no `scheduler`. "
             "Please configure a scheduler in the DeepSpeed config.")
      __lowercase			=		trainer.create_scheduler(num_training_steps=UpperCamelCase_,    optimizer=UpperCamelCase_)

   return optimizer, lr_scheduler




def 						_A						(						UpperCamelCase_  : List[Any],    UpperCamelCase_  : Optional[Any],    UpperCamelCase_  : List[str]=False)			->		Optional[int]:
   '''simple docstring'''


   from deepspeed.utils import logger as ds_logger

   __lowercase			=		trainer.model
   __lowercase			=		trainer.args

   __lowercase			=		trainer.accelerator.state.deepspeed_plugin.hf_ds_config

   # resume config update - some bits like `model` and `num_training_steps` only become available during train
   hf_deepspeed_config.trainer_config_finalize(UpperCamelCase_,    UpperCamelCase_,    UpperCamelCase_)

   # set the Deepspeed log level consistent with the Trainer
   ds_logger.setLevel(args.get_process_log_level())

   if inference:
      # only Z3 makes sense for the inference
      if not hf_deepspeed_config.is_zeroa():
         raise ValueError("ZeRO inference only makes sense with ZeRO Stage 3 - please adjust your config")

      # in case the training config is re-used for inference
      hf_deepspeed_config.del_config_sub_tree("optimizer")
      hf_deepspeed_config.del_config_sub_tree("lr_scheduler")
      __lowercase	,__lowercase			=		None, None
      __lowercase			=		None
   else:
      __lowercase			=		None  # important for when deepspeed_init is used as re-init
      __lowercase			=		list(filter(lambda UpperCamelCase_: p.requires_grad,    model.parameters()))
      __lowercase	,__lowercase			=		deepspeed_optim_sched(
          UpperCamelCase_,    UpperCamelCase_,    UpperCamelCase_,    UpperCamelCase_,    UpperCamelCase_)

   # keep for quick debug:
   # from pprint import pprint; pprint(config)

   return optimizer, lr_scheduler




def 						_A						(						UpperCamelCase_  : Dict,    UpperCamelCase_  : int)			->		int:
   '''simple docstring'''


   import glob

   __lowercase			=		sorted(glob.glob(F"""{checkpoint_path}/global_step*"""))

   if len(UpperCamelCase_) > 0:
      logger.info(F"""Attempting to resume from {checkpoint_path}""")
      # this magically updates self.optimizer and self.lr_scheduler
      __lowercase	,__lowercase			=		deepspeed_engine.load_checkpoint(
          UpperCamelCase_,    load_optimizer_states=UpperCamelCase_,    load_lr_scheduler_states=UpperCamelCase_)
      if load_path is None:
         raise ValueError(F"""[deepspeed] failed to resume from checkpoint {checkpoint_path}""")
   else:
      raise ValueError(F"""Can't find a valid checkpoint at {checkpoint_path}""")







"""simple docstring"""





import os
from dataclasses import replace

import jax
import wandb
from bigbird_flax import Args, DataCollator, FlaxBigBirdForNaturalQuestions, Trainer, build_tx, train_step, val_step
from datasets import load_dataset
from flax import jax_utils

from transformers import BigBirdTokenizerFast


if __name__ == "__main__":
   print('#################### AVAILABLE DEVICES ####################')
   print(jax.devices())
   print('###########################################################')

   # setup for wandb sweep
   _a				   =							Args()
   _a				   =							wandb.init(project='bigbird-natural-questions', config=args.__dict__)
   _a				   =							dict(logger.config)
   del wandb_args["batch_size"]
   _a				   =							replace(args, **wandb_args)
   _a				   =							args.base_dir + '-' + wandb.run.id
   _a				   =							replace(args, base_dir=base_dir)
   print(args)

   _a				   =							load_dataset('json', data_files=args.tr_data_path)['train']
   _a				   =							load_dataset('json', data_files=args.val_data_path)['train']

   # drop extra batch for now
   _a				   =							range(len(tr_dataset) - len(tr_dataset) % args.batch_size)
   _a				   =							tr_dataset.shuffle().select(indices)
   _a				   =							range(len(val_dataset) - len(val_dataset) % args.batch_size)
   _a				   =							val_dataset.shuffle().select(indices)

   if os.environ.get('TRAIN_ON_SMALL', 'false') == "true":
      _a				   =							tr_dataset.shuffle().select(range(8_00_00))
      _a				   =							val_dataset.shuffle().select(range(80_00))

   print(tr_dataset)
   print(val_dataset)

   _a				   =							FlaxBigBirdForNaturalQuestions.from_pretrained(
       args.model_id, block_size=args.block_size, num_random_blocks=args.num_random_blocks
   )
   _a				   =							BigBirdTokenizerFast.from_pretrained(args.model_id)
   _a				   =							DataCollator(pad_id=tokenizer.pad_token_id, max_length=40_96)

   _a				   =							{
       'lr': args.lr,
       'init_lr': args.init_lr,
       'warmup_steps': args.warmup_steps,
       'num_train_steps': args.max_epochs * (len(tr_dataset) // args.batch_size),
       'weight_decay': args.weight_decay,
   }
   _a   ,		_a				   =							build_tx(**tx_args)

   _a				   =							Trainer(
       args=args,
       data_collator=data_collator,
       model_save_fn=model.save_pretrained,
       train_step_fn=train_step,
       val_step_fn=val_step,
       logger=logger,
       scheduler_fn=lr,
   )

   _a				   =							None
   _a				   =							trainer.create_state(model, tx, num_train_steps=tx_args['num_train_steps'], ckpt_dir=ckpt_dir)
   try:
      trainer.train(state, tr_dataset, val_dataset)
   except KeyboardInterrupt:
      print('Oooops; TRAINING STOPPED UNFORTUNATELY')

   print('SAVING WEIGHTS IN `final-weights`')
   _a				   =							jax_utils.unreplicate(state.params)
   model.save_pretrained(os.path.join(args.base_dir, 'final-weights'), params=params)




