





import importlib
import importlib.metadata
import os
import warnings
from distutils.util import strtobool
from functools import lru_cache

import torch
from packaging import version
from packaging.version import parse

from .environment import parse_flag_from_env
from .versions import compare_versions, is_torch_version


try:
  import torch_xla.core.xla_model as xm  # noqa: F401

  _A				:							List[Any]							= True
except ImportError:
  _A				:							Optional[int]							= False


# Cache this result has it's a C FFI call which can be pretty time-consuming
_A				:							Any							= torch.distributed.is_available()


def        __snake_case       (						lowerCAmelCase_  )							->	Any:
    # Check we're not importing a "pkg_name" directory somewhere but the actual library by trying to grab the version
    SCREAMING_SNAKE_CASE__       							= importlib.util.find_spec(lowerCAmelCase_  ) is not None
    if package_exists:
        try:
            SCREAMING_SNAKE_CASE__       							= importlib.metadata.metadata(lowerCAmelCase_  )
            return True
        except importlib.metadata.PackageNotFoundError:
            return False


def        __snake_case       (						)							->	bool:
    return _torch_distributed_available


def        __snake_case       (						)							->	Optional[int]:
    try:
        pass
    except ImportError:
        print(
            '''Intel(R) oneCCL Bindings for PyTorch* is required to run DDP on Intel(R) GPUs, but it is not'''
            ''' detected. If you see \"ValueError: Invalid backend: \'ccl\'\" error, please install Intel(R) oneCCL'''
            ''' Bindings for PyTorch*.'''  )
    return (
        importlib.util.find_spec('''torch_ccl'''  ) is not None
        or importlib.util.find_spec('''oneccl_bindings_for_pytorch'''  ) is not None
    )


def        __snake_case       (						)							->	Dict:
    return importlib.metadata.version('''oneccl_bind_pt'''  )


def        __snake_case       (						)							->	List[Any]:
    return _is_package_available('''transformer_engine'''  )


@lru_cache
def        __snake_case       (						lowerCAmelCase_=True  )							->	Optional[int]:
    # Due to bugs on the amp series GPUs, we disable torch-xla on them
    if torch.cuda.is_available():
        return False
    if _tpu_available and check_device:
        try:
            # Will raise a RuntimeError if no XLA configuration is found
            SCREAMING_SNAKE_CASE__       							= xm.xla_device()
            return True
        except RuntimeError:
            return False
    return _tpu_available


def        __snake_case       (						)							->	Union[str, Any]:
    return _is_package_available('''deepspeed'''  )


def        __snake_case       (						lowerCAmelCase_=False  )							->	Tuple:
    if is_tpu_available():
        return not ignore_tpu
    if torch.cuda.is_available():
        return torch.cuda.is_bfaa_supported()
    if is_npu_available():
        return False
    return True


def        __snake_case       (						)							->	Dict:
    SCREAMING_SNAKE_CASE__       							= _is_package_available('''bitsandbytes'''  )
    if package_exists:
        SCREAMING_SNAKE_CASE__       							= version.parse(importlib.metadata.version('''bitsandbytes'''  )  )
        return compare_versions(lowerCAmelCase_   ,       '''>='''   ,       '''0.39.0'''  )
    return False


def        __snake_case       (						)							->	Tuple:
    SCREAMING_SNAKE_CASE__       							= _is_package_available('''bitsandbytes'''  )
    if package_exists:
        SCREAMING_SNAKE_CASE__       							= version.parse(importlib.metadata.version('''bitsandbytes'''  )  )
        return compare_versions(lowerCAmelCase_   ,       '''>='''   ,       '''0.37.2'''  )
    return False


def        __snake_case       (						)							->	Tuple:
    return _is_package_available('''bitsandbytes'''  )


def        __snake_case       (						)							->	Any:
    if strtobool(os.environ.get('''ACCELERATE_USE_MEGATRON_LM'''   ,       '''False'''  )  ) == 1:
        SCREAMING_SNAKE_CASE__       							= importlib.util.find_spec('''megatron'''  ) is not None
        if package_exists:
            try:
                SCREAMING_SNAKE_CASE__       							= parse(importlib.metadata.version('''megatron-lm'''  )  )
                return compare_versions(lowerCAmelCase_   ,       '''>='''   ,       '''2.2.0'''  )
            except Exception as e:
                warnings.warn(f'''Parse Megatron version failed. Exception:{e}'''  )
                return False


def        __snake_case       (						)							->	Optional[Any]:
    return _is_package_available('''safetensors'''  )


def        __snake_case       (						)							->	int:
    return _is_package_available('''transformers'''  )


def        __snake_case       (						)							->	Dict:
    return _is_package_available('''datasets'''  )


def        __snake_case       (						)							->	Dict:
    return _is_package_available('''aim'''  )


def        __snake_case       (						)							->	Union[str, Any]:
    return _is_package_available('''tensorboard'''  ) or _is_package_available('''tensorboardX'''  )


def        __snake_case       (						)							->	Tuple:
    return _is_package_available('''wandb'''  )


def        __snake_case       (						)							->	Optional[int]:
    return _is_package_available('''comet_ml'''  )


def        __snake_case       (						)							->	str:
    return _is_package_available('''boto3'''  )


def        __snake_case       (						)							->	List[Any]:
    if _is_package_available('''rich'''  ):
        if "ACCELERATE_DISABLE_RICH" in os.environ:
            warnings.warn(
                '''`ACCELERATE_DISABLE_RICH` is deprecated and will be removed in v0.22.0 and deactivated by default. Please use `ACCELERATE_ENABLE_RICH` if you wish to use `rich`.'''  )
            return not parse_flag_from_env('''ACCELERATE_DISABLE_RICH'''   ,       lowerCAmelCase_  )
        return parse_flag_from_env('''ACCELERATE_ENABLE_RICH'''   ,       lowerCAmelCase_  )
    return False


def        __snake_case       (						)							->	int:
    return _is_package_available('''sagemaker'''  )


def        __snake_case       (						)							->	List[str]:
    return _is_package_available('''tqdm'''  )


def        __snake_case       (						)							->	Union[str, Any]:
    return _is_package_available('''mlflow'''  )


def        __snake_case       (						)							->	Optional[Any]:
    return is_torch_version('''>='''   ,       '''1.12'''  ) and torch.backends.mps.is_available() and torch.backends.mps.is_built()


def        __snake_case       (						)							->	Optional[Any]:
    def get_major_and_minor_from_version(lowerCAmelCase_  ):
        return str(version.parse(lowerCAmelCase_  ).major  ) + "." + str(version.parse(lowerCAmelCase_  ).minor  )

    SCREAMING_SNAKE_CASE__       							= importlib.metadata.version('''torch'''  )
    if importlib.util.find_spec('''intel_extension_for_pytorch'''  ) is None:
        return False
    SCREAMING_SNAKE_CASE__       							= '''N/A'''
    try:
        SCREAMING_SNAKE_CASE__       							= importlib.metadata.version('''intel_extension_for_pytorch'''  )
    except importlib.metadata.PackageNotFoundError:
        return False
    SCREAMING_SNAKE_CASE__       							= get_major_and_minor_from_version(_torch_version  )
    SCREAMING_SNAKE_CASE__       							= get_major_and_minor_from_version(_ipex_version  )
    if torch_major_and_minor != ipex_major_and_minor:
        warnings.warn(
            f'''Intel Extension for PyTorch {ipex_major_and_minor} needs to work with PyTorch {ipex_major_and_minor}.*,'''
            f''' but PyTorch {_torch_version} is found. Please switch to the matching version and run again.'''  )
        return False
    return True


@lru_cache
def        __snake_case       (						lowerCAmelCase_=False  )							->	List[str]:
    if importlib.util.find_spec('''torch'''  ) is None or importlib.util.find_spec('''torch_npu'''  ) is None:
        return False

    import torch
    import torch_npu  # noqa: F401

    if check_device:
        try:
            # Will raise a RuntimeError if no NPU is found
            SCREAMING_SNAKE_CASE__       							= torch.npu.device_count()
            return torch.npu.is_available()
        except RuntimeError:
            return False
    return hasattr(lowerCAmelCase_   ,       '''npu'''  ) and torch.npu.is_available()



@lru_cache
def        __snake_case       (						lowerCAmelCase_=False  )							->	Union[str, Any]:
    if not parse_flag_from_env('''ACCELERATE_USE_XPU'''   ,       default=lowerCAmelCase_  ):
        return False
    if is_ipex_available():
        import torch

        if is_torch_version('''<='''   ,       '''1.12'''  ):
            return False
    else:
        return False

    import intel_extension_for_pytorch  # noqa: F401

    if check_device:
        try:
            # Will raise a RuntimeError if no XPU  is found
            SCREAMING_SNAKE_CASE__       							= torch.xpu.device_count()
            return torch.xpu.is_available()
        except RuntimeError:
            return False
    return hasattr(lowerCAmelCase_   ,       '''xpu'''  ) and torch.xpu.is_available()







import os
from dataclasses import replace

import jax
import wandb
from bigbird_flax import Args, DataCollator, FlaxBigBirdForNaturalQuestions, Trainer, build_tx, train_step, val_step
from datasets import load_dataset
from flax import jax_utils

from transformers import BigBirdTokenizerFast


if __name__ == "__main__":
  print("""#################### AVAILABLE DEVICES ####################""")
  print(jax.devices())
  print("""###########################################################""")

  # setup for wandb sweep
  _A				:							Any							= Args()
  _A				:							Union[str, Any]							= wandb.init(project="""bigbird-natural-questions""", config=args.__dict__)
  _A				:							Optional[Any]							= dict(logger.config)
  del wandb_args["batch_size"]
  _A				:							Any							= replace(args, **wandb_args)
  _A				:							Dict							= args.base_dir + """-""" + wandb.run.id
  _A				:							Dict							= replace(args, base_dir=base_dir)
  print(args)

  _A				:							Tuple							= load_dataset("""json""", data_files=args.tr_data_path)["""train"""]
  _A				:							Optional[Any]							= load_dataset("""json""", data_files=args.val_data_path)["""train"""]

  # drop extra batch for now
  _A				:							Dict							= range(len(tr_dataset) - len(tr_dataset) % args.batch_size)
  _A				:							int							= tr_dataset.shuffle().select(indices)
  _A				:							Optional[int]							= range(len(val_dataset) - len(val_dataset) % args.batch_size)
  _A				:							Dict							= val_dataset.shuffle().select(indices)

  if os.environ.get("""TRAIN_ON_SMALL""", """false""") == "true":
    _A				:							Optional[Any]							= tr_dataset.shuffle().select(range(8_00_00))
    _A				:							Tuple							= val_dataset.shuffle().select(range(80_00))

  print(tr_dataset)
  print(val_dataset)

  _A				:							int							= FlaxBigBirdForNaturalQuestions.from_pretrained(
      args.model_id, block_size=args.block_size, num_random_blocks=args.num_random_blocks
  )
  _A				:							Any							= BigBirdTokenizerFast.from_pretrained(args.model_id)
  _A				:							Optional[Any]							= DataCollator(pad_id=tokenizer.pad_token_id, max_length=40_96)

  _A				:							int							= {
      """lr""": args.lr,
      """init_lr""": args.init_lr,
      """warmup_steps""": args.warmup_steps,
      """num_train_steps""": args.max_epochs * (len(tr_dataset) // args.batch_size),
      """weight_decay""": args.weight_decay,
  }
  _A,					_A				:							Union[str, Any]							= build_tx(**tx_args)

  _A				:							str							= Trainer(
      args=args,
      data_collator=data_collator,
      model_save_fn=model.save_pretrained,
      train_step_fn=train_step,
      val_step_fn=val_step,
      logger=logger,
      scheduler_fn=lr,
  )

  _A				:							Tuple							= None
  _A				:							Dict							= trainer.create_state(model, tx, num_train_steps=tx_args["""num_train_steps"""], ckpt_dir=ckpt_dir)
  try:
    trainer.train(state, tr_dataset, val_dataset)
  except KeyboardInterrupt:
    print("""Oooops; TRAINING STOPPED UNFORTUNATELY""")

  print("""SAVING WEIGHTS IN `final-weights`""")
  _A				:							Tuple							= jax_utils.unreplicate(state.params)
  model.save_pretrained(os.path.join(args.base_dir, """final-weights"""), params=params)
