






import argparse
import os

import evaluate
import torch
from datasets import load_dataset
from torch.optim import AdamW
from torch.utils.data import DataLoader
from transformers import AutoModelForSequenceClassification, AutoTokenizer, get_linear_schedule_with_warmup, set_seed

from accelerate import Accelerator, DistributedType


########################################################################
# This is a fully working simple example to use Accelerate,
# specifically showcasing the checkpointing capability,
# and builds off the `nlp_example.py` script.
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#
# To help focus on the differences in the code, building `DataLoaders`
# was refactored into its own function.
# New additions from the base script can be found quickly by
# looking for the # New Code # tags
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################

UpperCAmelCase__          =	16
UpperCAmelCase__          =	32
def 		_a					(   a				:List[str]			,							a				:Tuple = 16				)							->    List[Any]:
 a							=   AutoTokenizer.from_pretrained('''bert-base-cased'''				)
 a							=   load_dataset('''glue'''			,							'''mrpc'''				)

 def tokenize_function(a				:List[Any]				):
  # max_length=None => use the model max length (it's actually the default)
  a							=   tokenizer(examples['''sentence1''']			,							examples['''sentence2''']			,							truncation=__A			,							max_length=__A				)
  return outputs

 # Apply the method we just defined to all the examples in all the splits of the dataset
 # starting with the main process first:
 with accelerator.main_process_first():
  a							=   datasets.map(
      __A			,							batched=__A			,							remove_columns=['''idx''', '''sentence1''', '''sentence2''']			,							)

 # We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
 # transformers library
 a							=   tokenized_datasets.rename_column('''label'''			,							'''labels'''				)

 def collate_fn(a				:str				):
  # On TPU it's best to pad everything to the same length or training will be very slow.
  a							=   128 if accelerator.distributed_type == DistributedType.TPU else None
  # When using mixed precision we want round multiples of 8/16
  if accelerator.mixed_precision == "fp8":
   a							=   16
  elif accelerator.mixed_precision != "no":
   a							=   8
  else:
   a							=   None

  return tokenizer.pad(
      __A			,							padding='''longest'''			,							max_length=__A			,							pad_to_multiple_of=__A			,							return_tensors='''pt'''			,							)

 # Instantiate dataloaders.
 a							=   DataLoader(
     tokenized_datasets['''train''']			,							shuffle=__A			,							collate_fn=__A			,							batch_size=__A				)
 a							=   DataLoader(
     tokenized_datasets['''validation''']			,							shuffle=__A			,							collate_fn=__A			,							batch_size=__A				)

 return train_dataloader, eval_dataloader


# For testing only
if os.environ.get("TESTING_MOCKED_DATALOADERS", None) == "1":
     from accelerate.test_utils.training import mocked_dataloaders

     UpperCAmelCase__          =	mocked_dataloaders  # noqa: F811
def 		_a					(   a				:int			,							a				:Optional[Any]				)							->    Any:



 if os.environ.get('''TESTING_MOCKED_DATALOADERS'''			,							__A				) == "1":
  a							=   2
 # Initialize accelerator
 a							=   Accelerator(cpu=args.cpu			,							mixed_precision=args.mixed_precision				)
 # Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
 a							=   config['''lr''']
 a							=   int(config['''num_epochs''']				)
 a							=   int(config['''seed''']				)
 a							=   int(config['''batch_size''']				)

 # New Code #
 # Parse out whether we are saving every epoch or after a certain number of batches
 if hasattr(args.checkpointing_steps			,							'''isdigit'''				):
  if args.checkpointing_steps == "epoch":
   a							=   args.checkpointing_steps
  elif args.checkpointing_steps.isdigit():
   a							=   int(args.checkpointing_steps				)
  else:
   raise ValueError(
       F"""Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed."""				)
 else:
  a							=   None

 set_seed(__A				)

 a						,  a							=   get_dataloaders(__A			,							__A				)
 a							=   evaluate.load('''glue'''			,							'''mrpc'''				)

 # If the batch size is too big we use gradient accumulation
 a							=   1
 if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
  a							=   batch_size // MAX_GPU_BATCH_SIZE
  a							=   MAX_GPU_BATCH_SIZE

 # Instantiate the model (we build the model here so that the seed also control new weights initialization)
 a							=   AutoModelForSequenceClassification.from_pretrained('''bert-base-cased'''			,							return_dict=__A				)

 # We could avoid this line since the accelerator is set with `device_placement=True` (default value).
 # Note that if you are placing tensors on devices manually, this line absolutely needs to be before the optimizer
 # creation otherwise training will not work on TPU (`accelerate` will kindly throw an error to make us aware of that).
 a							=   model.to(accelerator.device				)

 # Instantiate optimizer
 a							=   AdamW(params=model.parameters()			,							lr=__A				)

 # Instantiate scheduler
 a							=   get_linear_schedule_with_warmup(
     optimizer=__A			,							num_warmup_steps=100			,							num_training_steps=(len(__A				) * num_epochs) // gradient_accumulation_steps			,							)

 # Prepare everything
 # There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
 # prepare method.
 a						,  a						,  a						,  a						,  a							=   accelerator.prepare(
     __A			,							__A			,							__A			,							__A			,							__A				)

 # New Code #
 # We need to keep track of how many total steps we have iterated over
 a							=   0
 # We also need to keep track of the stating epoch so files are named properly
 a							=   0

 # We need to load the checkpoint back in before training here with `load_state`
 # The total number of epochs is adjusted based on where the state is being loaded from,
 # as we assume continuation of the same training script
 if args.resume_from_checkpoint:
  if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
   accelerator.print(F"""Resumed from checkpoint: {args.resume_from_checkpoint}"""				)
   accelerator.load_state(args.resume_from_checkpoint				)
   a							=   os.path.basename(args.resume_from_checkpoint				)
  else:
   # Get the most recent checkpoint
   a							=   [f.name for f in os.scandir(os.getcwd()				) if f.is_dir()]
   dirs.sort(key=os.path.getctime				)
   a							=   dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
  # Extract `epoch_{i}` or `step_{i}`
  a							=   os.path.splitext(__A				)[0]

  if "epoch" in training_difference:
   a							=   int(training_difference.replace('''epoch_'''			,							''''''				)				) + 1
   a							=   None
  else:
   a							=   int(training_difference.replace('''step_'''			,							''''''				)				)
   a							=   resume_step // len(__A				)
   resume_step -= starting_epoch * len(__A				)

    # Now we train the model
 for epoch in range(__A			,							__A				):
  model.train()
  # New Code #
  if args.resume_from_checkpoint and epoch == starting_epoch and resume_step is not None:
   # We need to skip steps until we reach the resumed step
   a							=   accelerator.skip_first_batches(__A			,							__A				)
   overall_step += resume_step
  else:
   # After the first iteration though, we need to go back to the original dataloader
   a							=   train_dataloader
  for step, batch in enumerate(__A				):
   # We could avoid this line since we set the accelerator with `device_placement=True`.
   batch.to(accelerator.device				)
   a							=   model(**__A				)
   a							=   outputs.loss
   a							=   loss / gradient_accumulation_steps
   accelerator.backward(__A				)
   if step % gradient_accumulation_steps == 0:
    optimizer.step()
    lr_scheduler.step()
    optimizer.zero_grad()
   # New Code #
   overall_step += 1

   # New Code #
   # We save the model, optimizer, lr_scheduler, and seed states by calling `save_state`
   # These are saved to folders named `step_{overall_step}`
   # Will contain files: "pytorch_model.bin", "optimizer.bin", "scheduler.bin", and "random_states.pkl"
   # If mixed precision was used, will also save a "scalar.bin" file
   if isinstance(__A			,							__A				):
    a							=   F"""step_{overall_step}"""
    if overall_step % checkpointing_steps == 0:
     if args.output_dir is not None:
      a							=   os.path.join(args.output_dir			,							__A				)
     accelerator.save_state(__A				)

  model.eval()
  for step, batch in enumerate(__A				):
   # We could avoid this line since we set the accelerator with `device_placement=True` (the default).
   batch.to(accelerator.device				)
   with torch.no_grad():
    a							=   model(**__A				)
   a							=   outputs.logits.argmax(dim=-1				)
   a						,  a							=   accelerator.gather_for_metrics((predictions, batch['''labels'''])				)
   metric.add_batch(
       predictions=__A			,							references=__A			,							)

  a							=   metric.compute()
  # Use accelerator.print to print only on the main process.
  accelerator.print(F"""epoch {epoch}:"""			,							__A				)

  # New Code #
  # We save the model, optimizer, lr_scheduler, and seed states by calling `save_state`
  # These are saved to folders named `epoch_{epoch}`
  # Will contain files: "pytorch_model.bin", "optimizer.bin", "scheduler.bin", and "random_states.pkl"
  # If mixed precision was used, will also save a "scalar.bin" file
  if checkpointing_steps == "epoch":
   a							=   F"""epoch_{epoch}"""
   if args.output_dir is not None:
    a							=   os.path.join(args.output_dir			,							__A				)
   accelerator.save_state(__A				)
def 		_a					(   )							->    List[Any]:
 a							=   argparse.ArgumentParser(description='''Simple example of training script.'''				)
 parser.add_argument(
     '''--mixed_precision'''			,							type=__A			,							default=__A			,							choices=['''no''', '''fp16''', '''bf16''', '''fp8''']			,							help='''Whether to use mixed precision. Choose'''
     '''between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10.'''
     '''and an Nvidia Ampere GPU.'''			,							)
 parser.add_argument('''--cpu'''			,							action='''store_true'''			,							help='''If passed, will train on the CPU.'''				)
 parser.add_argument(
     '''--checkpointing_steps'''			,							type=__A			,							default=__A			,							help='''Whether the various states should be saved at the end of every n steps, or \'epoch\' for each epoch.'''			,							)
 parser.add_argument(
     '''--output_dir'''			,							type=__A			,							default='''.'''			,							help='''Optional save directory where all checkpoint folders will be stored. Default is the current working directory.'''			,							)
 parser.add_argument(
     '''--resume_from_checkpoint'''			,							type=__A			,							default=__A			,							help='''If the training should continue from a checkpoint folder.'''			,							)
 a							=   parser.parse_args()
 a							=   {'''lr''': 2e-5, '''num_epochs''': 3, '''seed''': 42, '''batch_size''': 16}
 training_function(__A			,							__A				)


if __name__ == "__main__":
     main()










import enum
import inspect
from typing import Iterable, List, Optional, Tuple, Union






class 	lowercase_      (			enum.Enum     ):






      '''simple docstring'''




      __snake_case  						=						'''timm'''
      __snake_case  						=						'''transformers'''
def 		_a					(   a				:Optional[Iterable[str]]			,							a				:Optional[Iterable[int]]			,							a				:Optional[Iterable[str]]				)							->    Optional[Any]:
 if stage_names is None:
  raise ValueError('''Stage_names must be set for transformers backbones'''				)

 if out_features is not None:
  if not isinstance(a			,							(list,)				):
   raise ValueError(F"""out_features must be a list {type(a				)}"""				)
  if any(feat not in stage_names for feat in out_features				):
   raise ValueError(F"""out_features must be a subset of stage_names: {stage_names} got {out_features}"""				)

 if out_indices is not None:
  if not isinstance(a			,							(list, tuple)				):
   raise ValueError(F"""out_indices must be a list or tuple, got {type(a				)}"""				)
  if any(idx >= len(a				) for idx in out_indices				):
   raise ValueError('''out_indices must be valid indices for stage_names {stage_names}, got {out_indices}'''				)

 if out_features is not None and out_indices is not None:
  if len(a				) != len(a				):
   raise ValueError('''out_features and out_indices should have the same length if both are set'''				)
  if out_features != [stage_names[idx] for idx in out_indices]:
   raise ValueError('''out_features and out_indices should correspond to the same stages if both are set'''				)
def 		_a					(   a				:Optional[List[str]]			,							a				:Optional[Union[List[int], Tuple[int]]]			,							a				:List[str]			,							)							->    Union[str, Any]:
 if out_indices is None and out_features is None:
  a							=   [len(a				) - 1]
  a							=   [stage_names[-1]]
 elif out_indices is None and out_features is not None:
  a							=   [stage_names.index(a				) for layer in out_features]
 elif out_features is None and out_indices is not None:
  a							=   [stage_names[idx] for idx in out_indices]
 return out_features, out_indices
def 		_a					(   a				:Optional[List[str]]			,							a				:Optional[Union[List[int], Tuple[int]]]			,							a				:List[str]			,							)							->    Tuple[List[str], List[int]]:
 verify_out_features_out_indices(out_features=a			,							out_indices=a			,							stage_names=a				)
 a						,  a							=   _align_output_features_output_indices(
     out_features=a			,							out_indices=a			,							stage_names=a				)
 # Verify that the aligned out_features and out_indices are valid
 verify_out_features_out_indices(out_features=a			,							out_indices=a			,							stage_names=a				)
 return output_features, output_indices






class 	lowercase_      :






      '''simple docstring'''




      __snake_case  						=						None







      def    __lowerCAmelCase      (   self   :		Any				,      __UpperCAmelCase   :		Tuple						)					->None:




       """simple docstring"""


       if getattr(self				,      '''_backbone'''				,      __UpperCAmelCase						) is None:
        raise ValueError('''self._backbone must be set before calling _init_timm_backbone'''						)

       # These will diagree with the defaults for the transformers models e.g. for resnet50
       # the transformer model has out_features = ['stem', 'stage1', 'stage2', 'stage3', 'stage4']
       # the timm model has out_features = ['act', 'layer1', 'layer2', 'layer3', 'layer4']
       a							=   [stage['''module'''] for stage in self._backbone.feature_info.info]
       a							=   [stage['''num_chs'''] for stage in self._backbone.feature_info.info]
       a							=   self._backbone.feature_info.out_indices
       a							=   self._backbone.feature_info.module_name()

       # We verify the out indices and out features are valid
       verify_out_features_out_indices(
           out_features=__UpperCAmelCase				,      out_indices=__UpperCAmelCase				,      stage_names=self.stage_names						)
       a						,  a							=   out_features, out_indices







      def    __lowerCAmelCase      (   self   :		Any				,      __UpperCAmelCase   :		Tuple						)					->None:




       """simple docstring"""


       a							=   getattr(__UpperCAmelCase				,      '''stage_names'''						)
       a							=   getattr(__UpperCAmelCase				,      '''out_features'''				,      __UpperCAmelCase						)
       a							=   getattr(__UpperCAmelCase				,      '''out_indices'''				,      __UpperCAmelCase						)

       a							=   stage_names
       a						,  a							=   get_aligned_output_features_output_indices(
           out_features=__UpperCAmelCase				,      out_indices=__UpperCAmelCase				,      stage_names=__UpperCAmelCase						)
       # Number of channels for each stage. This is set in the transformer backbone model init
       a							=   None







      def    __lowerCAmelCase      (   self   :		str				,      __UpperCAmelCase   :		Union[str, Any]						)					->None:




       """simple docstring"""


       a							=   config

       a							=   getattr(__UpperCAmelCase				,      '''use_timm_backbone'''				,      __UpperCAmelCase						)
       a							=   BackboneType.TIMM if self.use_timm_backbone else BackboneType.TRANSFORMERS

       if self.backbone_type == BackboneType.TIMM:
        self._init_timm_backbone(__UpperCAmelCase						)
       elif self.backbone_type == BackboneType.TRANSFORMERS:
        self._init_transformers_backbone(__UpperCAmelCase						)
       else:
        raise ValueError(F"""backbone_type {self.backbone_type} not supported."""						)







      @property
      def    __lowerCAmelCase      (   self   :		Dict						)					->Optional[Any]:




       """simple docstring"""


       return self._out_features







      @out_features.setter
      def    __lowerCAmelCase      (   self   :		Tuple				,      __UpperCAmelCase   :		List[str]						)					->List[Any]:




       """simple docstring"""


       a						,  a							=   get_aligned_output_features_output_indices(
           out_features=__UpperCAmelCase				,      out_indices=__UpperCAmelCase				,      stage_names=self.stage_names						)







      @property
      def    __lowerCAmelCase      (   self   :		Dict						)					->int:




       """simple docstring"""


       return self._out_indices







      @out_indices.setter
      def    __lowerCAmelCase      (   self   :		Tuple				,      __UpperCAmelCase   :		Union[Tuple[int], List[int]]						)					->List[str]:




       """simple docstring"""


       a						,  a							=   get_aligned_output_features_output_indices(
           out_features=__UpperCAmelCase				,      out_indices=__UpperCAmelCase				,      stage_names=self.stage_names						)







      @property
      def    __lowerCAmelCase      (   self   :		Tuple						)					->str:




       """simple docstring"""


       return {stage: self.num_features[i] for i, stage in enumerate(self.stage_names						)}







      @property
      def    __lowerCAmelCase      (   self   :		int						)					->Optional[int]:




       """simple docstring"""


       return [self.out_feature_channels[name] for name in self.out_features]







      def    __lowerCAmelCase      (   self   :		Optional[Any]				,      *__UpperCAmelCase   :		List[str]				,      **__UpperCAmelCase   :		Optional[Any]						)					->Optional[int]:




       """simple docstring"""


       a							=   dict(inspect.signature(self.forward						).parameters						)
       a							=   {k: v for k, v in kwargs.items() if k in signature}
       return self(*__UpperCAmelCase				,      **__UpperCAmelCase						)







      def    __lowerCAmelCase      (   self   :		List[str]				,      __UpperCAmelCase   :		Dict				,      __UpperCAmelCase   :		Optional[bool] = None				,      __UpperCAmelCase   :		Optional[bool] = None				,      __UpperCAmelCase   :		Optional[bool] = None				,      )					->int:




       """simple docstring"""


       raise NotImplementedError('''This method should be implemented by the derived class.'''						)







      def    __lowerCAmelCase      (   self   :		Union[str, Any]						)					->Optional[Any]:




       """simple docstring"""


       a							=   super().to_dict()
       a							=   output.pop('''_out_features'''						)
       a							=   output.pop('''_out_indices'''						)
       return output






class 	lowercase_      :






      '''simple docstring'''




      @property
      def    __lowerCAmelCase      (   self   :		List[str]						)					->Optional[int]:




       """simple docstring"""


       return self._out_features







      @out_features.setter
      def    __lowerCAmelCase      (   self   :		int				,      __UpperCAmelCase   :		List[str]						)					->int:




       """simple docstring"""


       a						,  a							=   get_aligned_output_features_output_indices(
           out_features=__UpperCAmelCase				,      out_indices=__UpperCAmelCase				,      stage_names=self.stage_names						)







      @property
      def    __lowerCAmelCase      (   self   :		str						)					->str:




       """simple docstring"""


       return self._out_indices







      @out_indices.setter
      def    __lowerCAmelCase      (   self   :		str				,      __UpperCAmelCase   :		Union[Tuple[int], List[int]]						)					->int:




       """simple docstring"""


       a						,  a							=   get_aligned_output_features_output_indices(
           out_features=__UpperCAmelCase				,      out_indices=__UpperCAmelCase				,      stage_names=self.stage_names						)







      def    __lowerCAmelCase      (   self   :		Dict						)					->List[Any]:




       """simple docstring"""


       a							=   super().to_dict()
       a							=   output.pop('''_out_features'''						)
       a							=   output.pop('''_out_indices'''						)
       return output


