






import argparse
import gc
import os
import threading

import evaluate
import psutil
import torch
from datasets import load_dataset
from torch.distributed.fsdp.fully_sharded_data_parallel import FullOptimStateDictConfig, FullStateDictConfig
from torch.utils.data import DataLoader
from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    get_linear_schedule_with_warmup,
    set_seed,
)

from accelerate import Accelerator, DistributedType, FullyShardedDataParallelPlugin


########################################################################
# This is a fully working simple example to use Accelerate
#
# This example trains a Bert base model on GLUE MRPC
# in any of the following settings (with the same script):
#   - single CPU or single GPU
#   - multi GPUS (using PyTorch distributed mode)
#   - (multi) TPUs
#   - fp16 (mixed-precision) or fp32 (normal precision)
#   - FSDP
#
# This example also demonstrates the checkpointing and sharding capabilities
#
# To run it in each of these various modes, follow the instructions
# in the readme for examples:
# https://github.com/huggingface/accelerate/tree/main/examples
#
########################################################################


UpperCAmelCase__     :   List[Any]			 =	16
UpperCAmelCase__     :   int			 =	32







def 						a_      (__A			)							->	List[str]:



					"""simple docstring"""
					return int(x / 2**20			)




class        snake_case_					:

						"""simple docstring"""



						def __enter__(self:				Any      )						->    Optional[Any]:




											'''simple docstring'''



											gc.collect()
											torch.cuda.empty_cache()
											torch.cuda.reset_max_memory_allocated()  # reset the peak gauge to zero
											__a				:	Optional[Any]			=				torch.cuda.memory_allocated()
											__a				:	int			=				psutil.Process()

											__a				:	List[str]			=				self.cpu_mem_used()
											__a				:	Dict			=				True
											__a				:	Optional[Any]			=				threading.Thread(target=self.peak_monitor_func      )
											__a				:	Dict			=				True
											peak_monitor_thread.start()
											return self



						def 					UpperCAmelCase__				(self:				Any      )						->    Optional[Any]:




											'''simple docstring'''



											return self.process.memory_info().rss



						def 					UpperCAmelCase__				(self:				List[Any]      )						->    Any:




											'''simple docstring'''



											__a				:	Union[str, Any]			=				-1

											while True:
																__a				:	str			=				max(self.cpu_mem_used()   ,				self.cpu_peak      )

																# can't sleep or will not catch the peak right (this comment is here on purpose)
																# time.sleep(0.001) # 1msec

																if not self.peak_monitoring:
																					break



						def __exit__(self:				int   ,				*__UpperCAmelCase:				Optional[Any]      )						->    Any:




											'''simple docstring'''



											__a				:	Tuple			=				False

											gc.collect()
											torch.cuda.empty_cache()
											__a				:	Union[str, Any]			=				torch.cuda.memory_allocated()
											__a				:	List[Any]			=				torch.cuda.max_memory_allocated()
											__a				:	Tuple			=				bamb(self.end - self.begin      )
											__a				:	Optional[int]			=				bamb(self.peak - self.begin      )

											__a				:	List[Any]			=				self.cpu_mem_used()
											__a				:	Dict			=				bamb(self.cpu_end - self.cpu_begin      )
											__a				:	Optional[Any]			=				bamb(self.cpu_peak - self.cpu_begin      )
											# print(f"delta used/peak {self.used:4d}/{self.peaked:4d}")


# For testing only
if os.environ.get('''TESTING_MOCKED_DATALOADERS''', None) == "1":
			from accelerate.test_utils.training import mocked_dataloaders

			UpperCAmelCase__     :   int			 =	mocked_dataloaders  # noqa: F811







def 						a_      (__A							,					__A			)							->	List[Any]:



					"""simple docstring"""






					if os.environ.get("TESTING_MOCKED_DATALOADERS"							,					_UpperCAmelCase			) == "1":
										__a				:	Optional[int]			=				2

					# New Code #
					# Pass the advanced FSDP settings not part of the accelerate config by creating fsdp_plugin
					__a				:	List[str]			=				FullyShardedDataParallelPlugin(
					    state_dict_config=FullStateDictConfig(offload_to_cpu=_UpperCAmelCase							,					ranka_only=_UpperCAmelCase			)							,					optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=_UpperCAmelCase							,					ranka_only=_UpperCAmelCase			)							,					)

					# Initialize accelerator
					if args.with_tracking:
										__a				:	Any			=				Accelerator(
										    cpu=args.cpu							,					mixed_precision=args.mixed_precision							,					log_with="wandb"							,					project_dir=args.logging_dir							,					fsdp_plugin=_UpperCAmelCase							,					)
					else:
										__a				:	Optional[int]			=				Accelerator(fsdp_plugin=_UpperCAmelCase			)
					accelerator.print(accelerator.distributed_type			)

					if hasattr(args.checkpointing_steps							,					"isdigit"			):
										if args.checkpointing_steps == "epoch":
															__a				:	List[Any]			=				args.checkpointing_steps
										elif args.checkpointing_steps.isdigit():
															__a				:	Optional[int]			=				int(args.checkpointing_steps			)
										else:
															raise ValueError(
															    f'Argument `checkpointing_steps` must be either a number or `epoch`. `{args.checkpointing_steps}` passed.'			)
					else:
										__a				:	List[Any]			=				None
					# Sample hyper-parameters for learning rate, batch size, seed and a few other HPs
					__a				:	Union[str, Any]			=				config['lr']
					__a				:	Union[str, Any]			=				int(config["num_epochs"]			)
					__a				:	Tuple			=				int(config["seed"]			)
					__a				:	Any			=				int(config["batch_size"]			)

					# We need to initialize the trackers we use, and also store our configuration
					if args.with_tracking:
										__a				:	Dict			=				vars(_UpperCAmelCase			)
										accelerator.init_trackers("fsdp_glue_no_trainer"							,					_UpperCAmelCase			)

					__a				:	List[str]			=				AutoTokenizer.from_pretrained(args.model_name_or_path			)
					__a				:	Any			=				load_dataset("glue"							,					"mrpc"			)
					__a				:	Any			=				evaluate.load("glue"							,					"mrpc"			)

					def tokenize_function(__A			):
										# max_length=None => use the model max length (it's actually the default)
										__a				:	int			=				tokenizer(examples["sentence1"]							,					examples["sentence2"]							,					truncation=_UpperCAmelCase							,					max_length=_UpperCAmelCase			)
										return outputs

					# Apply the method we just defined to all the examples in all the splits of the dataset
					# starting with the main process first:
					with accelerator.main_process_first():
										__a				:	Union[str, Any]			=				datasets.map(
										    _UpperCAmelCase							,					batched=_UpperCAmelCase							,					remove_columns=["idx", "sentence1", "sentence2"]							,					)

					# We also rename the 'label' column to 'labels' which is the expected name for labels by the models of the
					# transformers library
					__a				:	Union[str, Any]			=				tokenized_datasets.rename_column("label"							,					"labels"			)

					# If the batch size is too big we use gradient accumulation
					__a				:	Optional[Any]			=				1
					if batch_size > MAX_GPU_BATCH_SIZE and accelerator.distributed_type != DistributedType.TPU:
										__a				:	str			=				batch_size // MAX_GPU_BATCH_SIZE
										__a				:	Optional[int]			=				MAX_GPU_BATCH_SIZE

					def collate_fn(__A			):
										# On TPU it's best to pad everything to the same length or training will be very slow.
										__a				:	List[str]			=				128 if accelerator.distributed_type == DistributedType.TPU else None
										# When using mixed precision we want round multiples of 8/16
										if accelerator.mixed_precision == "fp8":
															__a				:	Tuple			=				16
										elif accelerator.mixed_precision != "no":
															__a				:	List[Any]			=				8
										else:
															__a				:	Union[str, Any]			=				None

										return tokenizer.pad(
										    _UpperCAmelCase							,					padding="longest"							,					max_length=_UpperCAmelCase							,					pad_to_multiple_of=_UpperCAmelCase							,					return_tensors="pt"							,					)

					# Instantiate dataloaders.
					__a				:	int			=				DataLoader(
					    tokenized_datasets["train"]							,					shuffle=_UpperCAmelCase							,					collate_fn=_UpperCAmelCase							,					batch_size=_UpperCAmelCase			)
					__a				:	int			=				DataLoader(
					    tokenized_datasets["validation"]							,					shuffle=_UpperCAmelCase							,					collate_fn=_UpperCAmelCase							,					batch_size=_UpperCAmelCase			)

					set_seed(_UpperCAmelCase			)

					# Instantiate the model (we build the model here so that the seed also control new weights initialization)
					__a				:	Optional[int]			=				AutoModelForSequenceClassification.from_pretrained(
					    args.model_name_or_path							,					return_dict=_UpperCAmelCase							,					low_cpu_mem_usage=_UpperCAmelCase			)

					# New Code #
					# For FSDP feature, it is highly recommended and efficient to prepare the model before creating optimizer
					__a				:	int			=				accelerator.prepare(_UpperCAmelCase			)
					accelerator.print(_UpperCAmelCase			)

					# Instantiate optimizer
					# New Code #
					# For FSDP feature, at present it doesn't support multiple parameter groups,
					# so we need to create a single parameter group for the whole model
					__a				:	str			=				torch.optim.AdamW(params=model.parameters()							,					lr=_UpperCAmelCase							,					weight_decay=2e-4			)

					# Instantiate scheduler
					__a				:	Optional[Any]			=				get_linear_schedule_with_warmup(
					    optimizer=_UpperCAmelCase							,					num_warmup_steps=10							,					num_training_steps=(len(_UpperCAmelCase			) * num_epochs) // gradient_accumulation_steps							,					)

					# New Code #
					# For FSDP feature, prepare everything except the model as we have already prepared the model
					# before creating the optimizer
					# There is no specific order to remember, we just need to unpack the objects in the same order we gave them to the
					# prepare method.
					__a				:	Any			=				accelerator.prepare(
					    _UpperCAmelCase							,					_UpperCAmelCase							,					_UpperCAmelCase							,					_UpperCAmelCase			)

					__a				:	Optional[Any]			=				0

					# Potentially load in the weights and states from a previous save
					if args.resume_from_checkpoint:
										if args.resume_from_checkpoint is not None or args.resume_from_checkpoint != "":
															accelerator.print(f'Resumed from checkpoint: {args.resume_from_checkpoint}'			)
															accelerator.load_state(args.resume_from_checkpoint			)
															__a				:	Optional[Any]			=				os.path.basename(args.resume_from_checkpoint			)
										else:
															# Get the most recent checkpoint
															__a				:	Tuple			=				[f.name for f in os.scandir(os.getcwd()			) if f.is_dir()]
															dirs.sort(key=os.path.getctime			)
															__a				:	Optional[int]			=				dirs[-1]  # Sorts folders by date modified, most recent checkpoint is the last
										# Extract `epoch_{i}` or `step_{i}`
										__a				:	str			=				os.path.splitext(_UpperCAmelCase			)[0]

										if "epoch" in training_difference:
															num_epochs -= int(training_difference.replace("epoch_"							,					""			)			)
															__a				:	str			=				None
										else:
															__a				:	Optional[int]			=				int(training_difference.replace("step_"							,					""			)			)
															num_epochs -= resume_step // len(_UpperCAmelCase			)
															# If resuming by step, we also need to know exactly how far into the DataLoader we went
															__a				:	str			=				(num_epochs * len(_UpperCAmelCase			)) - resume_step

    # Now we train the model
					for epoch in range(_UpperCAmelCase			):
										# New Code #
										# context manager to track the peak memory usage during the training epoch
										with TorchTracemalloc() as tracemalloc:
															model.train()
															if args.with_tracking:
																				__a				:	Optional[Any]			=				0
															for step, batch in enumerate(_UpperCAmelCase			):
																				# We need to skip steps until we reach the resumed step
																				if args.resume_from_checkpoint and epoch == 0:
																									if resume_step is not None and step < resume_step:
																														pass
                # We could avoid this line since we set the accelerator with `device_placement=True`.
																				batch.to(accelerator.device			)
																				__a				:	List[Any]			=				model(**_UpperCAmelCase			)
																				__a				:	str			=				outputs.loss
																				__a				:	List[Any]			=				loss / gradient_accumulation_steps
																				# We keep track of the loss at each epoch
																				if args.with_tracking:
																									total_loss += loss.detach().float()
																				accelerator.backward(_UpperCAmelCase			)
																				if step % gradient_accumulation_steps == 0:
																									optimizer.step()
																									lr_scheduler.step()
																									optimizer.zero_grad()
																									# accelerator.print(lr_scheduler.get_lr())

																				overall_step += 1

																				if isinstance(_UpperCAmelCase							,					_UpperCAmelCase			):
																									__a				:	Optional[int]			=				f'step_{overall_step}'
																									if overall_step % checkpointing_steps == 0:
																														if args.output_dir is not None:
																																			__a				:	List[str]			=				os.path.join(args.output_dir							,					_UpperCAmelCase			)
																														accelerator.save_state(_UpperCAmelCase			)
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
										accelerator.print("Memory before entering the train : {}".format(bamb(tracemalloc.begin			)			)			)
										accelerator.print("Memory consumed at the end of the train (end-begin): {}".format(tracemalloc.used			)			)
										accelerator.print("Peak Memory consumed during the train (max-begin): {}".format(tracemalloc.peaked			)			)
										accelerator.print(
										    "Total Peak Memory consumed during the train (max): {}".format(
										        tracemalloc.peaked + bamb(tracemalloc.begin			)			)			)
										# Logging the peak memory usage of the GPU to the tracker
										if args.with_tracking:
															accelerator.log(
															    {
															        "train_total_peak_memory": tracemalloc.peaked + bamb(tracemalloc.begin			),
															    }							,					step=_UpperCAmelCase							,					)

										# New Code #
										# context manager to track the peak memory usage during the evaluation
										with TorchTracemalloc() as tracemalloc:
															model.eval()
															for step, batch in enumerate(_UpperCAmelCase			):
																				# We could avoid this line since we set the accelerator with `device_placement=True`.
																				batch.to(accelerator.device			)
																				with torch.no_grad():
																									__a				:	List[Any]			=				model(**_UpperCAmelCase			)
																				__a				:	Optional[Any]			=				outputs.logits.argmax(dim=-1			)
																				__a				:	Union[str, Any]			=				accelerator.gather_for_metrics((predictions, batch["labels"])			)
																				metric.add_batch(
																				    predictions=_UpperCAmelCase							,					references=_UpperCAmelCase							,					)

															__a				:	Dict			=				metric.compute()
															# Use accelerator.print to print only on the main process.
															accelerator.print(f'epoch {epoch}:'							,					_UpperCAmelCase			)
															if args.with_tracking:
																				accelerator.log(
																				    {
																				        "accuracy": eval_metric["accuracy"],
																				        "f1": eval_metric["f1"],
																				        "train_loss": total_loss.item() / len(_UpperCAmelCase			),
																				    }							,					step=_UpperCAmelCase							,					)

															if checkpointing_steps == "epoch":
																				__a				:	str			=				f'epoch_{epoch}'
																				if args.output_dir is not None:
																									__a				:	List[str]			=				os.path.join(args.output_dir							,					_UpperCAmelCase			)
																				accelerator.save_state(_UpperCAmelCase			)
        # New Code #
        # Printing the GPU memory usage details such as allocated memory, peak memory, and total memory usage
										accelerator.print("Memory before entering the eval : {}".format(bamb(tracemalloc.begin			)			)			)
										accelerator.print("Memory consumed at the end of the eval (end-begin): {}".format(tracemalloc.used			)			)
										accelerator.print("Peak Memory consumed during the eval (max-begin): {}".format(tracemalloc.peaked			)			)
										accelerator.print(
										    "Total Peak Memory consumed during the eval (max): {}".format(tracemalloc.peaked + bamb(tracemalloc.begin			)			)			)
										# Logging the peak memory usage of the GPU to the tracker
										if args.with_tracking:
															accelerator.log(
															    {
															        "eval_total_peak_memory": tracemalloc.peaked + bamb(tracemalloc.begin			),
															    }							,					step=_UpperCAmelCase							,					)

					if args.with_tracking:
										accelerator.end_training()







def 						a_      ()							->	Union[str, Any]:



					"""simple docstring"""
					__a				:	Dict			=				argparse.ArgumentParser(description="Simple example of training script."			)
					parser.add_argument(
					    "--mixed_precision"							,					type=_UpperCAmelCase							,					default=_UpperCAmelCase							,					choices=["no", "fp16", "bf16", "fp8"]							,					help="Whether to use mixed precision. Choose"
					    "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10."
					    "and an Nvidia Ampere GPU."							,					)
					parser.add_argument("--cpu"							,					action="store_true"							,					help="If passed, will train on the CPU."			)
					parser.add_argument(
					    "--checkpointing_steps"							,					type=_UpperCAmelCase							,					default=_UpperCAmelCase							,					help="Whether the various states should be saved at the end of every n steps, or \'epoch\' for each epoch."							,					)
					parser.add_argument(
					    "--resume_from_checkpoint"							,					type=_UpperCAmelCase							,					default=_UpperCAmelCase							,					help="If the training should continue from a checkpoint folder."							,					)
					parser.add_argument(
					    "--with_tracking"							,					action="store_true"							,					help="Whether to load in all available experiment trackers from the environment and use them for logging."							,					)
					parser.add_argument(
					    "--output_dir"							,					type=_UpperCAmelCase							,					default="."							,					help="Optional save directory where all checkpoint folders will be stored. Default is the current working directory."							,					)
					parser.add_argument(
					    "--logging_dir"							,					type=_UpperCAmelCase							,					default="logs"							,					help="Location on where to store experiment tracking logs`"							,					)
					parser.add_argument(
					    "--model_name_or_path"							,					type=_UpperCAmelCase							,					help="Path to pretrained model or model identifier from huggingface.co/models."							,					required=_UpperCAmelCase							,					)
					__a				:	Union[str, Any]			=				parser.parse_args()
					__a				:	Any			=				{'lr': 2e-5, 'num_epochs': 3, 'seed': 42, 'batch_size': 16}
					training_function(_UpperCAmelCase							,					_UpperCAmelCase			)


if __name__ == "__main__":
			main()













import copy

from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ..auto.configuration_auto import AutoConfig


UpperCAmelCase__			 =	logging.get_logger(__name__)

UpperCAmelCase__			 =	{
    '''facebook/musicgen-small''': '''https://huggingface.co/facebook/musicgen-small/resolve/main/config.json''',
    # See all Musicgen models at https://huggingface.co/models?filter=musicgen
}




class        snake_case_					(						__UpperCamelCase			):

						"""simple docstring"""



						snake_case__   					=			"""musicgen_decoder"""
						snake_case__   					=			["""past_key_values"""]



						def __init__(self:				str   ,				__UpperCAmelCase:				Union[str, Any]=2048   ,				__UpperCAmelCase:				Union[str, Any]=2048   ,				__UpperCAmelCase:				List[Any]=24   ,				__UpperCAmelCase:				Dict=4096   ,				__UpperCAmelCase:				int=16   ,				__UpperCAmelCase:				Dict=0.0   ,				__UpperCAmelCase:				str=True   ,				__UpperCAmelCase:				Tuple="gelu"   ,				__UpperCAmelCase:				Optional[Any]=1024   ,				__UpperCAmelCase:				Optional[Any]=0.1   ,				__UpperCAmelCase:				List[Any]=0.0   ,				__UpperCAmelCase:				int=0.0   ,				__UpperCAmelCase:				List[Any]=0.02   ,				__UpperCAmelCase:				Union[str, Any]=False   ,				__UpperCAmelCase:				Any=4   ,				__UpperCAmelCase:				Tuple=2048   ,				__UpperCAmelCase:				Tuple=2048   ,				__UpperCAmelCase:				Dict=None   ,				__UpperCAmelCase:				Dict=False   ,				**__UpperCAmelCase:				Any   ,				)						->    Optional[Any]:




											'''simple docstring'''



											__a				:	Dict			=				vocab_size
											__a				:	List[str]			=				max_position_embeddings
											__a				:	int			=				hidden_size
											__a				:	Union[str, Any]			=				ffn_dim
											__a				:	List[Any]			=				num_hidden_layers
											__a				:	List[Any]			=				num_attention_heads
											__a				:	List[Any]			=				dropout
											__a				:	str			=				attention_dropout
											__a				:	Union[str, Any]			=				activation_dropout
											__a				:	Optional[int]			=				activation_function
											__a				:	List[str]			=				initializer_factor
											__a				:	Union[str, Any]			=				layerdrop
											__a				:	Optional[int]			=				use_cache
											__a				:	List[Any]			=				scale_embedding  # scale factor will be sqrt(d_model) if True
											__a				:	str			=				num_codebooks
											super().__init__(
											    pad_token_id=__UpperCAmelCase   ,				bos_token_id=__UpperCAmelCase   ,				eos_token_id=__UpperCAmelCase   ,				tie_word_embeddings=__UpperCAmelCase   ,				**__UpperCAmelCase   ,				)




class        snake_case_					(						__UpperCamelCase			):

						"""simple docstring"""



						snake_case__   					=			"""musicgen"""
						snake_case__   					=			True



						def __init__(self:				Union[str, Any]   ,				**__UpperCAmelCase:				Optional[int]      )						->    List[str]:




											'''simple docstring'''



											super().__init__(**__UpperCAmelCase      )
											if "text_encoder" not in kwargs or "audio_encoder" not in kwargs or "decoder" not in kwargs:
																raise ValueError("Config has to be initialized with text_encoder, audio_encoder and decoder config"      )

											__a				:	List[Any]			=				kwargs.pop("text_encoder"      )
											__a				:	Optional[Any]			=				text_encoder_config.pop("model_type"      )

											__a				:	Union[str, Any]			=				kwargs.pop("audio_encoder"      )
											__a				:	Dict			=				audio_encoder_config.pop("model_type"      )

											__a				:	Optional[Any]			=				kwargs.pop("decoder"      )

											__a				:	Optional[int]			=				AutoConfig.for_model(__UpperCAmelCase   ,				**__UpperCAmelCase      )
											__a				:	int			=				AutoConfig.for_model(__UpperCAmelCase   ,				**__UpperCAmelCase      )
											__a				:	List[Any]			=				MusicgenDecoderConfig(**__UpperCAmelCase      )
											__a				:	Optional[int]			=				True



						@classmethod
						def 					UpperCAmelCase__				(cls:				Union[str, Any]   ,				__UpperCAmelCase:				PretrainedConfig   ,				__UpperCAmelCase:				PretrainedConfig   ,				__UpperCAmelCase:				MusicgenDecoderConfig   ,				**__UpperCAmelCase:				Any   ,				)						->    Tuple:




											'''simple docstring'''



											return cls(
											    text_encoder=text_encoder_config.to_dict()   ,				audio_encoder=audio_encoder_config.to_dict()   ,				decoder=decoder_config.to_dict()   ,				**__UpperCAmelCase   ,				)



						def 					UpperCAmelCase__				(self:				List[str]      )						->    int:




											'''simple docstring'''



											__a				:	Dict			=				copy.deepcopy(self.__dict__      )
											__a				:	Optional[Any]			=				self.text_encoder.to_dict()
											__a				:	Optional[Any]			=				self.audio_encoder.to_dict()
											__a				:	Tuple			=				self.decoder.to_dict()
											__a				:	List[Any]			=				self.__class__.model_type
											return output





