

'''simple docstring'''





import argparse
import time

import torch
import transformers
from measures_util import end_measure, log_measures, start_measure
from transformers import AutoConfig, AutoModelForCausalLM, AutoModelForSeqaSeqLM, AutoTokenizer

from accelerate.utils import compute_module_sizes


_lowerCAmelCase      :    List[Any]						       =    {
    "gpt-j-6b": {"is_causal": True, "model": "sgugger/sharded-gpt-j-6B", "tokenizer": "EleutherAI/gpt-j-6B"},
    "gpt-neox": {"is_causal": True, "model": "EleutherAI/gpt-neox-20b"},
    "opt": {"is_causal": True, "model": "facebook/opt-30b"},
    "T0pp": {"is_causal": False, "model": "bigscience/T0pp", "model_revision": "sharded"},
}

_lowerCAmelCase      :    Optional[Any]						       =    [
    "Hello, my name is",
    "Are unicorns real? Unicorns are",
    "For the first time in several years,",
    "My name is Julien and I am",
    "The goal of life is",
    "Whenever I\'m sad, I like to",
]






def  UpperCamelCase_(							):




			"""simple docstring"""






			__a						   =argparse.ArgumentParser(description='Run and time generations on a big model using Accelerate.'			)
			parser.add_argument('model_name' ,  type=UpperCamelCase__ ,  default=UpperCamelCase__ ,  help='The name of the model to try.'			)
			parser.add_argument(
			    '--tokenizer_name' ,  type=UpperCamelCase__ ,  default=UpperCamelCase__ ,  help='The name of the tokenizer (if different from the model.'			)
			parser.add_argument('--is_causal' ,  type=UpperCamelCase__ ,  default=UpperCamelCase__ ,  help='Whether or not the model is causal.'			)
			parser.add_argument(
			    '--model_revision' ,  type=UpperCamelCase__ ,  default=UpperCamelCase__ ,  help='The revision to use for the model checkpoint.'			)
			parser.add_argument('--torch_dtype' ,  type=UpperCamelCase__ ,  default=UpperCamelCase__ ,  help='The dtype for the model.'			)
			parser.add_argument('--disk_offload' ,  action='store_true'			)

			__a						   =parser.parse_args()

			# Sanitize args
			if args.model_name in DEFAULT_MODELS:
						__a						   =DEFAULT_MODELS[args.model_name]
						__a						   =defaults['model']
						if args.tokenizer_name is None:
									__a						   =defaults.get('tokenizer' ,  args.model_name			)
						if args.is_causal is None:
									__a						   =defaults['is_causal']
						if args.model_revision is None:
									__a						   =defaults.get('model_revision' ,  'main'			)

			if args.is_causal is None:
						raise ValueError('Could not infer the default for `--is_causal`, pass either True or False for it.'			)
			if args.tokenizer_name is None:
						__a						   =args.model_name
			if args.model_revision is None:
						__a						   ='main'

			return args






def  UpperCamelCase_(							):




			"""simple docstring"""






			transformers.utils.logging.set_verbosity_error()
			__a						   =parse_args()

			if args.torch_dtype is None:
						__a						   =AutoConfig.from_pretrained(args.model_name			)
						__a						   =getattr(UpperCamelCase__ ,  'torch_dtype' ,  torch.floataa			)
			else:
						__a						   =getattr(UpperCamelCase__ ,  args.torch_dtype			)
			__a						   =AutoModelForCausalLM if args.is_causal else AutoModelForSeqaSeqLM
			__a						   ={
			    'torch_dtype': torch_dtype,
			    'revision': args.model_revision,
			}
			if args.disk_offload:
						__a						   ='tmp_offload'
						__a						   =True

			__a						   =start_measure()
			__a						   =model_cls.from_pretrained(args.model_name ,  device_map='auto' ,  **UpperCamelCase__			)
			__a						   =end_measure(UpperCamelCase__			)
			log_measures(UpperCamelCase__ ,  'Model loading'			)

			__a						   =compute_module_sizes(UpperCamelCase__			)
			__a						   ={v: 0 for v in model.hf_device_map.values()}
			for module, device in model.hf_device_map.items():
						device_size[device] += module_sizes[module]
			__a						   ='\n'.join([F'- {device}: {size // 2**20}MiB' for device, size in device_size.items()]			)
			print(F'\nTheoretical use:\n{message}'			)

			__a						   =AutoTokenizer.from_pretrained(args.tokenizer_name			)

			__a						   =start_measure()
			__a						   =[]
			__a						   =[]
			__a						   =[]
			for prompt in PROMPTS:
						__a						   =tokenizer(UpperCamelCase__ ,  return_tensors='pt'			).to(0			)
						__a						   =inputs['input_ids'][0].tolist()
						__a						   =time.time()
						__a						   =model.generate(inputs['input_ids']			)
						__a						   =time.time()
						__a						   =outputs[0].tolist()
						__a						   =len(UpperCamelCase__			) if outputs[: len(UpperCamelCase__			)] != tokens else len(UpperCamelCase__			) - len(UpperCamelCase__			)
						__a						   =after_generate - before_generate

						__a						   =tokenizer.decode(UpperCamelCase__ ,  skip_special_tokens=UpperCamelCase__			)
						texts_outs.append(UpperCamelCase__			)
						generation_times.append(UpperCamelCase__			)
						gen_tokens.append(UpperCamelCase__			)
						print(F'Prompt: {prompt}\nGeneration {text_out}\nIn {generation_time:.2f}s for {num_gen_tokens} tokens\n'			)

			__a						   =end_measure(UpperCamelCase__			)
			log_measures(UpperCamelCase__ ,  'Model generation'			)

			__a						   =[gen / tok for gen, tok in zip(UpperCamelCase__ ,  UpperCamelCase__			)]
			__a						   =sum(UpperCamelCase__			) / len(UpperCamelCase__			)
			print(F'Average time of generation per token: {avg_gen:.2f}s'			)
			print(F'First generation (avg time per token): {generation_times_per_token[0]:.2f}s'			)
			__a						   =sum(generation_times_per_token[1:]			) / (len(UpperCamelCase__			) - 1)
			print(F'Average time of generation per token (excluding the first): {avg_gen:.2f}s'			)


if __name__ == "__main__":
						main()









from typing import Optional

from ..utils.logging import get_logger
from .audio_classificiation import AudioClassification
from .automatic_speech_recognition import AutomaticSpeechRecognition
from .base import TaskTemplate
from .image_classification import ImageClassification
from .language_modeling import LanguageModeling
from .question_answering import QuestionAnsweringExtractive
from .summarization import Summarization
from .text_classification import TextClassification


_lowerCAmelCase      :    List[Any]						       =    [
    "AutomaticSpeechRecognition",
    "AudioClassification",
    "ImageClassification",
    "LanguageModeling",
    "QuestionAnsweringExtractive",
    "Summarization",
    "TaskTemplate",
    "TextClassification",
]

_lowerCAmelCase      :    List[str]						       =    get_logger(__name__)


_lowerCAmelCase      :    Tuple						       =    {
    AutomaticSpeechRecognition.task: AutomaticSpeechRecognition,
    AudioClassification.task: AudioClassification,
    ImageClassification.task: ImageClassification,
    LanguageModeling.task: LanguageModeling,
    QuestionAnsweringExtractive.task: QuestionAnsweringExtractive,
    Summarization.task: Summarization,
    TextClassification.task: TextClassification,
}






def  UpperCamelCase_(							_snake_case     :						dict			):




			"""simple docstring"""






			__a						   =task_template_dict.get('task'			)
			if task_name is None:
						logger.warning(F'Couldn\'t find template for task \'{task_name}\'. Available templates: {list(_snake_case			)}'			)
						return None
			__a						   =NAMEaTEMPLATE.get(_snake_case			)
			return template.from_dict(_snake_case			)






