





import os
import warnings
from pathlib import Path
from typing import Optional, Union

from .dynamic_module_utils import custom_object_save
from .tokenization_utils_base import PreTrainedTokenizerBase
from .utils import PushToHubMixin, copy_func, direct_transformers_import, logging


__lowercase													=			logging.get_logger(__name__)

# Dynamically import the Transformers module to grab the attribute classes of the processor form their names.
__lowercase													=			direct_transformers_import(Path(__file__).parent)


__lowercase													=			{
    """AutoTokenizer""": """PreTrainedTokenizerBase""",
    """AutoFeatureExtractor""": """FeatureExtractionMixin""",
    """AutoImageProcessor""": """ImageProcessingMixin""",
}

class   __snake_case				(			_a   ):






    '''simple docstring'''
    _lowerCAmelCase     =	["""feature_extractor""", """tokenizer"""]
    # Names need to be attr_class for attr in attributes
    _lowerCAmelCase     =	None
    _lowerCAmelCase     =	None
    _lowerCAmelCase     =	None





    def __init__(   self						:		List[str]			,						*snake_case						:		Dict			,						**snake_case						:		Any							):

      """simple docstring"""





      for key in kwargs:
        if key not in self.attributes:
          raise TypeError(f'''Unexpected keyword argument {key}.'''							)
      for arg, attribute_name in zip(snake_case			,						self.attributes							):
        if attribute_name in kwargs:
          raise TypeError(f'''Got multiple values for argument {attribute_name}.'''							)
        else:
          __lowerCAmelCase			=				arg

      if len(snake_case							) != len(self.attributes							):
        raise ValueError(
            f'''This processor requires {len(self.attributes							)} arguments: {", ".join(self.attributes							)}. Got '''
            f'''{len(snake_case							)} arguments instead.'''							)

      # Check each arg is of the proper class (this will also catch a user initializing in the wrong order)
      for attribute_name, arg in kwargs.items():
        __lowerCAmelCase			=				getattr(self			,						f'''{attribute_name}_class'''							)
        # Nothing is ever going to be an instance of "AutoXxx", in that case we check the base class.
        __lowerCAmelCase			=				AUTO_TO_BASE_CLASS_MAPPING.get(snake_case			,						snake_case							)
        if isinstance(snake_case			,						snake_case							):
          __lowerCAmelCase			=				tuple(getattr(snake_case			,						snake_case							) for n in class_name if n is not None							)
        else:
          __lowerCAmelCase			=				getattr(snake_case			,						snake_case							)

        if not isinstance(snake_case			,						snake_case							):
          raise ValueError(
              f'''Received a {type(snake_case							).__name__} for argument {attribute_name}, but a {class_name} was expected.'''							)

        setattr(self			,						snake_case			,						snake_case							)





    def __repr__(   self						:		List[str]							):

      """simple docstring"""





      __lowerCAmelCase			=				[f'''- {name}: {repr(getattr(self			,						snake_case							)							)}''' for name in self.attributes]
      __lowerCAmelCase			=				'''\n'''.join(snake_case							)
      return f'''{self.__class__.__name__}:\n{attributes_repr}'''





    def 	A		(   self						:		List[str]			,						snake_case						:		Dict			,						snake_case						:		bool = False			,						**snake_case						:		Tuple							):

      """simple docstring"""





      os.makedirs(snake_case			,						exist_ok=snake_case							)

      if push_to_hub:
        __lowerCAmelCase			=				kwargs.pop('''commit_message'''			,						snake_case							)
        __lowerCAmelCase			=				kwargs.pop('''repo_id'''			,						save_directory.split(os.path.sep							)[-1]							)
        __lowerCAmelCase			=				self._create_repo(snake_case			,						**snake_case							)
        __lowerCAmelCase			=				self._get_files_timestamps(snake_case							)
      # If we have a custom config, we copy the file defining it in the folder and set the attributes so it can be
      # loaded from the Hub.
      if self._auto_class is not None:
        __lowerCAmelCase			=				[getattr(self			,						snake_case							) for attribute_name in self.attributes]
        __lowerCAmelCase			=				[(a.init_kwargs if isinstance(snake_case			,						snake_case							) else a) for a in attrs]
        custom_object_save(self			,						snake_case			,						config=snake_case							)

      for attribute_name in self.attributes:
        __lowerCAmelCase			=				getattr(self			,						snake_case							)
        # Include the processor class in the attribute config so this processor can then be reloaded with the
        # `AutoProcessor` API.
        if hasattr(snake_case			,						'''_set_processor_class'''							):
          attribute._set_processor_class(self.__class__.__name__							)
        attribute.save_pretrained(snake_case							)

      if self._auto_class is not None:
        # We added an attribute to the init_kwargs of the tokenizers, which needs to be cleaned up.
        for attribute_name in self.attributes:
          __lowerCAmelCase			=				getattr(self			,						snake_case							)
          if isinstance(snake_case			,						snake_case							):
            del attribute.init_kwargs["auto_map"]

      if push_to_hub:
        self._upload_modified_files(
            snake_case			,						snake_case			,						snake_case			,						commit_message=snake_case			,						token=kwargs.get('''use_auth_token'''							)			,						)





    @classmethod
    def 	A		(   cls						:		Any			,						snake_case						:		Union[str, os.PathLike]			,						snake_case						:		Optional[Union[str, os.PathLike]] = None			,						snake_case						:		bool = False			,						snake_case						:		bool = False			,						snake_case						:		Optional[Union[str, bool]] = None			,						snake_case						:		str = "main"			,						**snake_case						:		Optional[int]			,						):

      """simple docstring"""





      __lowerCAmelCase			=				cache_dir
      __lowerCAmelCase			=				force_download
      __lowerCAmelCase			=				local_files_only
      __lowerCAmelCase			=				revision

      __lowerCAmelCase			=				kwargs.pop('''use_auth_token'''			,						snake_case							)
      if use_auth_token is not None:
        warnings.warn(
            '''The `use_auth_token` argument is deprecated and will be removed in v5 of Transformers.'''			,						snake_case							)
        if token is not None:
          raise ValueError(
              '''`token` and `use_auth_token` are both specified. Please set only the argument `token`.'''							)
        __lowerCAmelCase			=				use_auth_token

      if token is not None:
        # change to `token` in a follow-up PR
        __lowerCAmelCase			=				token

      __lowerCAmelCase			=				cls._get_arguments_from_pretrained(snake_case			,						**snake_case							)
      return cls(*snake_case							)





    @classmethod
    def 	A		(   cls						:		Union[str, Any]			,						snake_case						:		Dict="AutoProcessor"							):

      """simple docstring"""





      if not isinstance(snake_case			,						snake_case							):
        __lowerCAmelCase			=				auto_class.__name__

      import transformers.models.auto as auto_module

      if not hasattr(snake_case			,						snake_case							):
        raise ValueError(f'''{auto_class} is not a valid auto class.'''							)

      __lowerCAmelCase			=				auto_class





    @classmethod
    def 	A		(   cls						:		List[Any]			,						snake_case						:		int			,						**snake_case						:		Optional[int]							):

      """simple docstring"""





      __lowerCAmelCase			=				[]
      for attribute_name in cls.attributes:
        __lowerCAmelCase			=				getattr(cls			,						f'''{attribute_name}_class'''							)
        if isinstance(snake_case			,						snake_case							):
          __lowerCAmelCase			=				tuple(getattr(snake_case			,						snake_case							) if n is not None else None for n in class_name							)
          __lowerCAmelCase			=				kwargs.get('''use_fast'''			,						snake_case							)
          if use_fast and classes[1] is not None:
            __lowerCAmelCase			=				classes[1]
          else:
            __lowerCAmelCase			=				classes[0]
        else:
          __lowerCAmelCase			=				getattr(snake_case			,						snake_case							)

        args.append(attribute_class.from_pretrained(snake_case			,						**snake_case							)							)
      return args





    @property
    def 	A		(   self						:		Tuple							):

      """simple docstring"""





      __lowerCAmelCase			=				getattr(self			,						self.attributes[0]							)
      return getattr(snake_case			,						'''model_input_names'''			,						snake_case							)


__lowercase													=			copy_func(ProcessorMixin.push_to_hub)
if ProcessorMixin.push_to_hub.__doc__ is not None:
      __lowercase													=			ProcessorMixin.push_to_hub.__doc__.format(
          object="""processor""", object_class="""AutoProcessor""", object_files="""processor files"""
      )







from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...file_utils import TensorType, is_torch_available
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import logging


__lowercase													=			logging.get_logger(__name__)

__lowercase													=			{
    """facebook/blenderbot-3B""": """https://huggingface.co/facebook/blenderbot-3B/resolve/main/config.json""",
    # See all Blenderbot models at https://huggingface.co/models?filter=blenderbot
}

class   __snake_case				(			_a   ):






    '''simple docstring'''
    _lowerCAmelCase     =	"""blenderbot"""
    _lowerCAmelCase     =	["""past_key_values"""]
    _lowerCAmelCase     =	{"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}





    def __init__(   self						:		Any			,						snake_case						:		Union[str, Any]=8_008			,						snake_case						:		List[str]=128			,						snake_case						:		Tuple=2			,						snake_case						:		int=10_240			,						snake_case						:		Optional[int]=32			,						snake_case						:		Dict=24			,						snake_case						:		List[Any]=10_240			,						snake_case						:		Optional[int]=32			,						snake_case						:		List[str]=0.0			,						snake_case						:		Optional[int]=0.0			,						snake_case						:		Optional[Any]=True			,						snake_case						:		int=True			,						snake_case						:		List[str]="gelu"			,						snake_case						:		Any=2_560			,						snake_case						:		Optional[Any]=0.1			,						snake_case						:		Union[str, Any]=0.0			,						snake_case						:		List[Any]=0.0			,						snake_case						:		Dict=0.02			,						snake_case						:		int=1			,						snake_case						:		Dict=False			,						snake_case						:		List[str]=0			,						snake_case						:		Optional[Any]=1			,						snake_case						:		List[Any]=2			,						snake_case						:		List[str]=3			,						snake_case						:		List[str]=2			,						**snake_case						:		int			,						):

      """simple docstring"""





      __lowerCAmelCase			=				vocab_size
      __lowerCAmelCase			=				max_position_embeddings
      __lowerCAmelCase			=				d_model
      __lowerCAmelCase			=				encoder_ffn_dim
      __lowerCAmelCase			=				encoder_layers
      __lowerCAmelCase			=				encoder_attention_heads
      __lowerCAmelCase			=				decoder_ffn_dim
      __lowerCAmelCase			=				decoder_layers
      __lowerCAmelCase			=				decoder_attention_heads
      __lowerCAmelCase			=				dropout
      __lowerCAmelCase			=				attention_dropout
      __lowerCAmelCase			=				activation_dropout
      __lowerCAmelCase			=				activation_function
      __lowerCAmelCase			=				init_std
      __lowerCAmelCase			=				encoder_layerdrop
      __lowerCAmelCase			=				decoder_layerdrop
      __lowerCAmelCase			=				use_cache
      __lowerCAmelCase			=				encoder_layers
      __lowerCAmelCase			=				scale_embedding  # scale factor will be sqrt(d_model) if True

      super().__init__(
          pad_token_id=snake_case			,						bos_token_id=snake_case			,						eos_token_id=snake_case			,						is_encoder_decoder=snake_case			,						decoder_start_token_id=snake_case			,						encoder_no_repeat_ngram_size=snake_case			,						forced_eos_token_id=snake_case			,						**snake_case			,						)


class   __snake_case				(			_a   ):






    '''simple docstring'''





    @property
    def 	A		(   self						:		int							):

      """simple docstring"""





      if self.task in ["default", "seq2seq-lm"]:
        __lowerCAmelCase			=				OrderedDict(
            [
                ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
            ]							)
        if self.use_past:
          __lowerCAmelCase			=				{0: '''batch'''}
          __lowerCAmelCase			=				{0: '''batch''', 1: '''past_decoder_sequence + sequence'''}
        else:
          __lowerCAmelCase			=				{0: '''batch''', 1: '''decoder_sequence'''}
          __lowerCAmelCase			=				{0: '''batch''', 1: '''decoder_sequence'''}
        if self.use_past:
          self.fill_with_past_key_values_(snake_case			,						direction='''inputs'''							)
      elif self.task == "causal-lm":
        __lowerCAmelCase			=				OrderedDict(
            [
                ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
            ]							)
        if self.use_past:
          __lowerCAmelCase	,						__lowerCAmelCase			=				self.num_layers
          for i in range(snake_case							):
            __lowerCAmelCase			=				{0: '''batch''', 2: '''past_sequence + sequence'''}
            __lowerCAmelCase			=				{0: '''batch''', 2: '''past_sequence + sequence'''}
      else:
        __lowerCAmelCase			=				OrderedDict(
            [
                ('''input_ids''', {0: '''batch''', 1: '''encoder_sequence'''}),
                ('''attention_mask''', {0: '''batch''', 1: '''encoder_sequence'''}),
                ('''decoder_input_ids''', {0: '''batch''', 1: '''decoder_sequence'''}),
                ('''decoder_attention_mask''', {0: '''batch''', 1: '''decoder_sequence'''}),
            ]							)

      return common_inputs





    @property
    # Copied from transformers.models.bart.configuration_bart.BartOnnxConfig.outputs
    def 	A		(   self						:		Optional[Any]							):

      """simple docstring"""





      if self.task in ["default", "seq2seq-lm"]:
        __lowerCAmelCase			=				super().outputs
      else:
        __lowerCAmelCase			=				super(snake_case			,						self							).outputs
        if self.use_past:
          __lowerCAmelCase	,						__lowerCAmelCase			=				self.num_layers
          for i in range(snake_case							):
            __lowerCAmelCase			=				{0: '''batch''', 2: '''past_sequence + sequence'''}
            __lowerCAmelCase			=				{0: '''batch''', 2: '''past_sequence + sequence'''}
      return common_outputs





    def 	A		(   self						:		int			,						snake_case						:		PreTrainedTokenizer			,						snake_case						:		int = -1			,						snake_case						:		int = -1			,						snake_case						:		bool = False			,						snake_case						:		Optional[TensorType] = None			,						):

      """simple docstring"""





      __lowerCAmelCase			=				self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
          snake_case			,						snake_case			,						snake_case			,						snake_case			,						snake_case							)
      # Generate decoder inputs
      __lowerCAmelCase			=				seq_length if not self.use_past else 1
      __lowerCAmelCase			=				self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
          snake_case			,						snake_case			,						snake_case			,						snake_case			,						snake_case							)
      __lowerCAmelCase			=				{f'''decoder_{name}''': tensor for name, tensor in decoder_inputs.items()}
      __lowerCAmelCase			=				dict(**snake_case			,						**snake_case							)

      if self.use_past:
        if not is_torch_available():
          raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''							)
        else:
          import torch
        __lowerCAmelCase	,						__lowerCAmelCase			=				common_inputs['''input_ids'''].shape
        __lowerCAmelCase			=				common_inputs['''decoder_input_ids'''].shape[1]
        __lowerCAmelCase	,						__lowerCAmelCase			=				self.num_attention_heads
        __lowerCAmelCase			=				(
            batch,
            num_encoder_attention_heads,
            encoder_seq_length,
            self._config.hidden_size // num_encoder_attention_heads,
        )
        __lowerCAmelCase			=				decoder_seq_length
        __lowerCAmelCase			=				(
            batch,
            num_decoder_attention_heads,
            decoder_past_length,
            self._config.hidden_size // num_decoder_attention_heads,
        )
        __lowerCAmelCase			=				torch.cat(
            [common_inputs['''decoder_attention_mask'''], torch.ones(snake_case			,						snake_case							)]			,						dim=1							)
        __lowerCAmelCase			=				[]
        __lowerCAmelCase	,						__lowerCAmelCase			=				self.num_layers

        for _ in range(snake_case							):
          common_inputs["past_key_values"].append(
              (
                  torch.zeros(snake_case							),
                  torch.zeros(snake_case							),
                  torch.zeros(snake_case							),
                  torch.zeros(snake_case							),
              )							)
      return common_inputs





    def 	A		(   self						:		str			,						snake_case						:		PreTrainedTokenizer			,						snake_case						:		int = -1			,						snake_case						:		int = -1			,						snake_case						:		bool = False			,						snake_case						:		Optional[TensorType] = None			,						):

      """simple docstring"""





      __lowerCAmelCase			=				self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
          snake_case			,						snake_case			,						snake_case			,						snake_case			,						snake_case							)

      if self.use_past:
        if not is_torch_available():
          raise ValueError('''Cannot generate dummy past_keys inputs without PyTorch installed.'''							)
        else:
          import torch
        __lowerCAmelCase	,						__lowerCAmelCase			=				common_inputs['''input_ids'''].shape
        __lowerCAmelCase			=				seqlen
        __lowerCAmelCase	,						__lowerCAmelCase			=				self.num_layers
        __lowerCAmelCase	,						__lowerCAmelCase			=				self.num_attention_heads
        __lowerCAmelCase			=				(
            batch,
            num_encoder_attention_heads,
            past_key_values_length,
            self._config.hidden_size // num_encoder_attention_heads,
        )
        __lowerCAmelCase			=				common_inputs['''attention_mask'''].dtype
        __lowerCAmelCase			=				torch.cat(
            [common_inputs['''attention_mask'''], torch.ones(snake_case			,						snake_case			,						dtype=snake_case							)]			,						dim=1							)
        __lowerCAmelCase			=				[
            (torch.zeros(snake_case							), torch.zeros(snake_case							)) for _ in range(snake_case							)
        ]
      return common_inputs





    def 	A		(   self						:		Optional[int]			,						snake_case						:		PreTrainedTokenizer			,						snake_case						:		int = -1			,						snake_case						:		int = -1			,						snake_case						:		bool = False			,						snake_case						:		Optional[TensorType] = None			,						):

      """simple docstring"""





      __lowerCAmelCase			=				compute_effective_axis_dimension(
          snake_case			,						fixed_dimension=OnnxConfig.default_fixed_batch			,						num_token_to_add=0							)

      # If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
      __lowerCAmelCase			=				tokenizer.num_special_tokens_to_add(snake_case							)
      __lowerCAmelCase			=				compute_effective_axis_dimension(
          snake_case			,						fixed_dimension=OnnxConfig.default_fixed_sequence			,						num_token_to_add=snake_case							)

      # Generate dummy inputs according to compute batch and sequence
      __lowerCAmelCase			=				[''' '''.join([tokenizer.unk_token]							) * seq_length] * batch_size
      __lowerCAmelCase			=				dict(tokenizer(snake_case			,						return_tensors=snake_case							)							)
      return common_inputs





    def 	A		(   self						:		str			,						snake_case						:		PreTrainedTokenizer			,						snake_case						:		int = -1			,						snake_case						:		int = -1			,						snake_case						:		bool = False			,						snake_case						:		Optional[TensorType] = None			,						):

      """simple docstring"""





      if self.task in ["default", "seq2seq-lm"]:
        __lowerCAmelCase			=				self._generate_dummy_inputs_for_default_and_seqaseq_lm(
            snake_case			,						batch_size=snake_case			,						seq_length=snake_case			,						is_pair=snake_case			,						framework=snake_case							)

      elif self.task == "causal-lm":
        __lowerCAmelCase			=				self._generate_dummy_inputs_for_causal_lm(
            snake_case			,						batch_size=snake_case			,						seq_length=snake_case			,						is_pair=snake_case			,						framework=snake_case							)
      else:
        __lowerCAmelCase			=				self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
            snake_case			,						batch_size=snake_case			,						seq_length=snake_case			,						is_pair=snake_case			,						framework=snake_case							)

      return common_inputs





    def 	A		(   self						:		Optional[Any]			,						snake_case						:		int			,						snake_case						:		str			,						snake_case						:		str			,						snake_case						:		List[str]							):

      """simple docstring"""





      if self.task in ["default", "seq2seq-lm"]:
        __lowerCAmelCase			=				super()._flatten_past_key_values_(snake_case			,						snake_case			,						snake_case			,						snake_case							)
      else:
        __lowerCAmelCase			=				super(snake_case			,						self							)._flatten_past_key_values_(
            snake_case			,						snake_case			,						snake_case			,						snake_case							)





    def 	A		(   self						:		List[Any]			,						snake_case						:		Mapping[str, Mapping[int, str]]			,						snake_case						:		str							):

      """simple docstring"""





      if direction not in ["inputs", "outputs"]:
        raise ValueError(f'''direction must either be "inputs" or "outputs", but {direction} was given'''							)

      __lowerCAmelCase			=				'''past_key_values''' if direction == '''inputs''' else '''present'''
      __lowerCAmelCase	,						__lowerCAmelCase			=				self.num_layers

      __lowerCAmelCase			=				'''past_encoder_sequence'''
      __lowerCAmelCase			=				'''past_decoder_sequence''' if direction == '''inputs''' else '''past_decoder_sequence + sequence'''

      for i in range(snake_case							):
        __lowerCAmelCase			=				{0: '''batch''', 2: decoder_sequence}
        __lowerCAmelCase			=				{0: '''batch''', 2: decoder_sequence}
        __lowerCAmelCase			=				{0: '''batch''', 2: encoder_sequence}
        __lowerCAmelCase			=				{0: '''batch''', 2: encoder_sequence}
