






"""simple docstring"""


from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig, OnnxConfigWithPast, OnnxSeqaSeqConfigWithPast
from ...onnx.utils import compute_effective_axis_dimension
from ...utils import TensorType, is_torch_available, logging


__SCREAMING_SNAKE_CASE      =      logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE      =      {
    'facebook/mbart-large-cc25': 'https://huggingface.co/facebook/mbart-large-cc25/resolve/main/config.json',
    # See all MBART models at https://huggingface.co/models?filter=mbart
}




class 	__UpperCamelCase    (    UpperCamelCase	):
					lowercase_						:							int			=     """mbart"""
					lowercase_						:							Any			=     ["""past_key_values"""]
					lowercase_						:							int			=     {"""num_attention_heads""": """encoder_attention_heads""", """hidden_size""": """d_model"""}
					def __init__(       self       :  List[str]    ,   UpperCAmelCase       :  Any=5_0265    ,   UpperCAmelCase       :  Dict=1024    ,   UpperCAmelCase       :  str=12    ,   UpperCAmelCase       :  List[Any]=4096    ,   UpperCAmelCase       :  int=16    ,   UpperCAmelCase       :  Optional[Any]=12    ,   UpperCAmelCase       :  int=4096    ,   UpperCAmelCase       :  Tuple=16    ,   UpperCAmelCase       :  Any=0.0    ,   UpperCAmelCase       :  Union[str, Any]=0.0    ,   UpperCAmelCase       :  List[Any]=True    ,   UpperCAmelCase       :  Union[str, Any]=True    ,   UpperCAmelCase       :  str="gelu"    ,   UpperCAmelCase       :  int=1024    ,   UpperCAmelCase       :  Dict=0.1    ,   UpperCAmelCase       :  List[str]=0.0    ,   UpperCAmelCase       :  Tuple=0.0    ,   UpperCAmelCase       :  Union[str, Any]=0.0_2    ,   UpperCAmelCase       :  Any=0.0    ,   UpperCAmelCase       :  Union[str, Any]=False    ,   UpperCAmelCase       :  Optional[Any]=1    ,   UpperCAmelCase       :  Optional[Any]=0    ,   UpperCAmelCase       :  List[str]=2    ,   UpperCAmelCase       :  Tuple=2    ,   **UpperCAmelCase       :  List[Any]    ,   )  ->					Optional[Any]:
											lowerCAmelCase      :Union[str, Any]				  = vocab_size
											lowerCAmelCase      :Tuple				  = max_position_embeddings
											lowerCAmelCase      :int				  = d_model
											lowerCAmelCase      :Union[str, Any]				  = encoder_ffn_dim
											lowerCAmelCase      :Union[str, Any]				  = encoder_layers
											lowerCAmelCase      :Any				  = encoder_attention_heads
											lowerCAmelCase      :List[Any]				  = decoder_ffn_dim
											lowerCAmelCase      :int				  = decoder_layers
											lowerCAmelCase      :Union[str, Any]				  = decoder_attention_heads
											lowerCAmelCase      :Tuple				  = dropout
											lowerCAmelCase      :Tuple				  = attention_dropout
											lowerCAmelCase      :str				  = activation_dropout
											lowerCAmelCase      :Any				  = activation_function
											lowerCAmelCase      :Optional[Any]				  = init_std
											lowerCAmelCase      :Any				  = encoder_layerdrop
											lowerCAmelCase      :Optional[int]				  = decoder_layerdrop
											lowerCAmelCase      :Any				  = classifier_dropout
											lowerCAmelCase      :List[Any]				  = use_cache
											lowerCAmelCase      :str				  = encoder_layers
											lowerCAmelCase      :Union[str, Any]				  = scale_embedding  # scale factor will be sqrt(d_model) if True
											super().__init__(
											    pad_token_id=UpperCAmelCase    ,   bos_token_id=UpperCAmelCase    ,   eos_token_id=UpperCAmelCase    ,   is_encoder_decoder=UpperCAmelCase    ,   forced_eos_token_id=UpperCAmelCase    ,   **UpperCAmelCase    ,   )




class 	__UpperCamelCase    (    UpperCamelCase	):
					@property
					def 					UpperCAmelCase__   (       self       :  Union[str, Any]      )  ->					Mapping[str, Mapping[int, str]]:
											if self.task in ["default", "seq2seq-lm"]:
																	lowerCAmelCase      :List[str]				  = OrderedDict(
																	    [
																	        ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
																	        ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
																	    ]      )

																	if self.use_past:
																							lowerCAmelCase      :List[str]				  = {0: 'batch'}
																							lowerCAmelCase      :Dict				  = {0: 'batch', 1: 'past_decoder_sequence + sequence'}
																	else:
																							lowerCAmelCase      :str				  = {0: 'batch', 1: 'decoder_sequence'}
																							lowerCAmelCase      :int				  = {0: 'batch', 1: 'decoder_sequence'}

																	if self.use_past:
																							self.fill_with_past_key_values_(UpperCAmelCase    ,   direction='inputs'      )
											elif self.task == "causal-lm":
																	# TODO: figure this case out.
																	lowerCAmelCase      :Optional[Any]				  = OrderedDict(
																	    [
																	        ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
																	        ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
																	    ]      )
																	if self.use_past:
																							lowerCAmelCase  ,   lowerCAmelCase      :Any				  = self.num_layers
																							for i in range(UpperCAmelCase      ):
																													lowerCAmelCase      :Tuple				  = {0: 'batch', 2: 'past_sequence + sequence'}
																													lowerCAmelCase      :Any				  = {0: 'batch', 2: 'past_sequence + sequence'}
											else:
																	lowerCAmelCase      :int				  = OrderedDict(
																	    [
																	        ('input_ids', {0: 'batch', 1: 'encoder_sequence'}),
																	        ('attention_mask', {0: 'batch', 1: 'encoder_sequence'}),
																	        ('decoder_input_ids', {0: 'batch', 1: 'decoder_sequence'}),
																	        ('decoder_attention_mask', {0: 'batch', 1: 'decoder_sequence'}),
																	    ]      )

											return common_inputs
					@property
					def 					UpperCAmelCase__   (       self       :  List[Any]      )  ->					Mapping[str, Mapping[int, str]]:
											if self.task in ["default", "seq2seq-lm"]:
																	lowerCAmelCase      :List[str]				  = super().outputs
											else:
																	lowerCAmelCase      :Any				  = super(UpperCAmelCase    ,   self      ).outputs
																	if self.use_past:
																							lowerCAmelCase  ,   lowerCAmelCase      :Optional[int]				  = self.num_layers
																							for i in range(UpperCAmelCase      ):
																													lowerCAmelCase      :Optional[Any]				  = {0: 'batch', 2: 'past_sequence + sequence'}
																													lowerCAmelCase      :List[str]				  = {0: 'batch', 2: 'past_sequence + sequence'}
											return common_outputs
					def 					UpperCAmelCase__   (       self       :  List[Any]    ,   UpperCAmelCase       :  PreTrainedTokenizer    ,   UpperCAmelCase       :  int = -1    ,   UpperCAmelCase       :  int = -1    ,   UpperCAmelCase       :  bool = False    ,   UpperCAmelCase       :  Optional[TensorType] = None    ,   )  ->					Mapping[str, Any]:
											lowerCAmelCase      :Any				  = self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
											    UpperCAmelCase    ,   UpperCAmelCase    ,   UpperCAmelCase    ,   UpperCAmelCase    ,   UpperCAmelCase      )

											# Generate decoder inputs
											lowerCAmelCase      :Union[str, Any]				  = seq_length if not self.use_past else 1
											lowerCAmelCase      :Tuple				  = self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
											    UpperCAmelCase    ,   UpperCAmelCase    ,   UpperCAmelCase    ,   UpperCAmelCase    ,   UpperCAmelCase      )
											lowerCAmelCase      :Any				  = {f"""decoder_{name}""": tensor for name, tensor in decoder_inputs.items()}
											lowerCAmelCase      :Dict				  = dict(**UpperCAmelCase    ,   **UpperCAmelCase      )

											if self.use_past:
																	if not is_torch_available():
																							raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'      )
																	else:
																							import torch
																	lowerCAmelCase  ,   lowerCAmelCase      :Union[str, Any]				  = common_inputs['input_ids'].shape
																	lowerCAmelCase      :Dict				  = common_inputs['decoder_input_ids'].shape[1]
																	lowerCAmelCase  ,   lowerCAmelCase      :Union[str, Any]				  = self.num_attention_heads
																	lowerCAmelCase      :str				  = (
																	    batch,
																	    num_encoder_attention_heads,
																	    encoder_seq_length,
																	    self._config.hidden_size // num_encoder_attention_heads,
																	)
																	lowerCAmelCase      :Dict				  = decoder_seq_length + 3
																	lowerCAmelCase      :int				  = (
																	    batch,
																	    num_decoder_attention_heads,
																	    decoder_past_length,
																	    self._config.hidden_size // num_decoder_attention_heads,
																	)

																	lowerCAmelCase      :List[Any]				  = torch.cat(
																	    [common_inputs['decoder_attention_mask'], torch.ones(UpperCAmelCase    ,   UpperCAmelCase      )]    ,   dim=1      )

																	lowerCAmelCase      :Any				  = []
																	# If the number of encoder and decoder layers are present in the model configuration, both are considered
																	lowerCAmelCase  ,   lowerCAmelCase      :int				  = self.num_layers
																	lowerCAmelCase      :Optional[Any]				  = min(UpperCAmelCase    ,   UpperCAmelCase      )
																	lowerCAmelCase      :Optional[Any]				  = max(UpperCAmelCase    ,   UpperCAmelCase      ) - min_num_layers
																	lowerCAmelCase      :int				  = 'encoder' if num_encoder_layers > num_decoder_layers else 'decoder'

																	for _ in range(UpperCAmelCase      ):
																							common_inputs["past_key_values"].append(
																							    (
																							        torch.zeros(UpperCAmelCase      ),
																							        torch.zeros(UpperCAmelCase      ),
																							        torch.zeros(UpperCAmelCase      ),
																							        torch.zeros(UpperCAmelCase      ),
																							    )      )
																	# TODO: test this.
																	lowerCAmelCase      :int				  = encoder_shape if remaining_side_name == 'encoder' else decoder_shape
																	for _ in range(UpperCAmelCase    ,   UpperCAmelCase      ):
																							common_inputs["past_key_values"].append((torch.zeros(UpperCAmelCase      ), torch.zeros(UpperCAmelCase      ))      )
											return common_inputs
					def 					UpperCAmelCase__   (       self       :  List[Any]    ,   UpperCAmelCase       :  PreTrainedTokenizer    ,   UpperCAmelCase       :  int = -1    ,   UpperCAmelCase       :  int = -1    ,   UpperCAmelCase       :  bool = False    ,   UpperCAmelCase       :  Optional[TensorType] = None    ,   )  ->					Mapping[str, Any]:
											lowerCAmelCase      :Dict				  = self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
											    UpperCAmelCase    ,   UpperCAmelCase    ,   UpperCAmelCase    ,   UpperCAmelCase    ,   UpperCAmelCase      )

											if self.use_past:
																	if not is_torch_available():
																							raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'      )
																	else:
																							import torch
																	lowerCAmelCase  ,   lowerCAmelCase      :List[Any]				  = common_inputs['input_ids'].shape
																	# Not using the same length for past_key_values
																	lowerCAmelCase      :List[Any]				  = seqlen + 2
																	lowerCAmelCase  ,   lowerCAmelCase      :List[Any]				  = self.num_layers
																	lowerCAmelCase  ,   lowerCAmelCase      :Tuple				  = self.num_attention_heads
																	lowerCAmelCase      :List[Any]				  = (
																	    batch,
																	    num_encoder_attention_heads,
																	    past_key_values_length,
																	    self._config.hidden_size // num_encoder_attention_heads,
																	)

																	lowerCAmelCase      :List[Any]				  = common_inputs['attention_mask'].dtype
																	lowerCAmelCase      :int				  = torch.cat(
																	    [common_inputs['attention_mask'], torch.ones(UpperCAmelCase    ,   UpperCAmelCase    ,   dtype=UpperCAmelCase      )]    ,   dim=1      )
																	lowerCAmelCase      :Dict				  = [
																	    (torch.zeros(UpperCAmelCase      ), torch.zeros(UpperCAmelCase      )) for _ in range(UpperCAmelCase      )
																	]
											return common_inputs
					def 					UpperCAmelCase__   (       self       :  List[str]    ,   UpperCAmelCase       :  PreTrainedTokenizer    ,   UpperCAmelCase       :  int = -1    ,   UpperCAmelCase       :  int = -1    ,   UpperCAmelCase       :  bool = False    ,   UpperCAmelCase       :  Optional[TensorType] = None    ,   )  ->					Mapping[str, Any]:
											# Copied from OnnxConfig.generate_dummy_inputs
											# Did not use super(OnnxConfigWithPast, self).generate_dummy_inputs for code clarity.
											# If dynamic axis (-1) we forward with a fixed dimension of 2 samples to avoid optimizations made by ONNX
											lowerCAmelCase      :Optional[Any]				  = compute_effective_axis_dimension(
											    UpperCAmelCase    ,   fixed_dimension=OnnxConfig.default_fixed_batch    ,   num_token_to_add=0      )

											# If dynamic axis (-1) we forward with a fixed dimension of 8 tokens to avoid optimizations made by ONNX
											lowerCAmelCase      :List[str]				  = tokenizer.num_special_tokens_to_add(UpperCAmelCase      )
											lowerCAmelCase      :Optional[Any]				  = compute_effective_axis_dimension(
											    UpperCAmelCase    ,   fixed_dimension=OnnxConfig.default_fixed_sequence    ,   num_token_to_add=UpperCAmelCase      )

											# Generate dummy inputs according to compute batch and sequence
											lowerCAmelCase      :Union[str, Any]				  = [' '.join([tokenizer.unk_token]      ) * seq_length] * batch_size
											lowerCAmelCase      :Union[str, Any]				  = dict(tokenizer(UpperCAmelCase    ,   return_tensors=UpperCAmelCase      )      )
											return common_inputs
					def 					UpperCAmelCase__   (       self       :  int    ,   UpperCAmelCase       :  PreTrainedTokenizer    ,   UpperCAmelCase       :  int = -1    ,   UpperCAmelCase       :  int = -1    ,   UpperCAmelCase       :  bool = False    ,   UpperCAmelCase       :  Optional[TensorType] = None    ,   )  ->					Mapping[str, Any]:
											if self.task in ["default", "seq2seq-lm"]:
																	lowerCAmelCase      :int				  = self._generate_dummy_inputs_for_default_and_seqaseq_lm(
																	    UpperCAmelCase    ,   batch_size=UpperCAmelCase    ,   seq_length=UpperCAmelCase    ,   is_pair=UpperCAmelCase    ,   framework=UpperCAmelCase      )

											elif self.task == "causal-lm":
																	lowerCAmelCase      :Optional[Any]				  = self._generate_dummy_inputs_for_causal_lm(
																	    UpperCAmelCase    ,   batch_size=UpperCAmelCase    ,   seq_length=UpperCAmelCase    ,   is_pair=UpperCAmelCase    ,   framework=UpperCAmelCase      )
											else:
																	lowerCAmelCase      :Optional[Any]				  = self._generate_dummy_inputs_for_sequence_classification_and_question_answering(
																	    UpperCAmelCase    ,   batch_size=UpperCAmelCase    ,   seq_length=UpperCAmelCase    ,   is_pair=UpperCAmelCase    ,   framework=UpperCAmelCase      )

											return common_inputs
					def 					UpperCAmelCase__   (       self       :  Optional[int]    ,   UpperCAmelCase       :  Union[str, Any]    ,   UpperCAmelCase       :  int    ,   UpperCAmelCase       :  List[str]    ,   UpperCAmelCase       :  List[str]      )  ->					Optional[Any]:
											if self.task in ["default", "seq2seq-lm"]:
																	lowerCAmelCase      :Optional[Any]				  = super()._flatten_past_key_values_(UpperCAmelCase    ,   UpperCAmelCase    ,   UpperCAmelCase    ,   UpperCAmelCase      )
											else:
																	lowerCAmelCase      :Optional[Any]				  = super(UpperCAmelCase    ,   self      )._flatten_past_key_values_(
																	    UpperCAmelCase    ,   UpperCAmelCase    ,   UpperCAmelCase    ,   UpperCAmelCase      )







"""simple docstring"""


import unittest
import warnings
from dataclasses import dataclass

from transformers.convert_slow_tokenizer import SpmConverter
from transformers.testing_utils import get_tests_dir




@dataclass
class 	__UpperCamelCase    :
					lowercase_						:							str




class 	__UpperCamelCase    (    unittest.TestCase	):
					def 					UpperCAmelCase__   (       self       :  Union[str, Any]      )  ->					Optional[Any]:
											lowerCAmelCase      :List[str]				  = get_tests_dir('fixtures/test_sentencepiece.model'      )
											lowerCAmelCase      :Tuple				  = get_tests_dir('fixtures/test_sentencepiece_with_bytefallback.model'      )

											lowerCAmelCase      :List[str]				  = FakeOriginalTokenizer(vocab_file=UpperCAmelCase      )

											with warnings.catch_warnings(record=UpperCAmelCase      ) as w:
																	lowerCAmelCase      :Dict				  = SpmConverter(UpperCAmelCase      )
											self.assertEqual(len(UpperCAmelCase      )    ,   0      )

											lowerCAmelCase      :int				  = FakeOriginalTokenizer(vocab_file=UpperCAmelCase      )

											with warnings.catch_warnings(record=UpperCAmelCase      ) as w:
																	lowerCAmelCase      :List[str]				  = SpmConverter(UpperCAmelCase      )
											self.assertEqual(len(UpperCAmelCase      )    ,   1      )

											self.assertIn(
											    'The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option'
											    ' which is not implemented in the fast tokenizers.'    ,   str(w[0].message      )    ,   )