






'''simple docstring'''







import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
       from .tokenization_mbartaa import MBartaaTokenizer
else:
       __A      :       List[Any]      =	None


__A      :       Any      =	logging.get_logger(__name__)

__A      :       Union[str, Any]      =	{'vocab_file': 'sentencepiece.bpe.model', 'tokenizer_file': 'tokenizer.json'}

__A      :       int      =	{
    'vocab_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'
        ),
    },
    'tokenizer_file': {
        'facebook/mbart-large-50-one-to-many-mmt': (
            'https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json'
        ),
    },
}

__A      :       Optional[Any]      =	{
    'facebook/mbart-large-50-one-to-many-mmt': 1_0_2_4,
}

# fmt: off
__A      :       List[Any]      =	['ar_AR', 'cs_CZ', 'de_DE', 'en_XX', 'es_XX', 'et_EE', 'fi_FI', 'fr_XX', 'gu_IN', 'hi_IN', 'it_IT', 'ja_XX', 'kk_KZ', 'ko_KR', 'lt_LT', 'lv_LV', 'my_MM', 'ne_NP', 'nl_XX', 'ro_RO', 'ru_RU', 'si_LK', 'tr_TR', 'vi_VN', 'zh_CN', 'af_ZA', 'az_AZ', 'bn_IN', 'fa_IR', 'he_IL', 'hr_HR', 'id_ID', 'ka_GE', 'km_KH', 'mk_MK', 'ml_IN', 'mn_MN', 'mr_IN', 'pl_PL', 'ps_AF', 'pt_XX', 'sv_SE', 'sw_KE', 'ta_IN', 'te_IN', 'th_TH', 'tl_XX', 'uk_UA', 'ur_PK', 'xh_ZA', 'gl_ES', 'sl_SI']






class 					a__    (			lowerCamelCase						):
     '''simple docstring'''







     _snake_case             =  VOCAB_FILES_NAMES
     _snake_case             =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _snake_case             =  PRETRAINED_VOCAB_FILES_MAP
     _snake_case             =  ['input_ids', 'attention_mask']
     _snake_case             =  MBartaaTokenizer

     _snake_case             =  []
     _snake_case             =  []






     def __init__(self    :	Optional[int]				,				lowercase__    :	Tuple=None				,				lowercase__    :	Union[str, Any]=None				,				lowercase__    :	Union[str, Any]=None				,				lowercase__    :	Any=None				,				lowercase__    :	List[Any]="</s>"				,				lowercase__    :	int="</s>"				,				lowercase__    :	Dict="<s>"				,				lowercase__    :	Dict="<unk>"				,				lowercase__    :	Tuple="<pad>"				,				lowercase__    :	Union[str, Any]="<mask>"				,				**lowercase__    :	Optional[int]				,				)  ->       List[str]:
           # Mask token behave like a normal word, i.e. include the space before it
           __A		   =      AddedToken(lowercase__				,				lstrip=lowercase__				,				rstrip=lowercase__					) if isinstance(lowercase__				,				lowercase__					) else mask_token

           __A		   =      kwargs.get("additional_special_tokens"				,				[]					)
           kwargs["additional_special_tokens"] += [
               code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
           ]

           super().__init__(
               lowercase__				,				src_lang=lowercase__				,				tgt_lang=lowercase__				,				tokenizer_file=lowercase__				,				eos_token=lowercase__				,				sep_token=lowercase__				,				cls_token=lowercase__				,				unk_token=lowercase__				,				pad_token=lowercase__				,				mask_token=lowercase__				,				**lowercase__				,				)

           __A		   =      vocab_file
           __A		   =      False if not self.vocab_file else True

           __A		   =      {
               lang_code: self.convert_tokens_to_ids(lowercase__					) for lang_code in FAIRSEQ_LANGUAGE_CODES
           }

           __A		   =      src_lang if src_lang is not None else "en_XX"
           __A		   =      tgt_lang
           __A		   =      self.lang_code_to_id[self._src_lang]
           self.set_src_lang_special_tokens(self._src_lang					)






     @property
     def     _UpperCamelCase   (self    :	str					)  ->       str:
           return self._src_lang






     @src_lang.setter
     def     _UpperCamelCase   (self    :	Optional[Any]				,				lowercase__    :	str					)  ->       None:
           __A		   =      new_src_lang
           self.set_src_lang_special_tokens(self._src_lang					)






     def     _UpperCamelCase   (self    :	Any				,				lowercase__    :	List[int]				,				lowercase__    :	Optional[List[int]] = None					)  ->       List[int]:
           if token_ids_a is None:
                 return self.prefix_tokens + token_ids_a + self.suffix_tokens
           # We don't expect to process pairs, but leave the pair logic for API consistency
           return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens






     def     _UpperCamelCase   (self    :	Union[str, Any]				,				lowercase__    :	List[str]				,				lowercase__    :	str = "en_XX"				,				lowercase__    :	Optional[List[str]] = None				,				lowercase__    :	str = "ro_RO"				,				**lowercase__    :	Tuple				,				)  ->       BatchEncoding:
           __A		   =      src_lang
           __A		   =      tgt_lang
           return super().prepare_seqaseq_batch(lowercase__				,				lowercase__				,				**lowercase__					)






     def     _UpperCamelCase   (self    :	Tuple					)  ->       Any:
           return self.set_src_lang_special_tokens(self.src_lang					)






     def     _UpperCamelCase   (self    :	List[Any]					)  ->       Any:
           return self.set_tgt_lang_special_tokens(self.tgt_lang					)






     def     _UpperCamelCase   (self    :	Tuple				,				lowercase__    :	str					)  ->       None:
           __A		   =      self.convert_tokens_to_ids(lowercase__					)
           __A		   =      [self.cur_lang_code_id]
           __A		   =      [self.eos_token_id]

           __A		   =      self.convert_ids_to_tokens(self.prefix_tokens					)
           __A		   =      self.convert_ids_to_tokens(self.suffix_tokens					)

           __A		   =      processors.TemplateProcessing(
               single=prefix_tokens_str + ["$A"] + suffix_tokens_str				,				pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str				,				special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,				self.prefix_tokens + self.suffix_tokens					)					)				,				)






     def     _UpperCamelCase   (self    :	List[str]				,				lowercase__    :	str					)  ->       None:
           __A		   =      self.convert_tokens_to_ids(lowercase__					)
           __A		   =      [self.cur_lang_code_id]
           __A		   =      [self.eos_token_id]

           __A		   =      self.convert_ids_to_tokens(self.prefix_tokens					)
           __A		   =      self.convert_ids_to_tokens(self.suffix_tokens					)

           __A		   =      processors.TemplateProcessing(
               single=prefix_tokens_str + ["$A"] + suffix_tokens_str				,				pair=prefix_tokens_str + ["$A", "$B"] + suffix_tokens_str				,				special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,				self.prefix_tokens + self.suffix_tokens					)					)				,				)






     def     _UpperCamelCase   (self    :	str				,				lowercase__    :	Tuple				,				lowercase__    :	str				,				lowercase__    :	Optional[str]				,				lowercase__    :	Optional[str]				,				**lowercase__    :	Dict					)  ->       List[Any]:
           if src_lang is None or tgt_lang is None:
                 raise ValueError("Translation requires a `src_lang` and a `tgt_lang` for this model"					)
           __A		   =      src_lang
           __A		   =      self(lowercase__				,				add_special_tokens=lowercase__				,				return_tensors=lowercase__				,				**lowercase__					)
           __A		   =      self.convert_tokens_to_ids(lowercase__					)
           __A		   =      tgt_lang_id
           return inputs






     def     _UpperCamelCase   (self    :	Tuple				,				lowercase__    :	str				,				lowercase__    :	Optional[str] = None					)  ->       Tuple[str]:
           if not self.can_save_slow_tokenizer:
                 raise ValueError(
                     "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                     "tokenizer."					)

           if not os.path.isdir(lowercase__					):
                 logger.error(f"Vocabulary path ({save_directory}) should be a directory"					)
                 return
           __A		   =      os.path.join(
               lowercase__				,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]					)

           if os.path.abspath(self.vocab_file					) != os.path.abspath(lowercase__					):
                 copyfile(self.vocab_file				,				lowercase__					)

           return (out_vocab_file,)










'''simple docstring'''







import copy
import inspect

from transformers.testing_utils import require_torch, torch_device
from transformers.utils.backbone_utils import BackboneType






@require_torch
class 					a__    :
     '''simple docstring'''







     _snake_case             =  ()
     _snake_case             =  True






     def     _UpperCamelCase   (self    :	Optional[Any]					)  ->       Union[str, Any]:
           __A		   =      self.config_class

           # test default config
           __A		   =      config_class()
           self.assertIsNotNone(lowercase__					)
           __A		   =      ["stem"] + [f"stage{idx}" for idx in range(1				,				len(config.depths					) + 1					)]
           self.assertEqual(config.stage_names				,				lowercase__					)
           self.assertTrue(set(config.out_features					).issubset(set(config.stage_names					)					)					)

           # Test out_features and out_indices are correctly set
           # out_features and out_indices both None
           __A		   =      config_class(out_features=lowercase__				,				out_indices=lowercase__					)
           self.assertEqual(config.out_features				,				[config.stage_names[-1]]					)
           self.assertEqual(config.out_indices				,				[len(config.stage_names					) - 1]					)

           # out_features and out_indices both set
           __A		   =      config_class(out_features=["stem", "stage1"]				,				out_indices=[0, 1]					)
           self.assertEqual(config.out_features				,				["stem", "stage1"]					)
           self.assertEqual(config.out_indices				,				[0, 1]					)

           # Only out_features set
           __A		   =      config_class(out_features=["stage1", "stage3"]					)
           self.assertEqual(config.out_features				,				["stage1", "stage3"]					)
           self.assertEqual(config.out_indices				,				[1, 3]					)

           # Only out_indices set
           __A		   =      config_class(out_indices=[0, 2]					)
           self.assertEqual(config.out_features				,				[config.stage_names[0], config.stage_names[2]]					)
           self.assertEqual(config.out_indices				,				[0, 2]					)

           # Error raised when out_indices do not correspond to out_features
           with self.assertRaises(lowercase__					):
                 __A		   =      config_class(out_features=["stage1", "stage2"]				,				out_indices=[0, 2]					)






     def     _UpperCamelCase   (self    :	Optional[int]					)  ->       Optional[Any]:
           __A   ,				__A		   =      self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
                 __A		   =      model_class(lowercase__					)
                 __A		   =      inspect.signature(model.forward					)
                 # signature.parameters is an OrderedDict => so arg_names order is deterministic
                 __A		   =      [*signature.parameters.keys()]
                 __A		   =      ["pixel_values"]
                 self.assertListEqual(arg_names[:1]				,				lowercase__					)






     def     _UpperCamelCase   (self    :	Dict					)  ->       Union[str, Any]:
           __A   ,				__A		   =      self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
                 __A		   =      model_class(lowercase__					)
                 self.assertEqual(len(model.channels					)				,				len(config.out_features					)					)
                 __A		   =      model.num_features
                 __A		   =      [config.stage_names.index(lowercase__					) for feat in config.out_features]
                 __A		   =      [num_features[idx] for idx in out_indices]
                 self.assertListEqual(model.channels				,				lowercase__					)

                 __A		   =      copy.deepcopy(lowercase__					)
                 __A		   =      None
                 __A		   =      model_class(lowercase__					)
                 self.assertEqual(len(model.channels					)				,				1					)
                 self.assertListEqual(model.channels				,				[num_features[-1]]					)

                 __A		   =      copy.deepcopy(lowercase__					)
                 __A		   =      None
                 __A		   =      model_class(lowercase__					)
                 self.assertEqual(len(model.channels					)				,				1					)
                 self.assertListEqual(model.channels				,				[num_features[-1]]					)






     def     _UpperCamelCase   (self    :	Optional[Any]					)  ->       Optional[Any]:
           __A   ,				__A		   =      self.model_tester.prepare_config_and_inputs_for_common()

           for model_class in self.all_model_classes:
                 __A		   =      model_class(lowercase__					)
                 model.to(lowercase__					)
                 model.eval()
                 __A		   =      model(**lowercase__					)

                 self.assertEqual(len(result.feature_maps					)				,				len(config.out_features					)					)
                 self.assertEqual(len(model.channels					)				,				len(config.out_features					)					)
                 self.assertEqual(len(result.feature_maps					)				,				len(config.out_indices					)					)
                 self.assertEqual(len(model.channels					)				,				len(config.out_indices					)					)

                 # Check output of last stage is taken if out_features=None, out_indices=None
                 __A		   =      copy.deepcopy(lowercase__					)
                 __A		   =      None
                 __A		   =      model_class(lowercase__					)
                 model.to(lowercase__					)
                 model.eval()
                 __A		   =      model(**lowercase__					)

                 self.assertEqual(len(result.feature_maps					)				,				1					)
                 self.assertEqual(len(model.channels					)				,				1					)

                 __A		   =      copy.deepcopy(lowercase__					)
                 __A		   =      None
                 __A		   =      model_class(lowercase__					)
                 model.to(lowercase__					)
                 model.eval()
                 __A		   =      model(**lowercase__					)

                 self.assertEqual(len(result.feature_maps					)				,				1					)
                 self.assertEqual(len(model.channels					)				,				1					)

                 # Check backbone can be initialized with fresh weights
                 __A		   =      copy.deepcopy(lowercase__					)
                 __A		   =      False
                 __A		   =      model_class(lowercase__					)
                 model.to(lowercase__					)
                 model.eval()
                 __A		   =      model(**lowercase__					)






     def     _UpperCamelCase   (self    :	Dict					)  ->       Optional[int]:
           __A   ,				__A		   =      self.model_tester.prepare_config_and_inputs_for_common()

           for backbone_class in self.all_model_classes:
                 __A		   =      backbone_class(lowercase__					)

                 self.assertTrue(hasattr(lowercase__				,				"backbone_type"					)					)
                 self.assertTrue(hasattr(lowercase__				,				"stage_names"					)					)
                 self.assertTrue(hasattr(lowercase__				,				"num_features"					)					)
                 self.assertTrue(hasattr(lowercase__				,				"out_indices"					)					)
                 self.assertTrue(hasattr(lowercase__				,				"out_features"					)					)
                 self.assertTrue(hasattr(lowercase__				,				"out_feature_channels"					)					)
                 self.assertTrue(hasattr(lowercase__				,				"channels"					)					)

                 self.assertIsInstance(backbone.backbone_type				,				lowercase__					)
                 # Verify num_features has been initialized in the backbone init
                 self.assertIsNotNone(backbone.num_features					)
                 self.assertTrue(len(backbone.channels					) == len(backbone.out_indices					)					)
                 self.assertTrue(len(backbone.stage_names					) == len(backbone.num_features					)					)
                 self.assertTrue(len(backbone.channels					) <= len(backbone.num_features					)					)
                 self.assertTrue(len(backbone.out_feature_channels					) == len(backbone.stage_names					)					)






     def     _UpperCamelCase   (self    :	Tuple					)  ->       int:
           __A   ,				__A		   =      self.model_tester.prepare_config_and_inputs_for_common()
           __A		   =      inputs_dict["pixel_values"].shape[0]

           for backbone_class in self.all_model_classes:
                 __A		   =      backbone_class(lowercase__					)
                 backbone.to(lowercase__					)
                 backbone.eval()

                 __A		   =      backbone(**lowercase__					)

                 # Test default outputs and verify feature maps
                 self.assertIsInstance(outputs.feature_maps				,				lowercase__					)
                 self.assertTrue(len(outputs.feature_maps					) == len(backbone.channels					)					)
                 for feature_map, n_channels in zip(outputs.feature_maps				,				backbone.channels					):
                       self.assertTrue(feature_map.shape[:2]				,				(batch_size, n_channels)					)
                 self.assertIsNone(outputs.hidden_states					)
                 self.assertIsNone(outputs.attentions					)

                 # Test output_hidden_states=True
                 __A		   =      backbone(**lowercase__				,				output_hidden_states=lowercase__					)
                 self.assertIsNotNone(outputs.hidden_states					)
                 self.assertTrue(len(outputs.hidden_states					)				,				len(backbone.stage_names					)					)
                 for hidden_state, n_channels in zip(outputs.hidden_states				,				backbone.channels					):
                       self.assertTrue(hidden_state.shape[:2]				,				(batch_size, n_channels)					)

                 # Test output_attentions=True
                 if self.has_attentions:
                       __A		   =      backbone(**lowercase__				,				output_attentions=lowercase__					)
                       self.assertIsNotNone(outputs.attentions					)


