






'''simple docstring'''


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
  from .tokenization_mbartaa import MBartaaTokenizer
else:
  SCREAMING_SNAKE_CASE_				      =							None


SCREAMING_SNAKE_CASE_				      =							logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_				      =							{"vocab_file": "sentencepiece.bpe.model", "tokenizer_file": "tokenizer.json"}

SCREAMING_SNAKE_CASE_				      =							{
    "vocab_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model"
        ),
    },
    "tokenizer_file": {
        "facebook/mbart-large-50-one-to-many-mmt": (
            "https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json"
        ),
    },
}

SCREAMING_SNAKE_CASE_				      =							{
    "facebook/mbart-large-50-one-to-many-mmt": 1_0_2_4,
}

# fmt: off
SCREAMING_SNAKE_CASE_				      =							["ar_AR", "cs_CZ", "de_DE", "en_XX", "es_XX", "et_EE", "fi_FI", "fr_XX", "gu_IN", "hi_IN", "it_IT", "ja_XX", "kk_KZ", "ko_KR", "lt_LT", "lv_LV", "my_MM", "ne_NP", "nl_XX", "ro_RO", "ru_RU", "si_LK", "tr_TR", "vi_VN", "zh_CN", "af_ZA", "az_AZ", "bn_IN", "fa_IR", "he_IL", "hr_HR", "id_ID", "ka_GE", "km_KH", "mk_MK", "ml_IN", "mn_MN", "mr_IN", "pl_PL", "ps_AF", "pt_XX", "sv_SE", "sw_KE", "ta_IN", "te_IN", "th_TH", "tl_XX", "uk_UA", "ur_PK", "xh_ZA", "gl_ES", "sl_SI"]




class 							lowerCAmelCase					(							lowerCamelCase__   ):







   """simple docstring"""







   _A						=   VOCAB_FILES_NAMES
   _A						=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _A						=   PRETRAINED_VOCAB_FILES_MAP
   _A						=   ['input_ids', 'attention_mask']
   _A						=   MBartaaTokenizer

   _A						=   []
   _A						=   []



   def __init__( self   ,   _A=None   ,   _A=None   ,   _A=None   ,   _A=None   ,   _A="</s>"   ,   _A="</s>"   ,   _A="<s>"   ,   _A="<unk>"   ,   _A="<pad>"   ,   _A="<mask>"   ,   **_A   ,   )    ->   int:
    # Mask token behave like a normal word, i.e. include the space before it
    __a    :	Union[str, Any]									=						AddedToken(_A   ,   lstrip=_A   ,   rstrip=_A			) if isinstance(_A   ,   _A			) else mask_token

    __a    :	List[str]									=						kwargs.get('additional_special_tokens'   ,   []			)
    kwargs["additional_special_tokens"] += [
        code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
    ]

    super().__init__(
        _A   ,   src_lang=_A   ,   tgt_lang=_A   ,   tokenizer_file=_A   ,   eos_token=_A   ,   sep_token=_A   ,   cls_token=_A   ,   unk_token=_A   ,   pad_token=_A   ,   mask_token=_A   ,   **_A   ,   )

    __a    :	int									=						vocab_file
    __a    :	Dict									=						False if not self.vocab_file else True

    __a    :	str									=						{
        lang_code: self.convert_tokens_to_ids(_A			) for lang_code in FAIRSEQ_LANGUAGE_CODES
    }

    __a    :	Tuple									=						src_lang if src_lang is not None else 'en_XX'
    __a    :	Dict									=						tgt_lang
    __a    :	List[Any]									=						self.lang_code_to_id[self._src_lang]
    self.set_src_lang_special_tokens(self._src_lang			)



   @property
   def    __magic_name__    ( self			)    ->   str:
    return self._src_lang



   @src_lang.setter
   def    __magic_name__    ( self   ,   _A			)    ->   None:
    __a    :	List[str]									=						new_src_lang
    self.set_src_lang_special_tokens(self._src_lang			)



   def    __magic_name__    ( self   ,   _A   ,   _A = None			)    ->   List[int]:
    if token_ids_a is None:
     return self.prefix_tokens + token_ids_a + self.suffix_tokens
    # We don't expect to process pairs, but leave the pair logic for API consistency
    return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens



   def    __magic_name__    ( self   ,   _A   ,   _A = "en_XX"   ,   _A = None   ,   _A = "ro_RO"   ,   **_A   ,   )    ->   BatchEncoding:
    __a    :	List[str]									=						src_lang
    __a    :	Dict									=						tgt_lang
    return super().prepare_seqaseq_batch(_A   ,   _A   ,   **_A			)



   def    __magic_name__    ( self			)    ->   Any:
    return self.set_src_lang_special_tokens(self.src_lang			)



   def    __magic_name__    ( self			)    ->   Optional[int]:
    return self.set_tgt_lang_special_tokens(self.tgt_lang			)



   def    __magic_name__    ( self   ,   _A			)    ->   None:
    __a    :	Tuple									=						self.convert_tokens_to_ids(_A			)
    __a    :	Tuple									=						[self.cur_lang_code_id]
    __a    :	List[str]									=						[self.eos_token_id]

    __a    :	Tuple									=						self.convert_ids_to_tokens(self.prefix_tokens			)
    __a    :	Optional[Any]									=						self.convert_ids_to_tokens(self.suffix_tokens			)

    __a    :	str									=						processors.TemplateProcessing(
        single=prefix_tokens_str + ['$A'] + suffix_tokens_str   ,   pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str   ,   special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str   ,   self.prefix_tokens + self.suffix_tokens			)			)   ,   )



   def    __magic_name__    ( self   ,   _A			)    ->   None:
    __a    :	str									=						self.convert_tokens_to_ids(_A			)
    __a    :	List[Any]									=						[self.cur_lang_code_id]
    __a    :	Tuple									=						[self.eos_token_id]

    __a    :	Optional[Any]									=						self.convert_ids_to_tokens(self.prefix_tokens			)
    __a    :	int									=						self.convert_ids_to_tokens(self.suffix_tokens			)

    __a    :	Optional[int]									=						processors.TemplateProcessing(
        single=prefix_tokens_str + ['$A'] + suffix_tokens_str   ,   pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str   ,   special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str   ,   self.prefix_tokens + self.suffix_tokens			)			)   ,   )



   def    __magic_name__    ( self   ,   _A   ,   _A   ,   _A   ,   _A   ,   **_A			)    ->   List[Any]:
    if src_lang is None or tgt_lang is None:
     raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'			)
    __a    :	Union[str, Any]									=						src_lang
    __a    :	str									=						self(_A   ,   add_special_tokens=_A   ,   return_tensors=_A   ,   **_A			)
    __a    :	Optional[Any]									=						self.convert_tokens_to_ids(_A			)
    __a    :	Tuple									=						tgt_lang_id
    return inputs



   def    __magic_name__    ( self   ,   _A   ,   _A = None			)    ->   Tuple[str]:
    if not self.can_save_slow_tokenizer:
     raise ValueError(
         'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
         'tokenizer.'			)

    if not os.path.isdir(_A			):
     logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''			)
     return
    __a    :	int									=						os.path.join(
        _A   ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)

    if os.path.abspath(self.vocab_file			) != os.path.abspath(_A			):
     copyfile(self.vocab_file   ,   _A			)

    return (out_vocab_file,)












'''simple docstring'''


from typing import List, Optional, Tuple, Union

from . import logging
from .import_utils import is_torch_available, is_torch_version


if is_torch_available():
  import torch

SCREAMING_SNAKE_CASE_				      =							logging.get_logger(__name__)  # pylint: disable=invalid-name

try:
  from torch._dynamo import allow_in_graph as maybe_allow_in_graph
except (ImportError, ModuleNotFoundError):






  def   lowerCAmelCase__			(					cls		):
   return cls






def   lowerCAmelCase__			(					SCREAMING_SNAKE_CASE__       ,		SCREAMING_SNAKE_CASE__ = None       ,		SCREAMING_SNAKE_CASE__ = None       ,		SCREAMING_SNAKE_CASE__ = None       ,		SCREAMING_SNAKE_CASE__ = None       ,		):
 __a    :	int									=						device
 __a    :	Union[str, Any]									=						shape[0]

 __a    :	Union[str, Any]									=						layout or torch.strided
 __a    :	Any									=						device or torch.device('cpu'		)

 if generator is not None:
  __a    :	Optional[int]									=						generator.device.type if not isinstance(SCREAMING_SNAKE_CASE__       ,		SCREAMING_SNAKE_CASE__		) else generator[0].device.type
  if gen_device_type != device.type and gen_device_type == "cpu":
   __a    :	int									=						'cpu'
   if device != "mps":
    logger.info(
        f'''The passed generator was created on \'cpu\' even though a tensor on {device} was expected.'''
        f''' Tensors will be created on \'cpu\' and then moved to {device}. Note that one can probably'''
        f''' slighly speed up this function by passing a generator that was created on the {device} device.'''		)
  elif gen_device_type != device.type and gen_device_type == "cuda":
   raise ValueError(f'''Cannot generate a {device} tensor from a generator of type {gen_device_type}.'''		)

 if isinstance(SCREAMING_SNAKE_CASE__       ,		SCREAMING_SNAKE_CASE__		):
  __a    :	Optional[int]									=						(1,) + shape[1:]
  __a    :	Dict									=						[
      torch.randn(SCREAMING_SNAKE_CASE__       ,		generator=generator[i]       ,		device=SCREAMING_SNAKE_CASE__       ,		dtype=SCREAMING_SNAKE_CASE__       ,		layout=SCREAMING_SNAKE_CASE__		)
      for i in range(SCREAMING_SNAKE_CASE__		)
  ]
  __a    :	List[str]									=						torch.cat(SCREAMING_SNAKE_CASE__       ,		dim=0		).to(SCREAMING_SNAKE_CASE__		)
 else:
  __a    :	Tuple									=						torch.randn(SCREAMING_SNAKE_CASE__       ,		generator=SCREAMING_SNAKE_CASE__       ,		device=SCREAMING_SNAKE_CASE__       ,		dtype=SCREAMING_SNAKE_CASE__       ,		layout=SCREAMING_SNAKE_CASE__		).to(SCREAMING_SNAKE_CASE__		)

 return latents






def   lowerCAmelCase__			(					SCREAMING_SNAKE_CASE__		):
 if is_torch_version('<'       ,		'2.0.0'		) or not hasattr(SCREAMING_SNAKE_CASE__       ,		'_dynamo'		):
  return False
 return isinstance(SCREAMING_SNAKE_CASE__       ,		torch._dynamo.eval_frame.OptimizedModule		)




