





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
		from .tokenization_mbartaa import MBartaaTokenizer
else:
		SCREAMING_SNAKE_CASE_      :			int		  =      None


SCREAMING_SNAKE_CASE_      :			Optional[int]		  =      logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_      :			Any		  =      {'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

SCREAMING_SNAKE_CASE_      :			Union[str, Any]		  =      {
    '''vocab_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/mbart-large-50-one-to-many-mmt''': (
            '''https://huggingface.co/facebook/mbart-large-50-one-to-many-mmt/resolve/main/tokenizer.json'''
        ),
    },
}

SCREAMING_SNAKE_CASE_      :			str		  =      {
    '''facebook/mbart-large-50-one-to-many-mmt''': 1024,
}

# fmt: off
SCREAMING_SNAKE_CASE_      :			int		  =      ['''ar_AR''', '''cs_CZ''', '''de_DE''', '''en_XX''', '''es_XX''', '''et_EE''', '''fi_FI''', '''fr_XX''', '''gu_IN''', '''hi_IN''', '''it_IT''', '''ja_XX''', '''kk_KZ''', '''ko_KR''', '''lt_LT''', '''lv_LV''', '''my_MM''', '''ne_NP''', '''nl_XX''', '''ro_RO''', '''ru_RU''', '''si_LK''', '''tr_TR''', '''vi_VN''', '''zh_CN''', '''af_ZA''', '''az_AZ''', '''bn_IN''', '''fa_IR''', '''he_IL''', '''hr_HR''', '''id_ID''', '''ka_GE''', '''km_KH''', '''mk_MK''', '''ml_IN''', '''mn_MN''', '''mr_IN''', '''pl_PL''', '''ps_AF''', '''pt_XX''', '''sv_SE''', '''sw_KE''', '''ta_IN''', '''te_IN''', '''th_TH''', '''tl_XX''', '''uk_UA''', '''ur_PK''', '''xh_ZA''', '''gl_ES''', '''sl_SI''']

class 		snake_case_      (       UpperCAmelCase_	):







	'''simple docstring'''
	__UpperCamelCase        =		VOCAB_FILES_NAMES
	__UpperCamelCase        =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	__UpperCamelCase        =		PRETRAINED_VOCAB_FILES_MAP
	__UpperCamelCase        =		['''input_ids''', '''attention_mask''']
	__UpperCamelCase        =		MBartaaTokenizer

	__UpperCamelCase        =		[]
	__UpperCamelCase        =		[]




	def __init__(	self       :    int				,			__lowerCamelCase       :    Dict=None				,			__lowerCamelCase       :    Optional[Any]=None				,			__lowerCamelCase       :    str=None				,			__lowerCamelCase       :    Tuple=None				,			__lowerCamelCase       :    Tuple="</s>"				,			__lowerCamelCase       :    int="</s>"				,			__lowerCamelCase       :    Optional[int]="<s>"				,			__lowerCamelCase       :    str="<unk>"				,			__lowerCamelCase       :    Optional[int]="<pad>"				,			__lowerCamelCase       :    Tuple="<mask>"				,			**__lowerCamelCase       :    List[Any]				,			)					->   List[Any]:

			'''simple docstring'''




			__lowercase					=    AddedToken(__lowerCamelCase				,			lstrip=__lowerCamelCase				,			rstrip=__lowerCamelCase      ) if isinstance(__lowerCamelCase				,			__lowerCamelCase      ) else mask_token

			__lowercase					=    kwargs.get('additional_special_tokens'				,			[]      )
			kwargs["additional_special_tokens"] += [
			    code for code in FAIRSEQ_LANGUAGE_CODES if code not in kwargs["additional_special_tokens"]
			]

			super().__init__(
			    __lowerCamelCase				,			src_lang=__lowerCamelCase				,			tgt_lang=__lowerCamelCase				,			tokenizer_file=__lowerCamelCase				,			eos_token=__lowerCamelCase				,			sep_token=__lowerCamelCase				,			cls_token=__lowerCamelCase				,			unk_token=__lowerCamelCase				,			pad_token=__lowerCamelCase				,			mask_token=__lowerCamelCase				,			**__lowerCamelCase				,			)

			__lowercase					=    vocab_file
			__lowercase					=    False if not self.vocab_file else True

			__lowercase					=    {
			    lang_code: self.convert_tokens_to_ids(__lowerCamelCase      ) for lang_code in FAIRSEQ_LANGUAGE_CODES
			}

			__lowercase					=    src_lang if src_lang is not None else 'en_XX'
			__lowercase					=    tgt_lang
			__lowercase					=    self.lang_code_to_id[self._src_lang]
			self.set_src_lang_special_tokens(self._src_lang      )




	@property
	def       UpperCAmelCase (	self       :    int      )					->   str:

			'''simple docstring'''




			return self._src_lang




	@src_lang.setter
	def       UpperCAmelCase (	self       :    Optional[int]				,			__lowerCamelCase       :    str      )					->   None:

			'''simple docstring'''




			__lowercase					=    new_src_lang
			self.set_src_lang_special_tokens(self._src_lang      )




	def       UpperCAmelCase (	self       :    Dict				,			__lowerCamelCase       :    List[int]				,			__lowerCamelCase       :    Optional[List[int]] = None      )					->   List[int]:

			'''simple docstring'''




			if token_ids_a is None:
					return self.prefix_tokens + token_ids_a + self.suffix_tokens
			# We don't expect to process pairs, but leave the pair logic for API consistency
			return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




	def       UpperCAmelCase (	self       :    str				,			__lowerCamelCase       :    List[str]				,			__lowerCamelCase       :    str = "en_XX"				,			__lowerCamelCase       :    Optional[List[str]] = None				,			__lowerCamelCase       :    str = "ro_RO"				,			**__lowerCamelCase       :    Tuple				,			)					->   BatchEncoding:

			'''simple docstring'''




			__lowercase					=    src_lang
			__lowercase					=    tgt_lang
			return super().prepare_seqaseq_batch(__lowerCamelCase				,			__lowerCamelCase				,			**__lowerCamelCase      )




	def       UpperCAmelCase (	self       :    List[Any]      )					->   Tuple:

			'''simple docstring'''




			return self.set_src_lang_special_tokens(self.src_lang      )




	def       UpperCAmelCase (	self       :    Tuple      )					->   str:

			'''simple docstring'''




			return self.set_tgt_lang_special_tokens(self.tgt_lang      )




	def       UpperCAmelCase (	self       :    Tuple				,			__lowerCamelCase       :    str      )					->   None:

			'''simple docstring'''




			__lowercase					=    self.convert_tokens_to_ids(__lowerCamelCase      )
			__lowercase					=    [self.cur_lang_code_id]
			__lowercase					=    [self.eos_token_id]

			__lowercase					=    self.convert_ids_to_tokens(self.prefix_tokens      )
			__lowercase					=    self.convert_ids_to_tokens(self.suffix_tokens      )

			__lowercase					=    processors.TemplateProcessing(
			    single=prefix_tokens_str + ['$A'] + suffix_tokens_str				,			pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str				,			special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,			self.prefix_tokens + self.suffix_tokens      )      )				,			)




	def       UpperCAmelCase (	self       :    List[Any]				,			__lowerCamelCase       :    str      )					->   None:

			'''simple docstring'''




			__lowercase					=    self.convert_tokens_to_ids(__lowerCamelCase      )
			__lowercase					=    [self.cur_lang_code_id]
			__lowercase					=    [self.eos_token_id]

			__lowercase					=    self.convert_ids_to_tokens(self.prefix_tokens      )
			__lowercase					=    self.convert_ids_to_tokens(self.suffix_tokens      )

			__lowercase					=    processors.TemplateProcessing(
			    single=prefix_tokens_str + ['$A'] + suffix_tokens_str				,			pair=prefix_tokens_str + ['$A', '$B'] + suffix_tokens_str				,			special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str				,			self.prefix_tokens + self.suffix_tokens      )      )				,			)




	def       UpperCAmelCase (	self       :    List[str]				,			__lowerCamelCase       :    str				,			__lowerCamelCase       :    str				,			__lowerCamelCase       :    Optional[str]				,			__lowerCamelCase       :    Optional[str]				,			**__lowerCamelCase       :    List[str]      )					->   Optional[int]:

			'''simple docstring'''




			if src_lang is None or tgt_lang is None:
					raise ValueError('Translation requires a `src_lang` and a `tgt_lang` for this model'      )
			__lowercase					=    src_lang
			__lowercase					=    self(__lowerCamelCase				,			add_special_tokens=__lowerCamelCase				,			return_tensors=__lowerCamelCase				,			**__lowerCamelCase      )
			__lowercase					=    self.convert_tokens_to_ids(__lowerCamelCase      )
			__lowercase					=    tgt_lang_id
			return inputs






	def       UpperCAmelCase (	self       :    int				,			__lowerCamelCase       :    str				,			__lowerCamelCase       :    Optional[str] = None      )					->   Tuple[str]:

			'''simple docstring'''




			if not self.can_save_slow_tokenizer:
					raise ValueError(
					    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
					    'tokenizer.'      )

			if not os.path.isdir(__lowerCamelCase      ):
					logger.error(F"Vocabulary path ({save_directory}) should be a directory"      )
					return
			__lowercase					=    os.path.join(
			    __lowerCamelCase				,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )

			if os.path.abspath(self.vocab_file      ) != os.path.abspath(__lowerCamelCase      ):
					copyfile(self.vocab_file				,			__lowerCamelCase      )

			return (out_vocab_file,)











import shutil
import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SpeechaTextTokenizer, is_speech_available
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_torch, require_torchaudio
from transformers.utils import FEATURE_EXTRACTOR_NAME

from .test_feature_extraction_speech_to_text import floats_list


if is_speech_available():
		from transformers import SpeechaTextFeatureExtractor, SpeechaTextProcessor


SCREAMING_SNAKE_CASE_      :			Union[str, Any]		  =      get_tests_dir('''fixtures/test_sentencepiece.model''')

@require_torch
@require_torchaudio
@require_sentencepiece
class 		snake_case_      (       unittest.TestCase	):







	'''simple docstring'''




	def       UpperCAmelCase (	self       :    List[Any]      )					->   str:

			'''simple docstring'''




			__lowercase					=    tempfile.mkdtemp()

			__lowercase					=    ['<s>', '<pad>', '</s>', '<unk>', '▁This', '▁is', '▁a', '▁t', 'est']
			__lowercase					=    dict(zip(__lowerCamelCase				,			range(len(__lowerCamelCase      )      )      )      )
			__lowercase					=    Path(self.tmpdirname      )
			save_json(__lowerCamelCase				,			save_dir / VOCAB_FILES_NAMES['vocab_file']      )
			if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
					copyfile(__lowerCamelCase				,			save_dir / VOCAB_FILES_NAMES['spm_file']      )

			__lowercase					=    SpeechaTextTokenizer.from_pretrained(self.tmpdirname      )
			tokenizer.save_pretrained(self.tmpdirname      )

			__lowercase					=    {
			    'feature_size': 24,
			    'num_mel_bins': 24,
			    'padding_value': 0.0,
			    'sampling_rate': 16_000,
			    'return_attention_mask': False,
			    'do_normalize': True,
			}
			save_json(__lowerCamelCase				,			save_dir / FEATURE_EXTRACTOR_NAME      )




	def       UpperCAmelCase (	self       :    Dict				,			**__lowerCamelCase       :    Dict      )					->   Union[str, Any]:

			'''simple docstring'''




			return SpeechaTextTokenizer.from_pretrained(self.tmpdirname				,			**__lowerCamelCase      )




	def       UpperCAmelCase (	self       :    str				,			**__lowerCamelCase       :    List[str]      )					->   Tuple:

			'''simple docstring'''




			return SpeechaTextFeatureExtractor.from_pretrained(self.tmpdirname				,			**__lowerCamelCase      )




	def       UpperCAmelCase (	self       :    Optional[int]      )					->   str:

			'''simple docstring'''




			shutil.rmtree(self.tmpdirname      )




	def       UpperCAmelCase (	self       :    str      )					->   Any:

			'''simple docstring'''




			__lowercase					=    self.get_tokenizer()
			__lowercase					=    self.get_feature_extractor()

			__lowercase					=    SpeechaTextProcessor(tokenizer=__lowerCamelCase				,			feature_extractor=__lowerCamelCase      )

			processor.save_pretrained(self.tmpdirname      )
			__lowercase					=    SpeechaTextProcessor.from_pretrained(self.tmpdirname      )

			self.assertEqual(processor.tokenizer.get_vocab()				,			tokenizer.get_vocab()      )
			self.assertIsInstance(processor.tokenizer				,			__lowerCamelCase      )

			self.assertEqual(processor.feature_extractor.to_json_string()				,			feature_extractor.to_json_string()      )
			self.assertIsInstance(processor.feature_extractor				,			__lowerCamelCase      )




	def       UpperCAmelCase (	self       :    List[str]      )					->   Optional[Any]:

			'''simple docstring'''




			__lowercase					=    SpeechaTextProcessor(
			    tokenizer=self.get_tokenizer()				,			feature_extractor=self.get_feature_extractor()      )
			processor.save_pretrained(self.tmpdirname      )

			__lowercase					=    self.get_tokenizer(bos_token='(BOS)'				,			eos_token='(EOS)'      )
			__lowercase					=    self.get_feature_extractor(do_normalize=__lowerCamelCase				,			padding_value=1.0      )

			__lowercase					=    SpeechaTextProcessor.from_pretrained(
			    self.tmpdirname				,			bos_token='(BOS)'				,			eos_token='(EOS)'				,			do_normalize=__lowerCamelCase				,			padding_value=1.0      )

			self.assertEqual(processor.tokenizer.get_vocab()				,			tokenizer_add_kwargs.get_vocab()      )
			self.assertIsInstance(processor.tokenizer				,			__lowerCamelCase      )

			self.assertEqual(processor.feature_extractor.to_json_string()				,			feature_extractor_add_kwargs.to_json_string()      )
			self.assertIsInstance(processor.feature_extractor				,			__lowerCamelCase      )




	def       UpperCAmelCase (	self       :    Dict      )					->   str:

			'''simple docstring'''




			__lowercase					=    self.get_feature_extractor()
			__lowercase					=    self.get_tokenizer()

			__lowercase					=    SpeechaTextProcessor(tokenizer=__lowerCamelCase				,			feature_extractor=__lowerCamelCase      )

			__lowercase					=    floats_list((3, 1_000)      )

			__lowercase					=    feature_extractor(__lowerCamelCase				,			return_tensors='np'      )
			__lowercase					=    processor(__lowerCamelCase				,			return_tensors='np'      )

			for key in input_feat_extract.keys():
					self.assertAlmostEqual(input_feat_extract[key].sum()				,			input_processor[key].sum()				,			delta=1E-2      )




	def       UpperCAmelCase (	self       :    Union[str, Any]      )					->   str:

			'''simple docstring'''




			__lowercase					=    self.get_feature_extractor()
			__lowercase					=    self.get_tokenizer()

			__lowercase					=    SpeechaTextProcessor(tokenizer=__lowerCamelCase				,			feature_extractor=__lowerCamelCase      )

			__lowercase					=    'This is a test string'

			__lowercase					=    processor(text=__lowerCamelCase      )

			__lowercase					=    tokenizer(__lowerCamelCase      )

			for key in encoded_tok.keys():
					self.assertListEqual(encoded_tok[key]				,			encoded_processor[key]      )




	def       UpperCAmelCase (	self       :    str      )					->   str:

			'''simple docstring'''




			__lowercase					=    self.get_feature_extractor()
			__lowercase					=    self.get_tokenizer()

			__lowercase					=    SpeechaTextProcessor(tokenizer=__lowerCamelCase				,			feature_extractor=__lowerCamelCase      )

			__lowercase					=    [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

			__lowercase					=    processor.batch_decode(__lowerCamelCase      )
			__lowercase					=    tokenizer.batch_decode(__lowerCamelCase      )

			self.assertListEqual(__lowerCamelCase				,			__lowerCamelCase      )






	def       UpperCAmelCase (	self       :    Tuple      )					->   str:

			'''simple docstring'''




			__lowercase					=    self.get_feature_extractor()
			__lowercase					=    self.get_tokenizer()

			__lowercase					=    SpeechaTextProcessor(tokenizer=__lowerCamelCase				,			feature_extractor=__lowerCamelCase      )

			self.assertListEqual(
			    processor.model_input_names				,			feature_extractor.model_input_names				,			msg='`processor` and `feature_extractor` model input names do not match'				,			)




