




'''simple docstring'''





import unittest

from transformers import ErnieMTokenizer
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_SCREAMING_SNAKE_CASE							:     Union[str, Any]   	=				get_tests_dir("fixtures/spiece.model")




@require_sentencepiece
@require_tokenizers
class 				_snake_case	(			lowerCamelCase__		,		unittest.TestCase							):
			lowerCAmelCase_  :   Optional[Any]		 =					ErnieMTokenizer
			lowerCAmelCase_  :   Dict		 =					False
			lowerCAmelCase_  :   int		 =					True
			lowerCAmelCase_  :   List[Any]		 =					False
			lowerCAmelCase_  :   Optional[int]		 =					False




			def      lowerCAmelCase__       (      self							)       ->       Dict:

										'''simple docstring'''




										super().setUp()

										# We have a SentencePiece fixture for testing
										snake_case_			      =   ErnieMTokenizer(lowercase_       ,							unk_token="<unk>"       ,							pad_token="<pad>"							)
										tokenizer.save_pretrained(self.tmpdirname							)




			def      lowerCAmelCase__       (      self       ,							a__							)       ->       Dict:

										'''simple docstring'''




										snake_case_			      =   "this is a test"
										snake_case_			      =   "this is a test"
										return input_text, output_text




			def      lowerCAmelCase__       (      self							)       ->       Dict:

										'''simple docstring'''




										snake_case_			      =   "<pad>"
										snake_case_			      =   0

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase_							)       ,							lowercase_							)
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase_							)       ,							lowercase_							)




			def      lowerCAmelCase__       (      self							)       ->       List[str]:

										'''simple docstring'''




										snake_case_			      =   list(self.get_tokenizer().get_vocab().keys()							)

										self.assertEqual(vocab_keys[0]       ,							"<pad>"							)
										self.assertEqual(vocab_keys[1]       ,							"<unk>"							)
										self.assertEqual(vocab_keys[-1]       ,							"▁eloquent"							)
										self.assertEqual(len(lowercase_							)       ,							30_000							)




			def      lowerCAmelCase__       (      self							)       ->       List[str]:

										'''simple docstring'''




										self.assertEqual(self.get_tokenizer().vocab_size       ,							30_000							)




			def      lowerCAmelCase__       (      self							)       ->       Dict:

										'''simple docstring'''




										if not self.test_rust_tokenizer:
																	return

										snake_case_			      =   self.get_tokenizer()
										snake_case_			      =   self.get_rust_tokenizer()

										snake_case_			      =   "I was born in 92000, and this is falsé."

										snake_case_			      =   tokenizer.tokenize(lowercase_							)
										snake_case_			      =   rust_tokenizer.tokenize(lowercase_							)
										self.assertListEqual(lowercase_       ,							lowercase_							)

										snake_case_			      =   tokenizer.encode(lowercase_       ,							add_special_tokens=lowercase_							)
										snake_case_			      =   rust_tokenizer.encode(lowercase_       ,							add_special_tokens=lowercase_							)
										self.assertListEqual(lowercase_       ,							lowercase_							)

										snake_case_			      =   self.get_rust_tokenizer()
										snake_case_			      =   tokenizer.encode(lowercase_							)
										snake_case_			      =   rust_tokenizer.encode(lowercase_							)
										self.assertListEqual(lowercase_       ,							lowercase_							)




			def      lowerCAmelCase__       (      self							)       ->       Any:

										'''simple docstring'''




										snake_case_			      =   ErnieMTokenizer(lowercase_       ,							do_lower_case=lowercase_       ,							unk_token="<unk>"       ,							pad_token="<pad>"							)

										snake_case_			      =   tokenizer.tokenize("This is a test"							)
										self.assertListEqual(lowercase_       ,							["▁this", "▁is", "▁a", "▁test"]							)

										self.assertListEqual(tokenizer.convert_tokens_to_ids(lowercase_							)       ,							[48, 25, 21, 1_289]							)

										snake_case_			      =   tokenizer.tokenize("I was born in 92000, and this is falsé."							)
										# ErnieMTokenizer(paddlenlp implementation) outputs '9' instead of '_9' so to mimic that '_9' is changed to '9'
										self.assertListEqual(
										    lowercase_       ,							["▁i", "▁was", "▁born", "▁in", "9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "é", "."]							)
										snake_case_			      =   tokenizer.convert_tokens_to_ids(lowercase_							)
										self.assertListEqual(lowercase_       ,							[31, 23, 386, 19, 518, 3_050, 15, 17, 48, 25, 8_256, 18, 1, 9]							)

										snake_case_			      =   tokenizer.convert_ids_to_tokens(lowercase_							)
										self.assertListEqual(
										    lowercase_       ,							["▁i", "▁was", "▁born", "▁in", "9", "2000", ",", "▁and", "▁this", "▁is", "▁fal", "s", "<unk>", "."]       ,							)




			def      lowerCAmelCase__       (      self							)       ->       Tuple:

										'''simple docstring'''




										snake_case_			      =   ErnieMTokenizer(lowercase_       ,							unk_token="<unk>"       ,							pad_token="<pad>"							)

										snake_case_			      =   tokenizer.encode("sequence builders"							)
										snake_case_			      =   tokenizer.encode("multi-sequence build"							)

										snake_case_			      =   tokenizer.build_inputs_with_special_tokens(lowercase_							)
										snake_case_			      =   tokenizer.build_inputs_with_special_tokens(lowercase_       ,							lowercase_							)

										assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
										assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + [
										    tokenizer.sep_token_id
										] + text_a + [tokenizer.sep_token_id]




			@slow
			def      lowerCAmelCase__       (      self							)       ->       str:

										'''simple docstring'''




										snake_case_			      =   {"input_ids": [[0, 11_062, 82_772, 7, 15, 82_772, 538, 51_529, 237, 17_198, 1_290, 206, 9, 215_175, 1_314, 136, 17_198, 1_290, 206, 9, 56_359, 42, 122_009, 9, 16_466, 16, 87_344, 4_537, 9, 4_717, 78_381, 6, 159_958, 7, 15, 24_480, 618, 4, 527, 22_693, 9, 304, 4, 2_777, 24_480, 9_874, 4, 43_523, 594, 4, 803, 18_392, 33_189, 18, 4, 43_523, 24_447, 5, 5, 5, 16, 100, 24_955, 83_658, 9_626, 144_057, 15, 839, 22_335, 16, 136, 24_955, 83_658, 83_479, 15, 39_102, 724, 16, 678, 645, 6_460, 1_328, 4_589, 42, 122_009, 115_774, 23, 3_559, 1_328, 46_876, 7, 136, 53_894, 1_940, 42_227, 41_159, 17_721, 823, 425, 4, 27_512, 98_722, 206, 136, 5_531, 4_970, 919, 17_336, 5, 2], [0, 20_080, 618, 83, 82_775, 47, 479, 9, 1_517, 73, 53_894, 333, 80_581, 110_117, 18_811, 5_256, 1_295, 51, 152_526, 297, 7_986, 390, 124_416, 538, 35_431, 214, 98, 15_044, 25_737, 136, 7_108, 43_701, 23, 756, 135_355, 7, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 581, 63_773, 119_455, 6, 147_797, 88_203, 7, 645, 70, 21, 3_285, 10_269, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}
										# fmt: on

										self.tokenizer_integration_test_util(
										    expected_encoding=lowercase_       ,							model_name="susnato/ernie-m-base_pytorch"       ,							sequences=[
										        "Transformers (formerly known as pytorch-transformers and pytorch-pretrained-bert) provides "
										        "general-purpose architectures (BERT, GPT-2, RoBERTa, XLM, DistilBert, XLNet...) for Natural "
										        "Language Understanding (NLU) and Natural Language Generation (NLG) with over32+ pretrained "
										        "models in100+ languages and deep interoperability between Jax, PyTorch and TensorFlow.",
										        "BERT is designed to pre-train deep bidirectional representations from unlabeled text by jointly "
										        "conditioning on both left and right context in all layers.",
										        "The quick brown fox jumps over the lazy dog.",
										    ]       ,							)










'''simple docstring'''





import random
from typing import List, Optional, Union

import numpy as np
import torch

from ..state import AcceleratorState
from .constants import CUDA_DISTRIBUTED_TYPES
from .dataclasses import DistributedType, RNGType
from .imports import is_npu_available, is_tpu_available, is_xpu_available


if is_tpu_available(check_device=False):
		import torch_xla.core.xla_model as xm


def   UpperCamelCase_(			snake_case							: int							,       snake_case							: bool = False  ):

							'''simple docstring'''






							if device_specific:
														seed += AcceleratorState().process_index
							random.seed(snake_case  )
							np.random.seed(snake_case  )
							torch.manual_seed(snake_case  )
							if is_xpu_available():
														torch.xpu.manual_seed_all(snake_case  )
							elif is_npu_available():
														torch.npu.manual_seed_all(snake_case  )
							else:
														torch.cuda.manual_seed_all(snake_case  )
							# ^^ safe to call this function even if cuda is not available
							if is_tpu_available():
														xm.set_rng_state(snake_case  )


def   UpperCamelCase_(			snake_case							: Optional[RNGType] = None							,       snake_case							: Optional[torch.Generator] = None  ):

							'''simple docstring'''






							if rng_type == RNGType.TORCH:
														snake_case_			      =   torch.get_rng_state()
							elif rng_type == RNGType.CUDA:
														snake_case_			      =   torch.cuda.get_rng_state()
							elif rng_type == RNGType.XLA:
														assert is_tpu_available(), "Can't synchronize XLA seeds on an environment without TPUs."
														snake_case_			      =   torch.tensor(xm.get_rng_state()  )
							elif rng_type == RNGType.NPU:
														assert is_npu_available(), "Can't synchronize NPU seeds on an environment without NPUs."
														snake_case_			      =   torch.npu.get_rng_state()
							elif rng_type == RNGType.XPU:
														assert is_xpu_available(), "Can't synchronize XPU seeds on an environment without XPUs."
														snake_case_			      =   torch.xpu.get_rng_state()
							elif rng_type == RNGType.GENERATOR:
														assert generator is not None, "Need a generator to synchronize its seed."
														snake_case_			      =   generator.get_state()

							# Broadcast the rng state from device 0 to other devices
							snake_case_			      =   AcceleratorState()
							if state.distributed_type == DistributedType.TPU:
														snake_case_			      =   rng_state.to(xm.xla_device()  )
														xm.collective_broadcast([rng_state]  )
														xm.mark_step()
														snake_case_			      =   rng_state.cpu()
							elif (
							    state.distributed_type in CUDA_DISTRIBUTED_TYPES
							    or state.distributed_type == DistributedType.MULTI_NPU
							    or state.distributed_type == DistributedType.MULTI_XPU
							):
														snake_case_			      =   rng_state.to(state.device  )
														torch.distributed.broadcast(snake_case							,       0  )
														snake_case_			      =   rng_state.cpu()
							elif state.distributed_type == DistributedType.MULTI_CPU:
														torch.distributed.broadcast(snake_case							,       0  )

							# Set the broadcast rng state
							if rng_type == RNGType.TORCH:
														torch.set_rng_state(snake_case  )
							elif rng_type == RNGType.CUDA:
														torch.cuda.set_rng_state(snake_case  )
							elif rng_type == RNGType.NPU:
														torch.npu.set_rng_state(snake_case  )
							elif rng_type == RNGType.XPU:
														torch.xpu.set_rng_state(snake_case  )
							elif rng_type == RNGType.XLA:
														xm.set_rng_state(rng_state.item()  )
							elif rng_type == RNGType.GENERATOR:
														generator.set_state(snake_case  )




def   UpperCamelCase_(			snake_case							: List[Union[str, RNGType]]							,       snake_case							: Optional[torch.Generator] = None  ):

							'''simple docstring'''






							for rng_type in rng_types:
														synchronize_rng_state(RNGType(snake_case  )							,       generator=snake_case  )




