
from transformers.models.cpm.tokenization_cpm import CpmTokenizer
from transformers.testing_utils import custom_tokenizers

from ..xlnet.test_modeling_xlnet import XLNetModelTest






@custom_tokenizers
class 						UpperCAmelCase	(		A_				):



   def        _SCREAMING_SNAKE_CASE     (self	:      List[str]					,		snake_case__	:      Dict					,		snake_case__	:      Tuple					,		snake_case__	:      Union[str, Any]					,		snake_case__	:      List[str]					,		snake_case__	:      List[Any]				)	->			Any:


          '''simple docstring'''


          return True



   def        _SCREAMING_SNAKE_CASE     (self	:      str				)	->			Dict:


          '''simple docstring'''


          snake_case :     Union[str, Any]           =   CpmTokenizer.from_pretrained("TsinghuaAI/CPM-Generate"				)
          snake_case :     str           =   "Hugging Face大法好，谁用谁知道。"
          snake_case :     Optional[Any]           =   "Hugging Face大法好,谁用谁知道。<unk>"
          snake_case :     int           =   "▁Hu gg ing ▁ ▂ ▁F ace ▁大法 ▁好 ▁ , ▁谁 ▁用 ▁谁 ▁知 道 ▁ 。".split()

          snake_case :     Union[str, Any]           =   tokenizer.tokenize(snake_case__				)
          self.assertListEqual(snake_case__					,		snake_case__				)

          snake_case :     Tuple           =   tokens + [tokenizer.unk_token]

          snake_case :     int           =   [1_37_89, 1_32_83, 14_21, 8, 10, 11_64, 1_36_08, 1_65_28, 63, 8, 9, 4_40, 1_08, 4_40, 1_21, 90, 8, 12, 0]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__				)					,		snake_case__				)

          snake_case :     Dict           =   tokenizer.decode(snake_case__				)
          self.assertEqual(snake_case__					,		snake_case__				)







from transformers.models.cpm.tokenization_cpm import CpmTokenizer
from transformers.testing_utils import custom_tokenizers

from ..xlnet.test_modeling_xlnet import XLNetModelTest






@custom_tokenizers
class 						UpperCAmelCase	(		A_				):



   def        _SCREAMING_SNAKE_CASE     (self	:      List[str]					,		snake_case__	:      Dict					,		snake_case__	:      Tuple					,		snake_case__	:      Union[str, Any]					,		snake_case__	:      List[str]					,		snake_case__	:      List[Any]				)	->			Any:


          '''simple docstring'''


          return True



   def        _SCREAMING_SNAKE_CASE     (self	:      str				)	->			Dict:


          '''simple docstring'''


          snake_case :     Union[str, Any]           =   CpmTokenizer.from_pretrained("TsinghuaAI/CPM-Generate"				)
          snake_case :     str           =   "Hugging Face大法好，谁用谁知道。"
          snake_case :     Optional[Any]           =   "Hugging Face大法好,谁用谁知道。<unk>"
          snake_case :     int           =   "▁Hu gg ing ▁ ▂ ▁F ace ▁大法 ▁好 ▁ , ▁谁 ▁用 ▁谁 ▁知 道 ▁ 。".split()

          snake_case :     Union[str, Any]           =   tokenizer.tokenize(snake_case__				)
          self.assertListEqual(snake_case__					,		snake_case__				)

          snake_case :     Tuple           =   tokens + [tokenizer.unk_token]

          snake_case :     int           =   [1_37_89, 1_32_83, 14_21, 8, 10, 11_64, 1_36_08, 1_65_28, 63, 8, 9, 4_40, 1_08, 4_40, 1_21, 90, 8, 12, 0]
          self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__				)					,		snake_case__				)

          snake_case :     Dict           =   tokenizer.decode(snake_case__				)
          self.assertEqual(snake_case__					,		snake_case__				)





