


'''simple docstring'''







import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 		_snake_case    (				snake_case_  ,  unittest.TestCase		):






						'''simple docstring'''







						__snake_case   =      GPTNeoXJapaneseTokenizer
						__snake_case   =      False
						__snake_case   =      {"do_clean_text": False, "add_prefix_space": False}







						def lowerCAmelCase__ (  self:	Any     )							->	List[str]:
										super().setUp()

										__magic_name__				:					Union[str, Any]						       =       [
										    "こん",
										    "こんに",
										    "にちは",
										    "ばんは",
										    "世界,㔺界",
										    "、",
										    "。",
										    "<BR>",
										    "<SP>",
										    "<TAB>",
										    "<URL>",
										    "<EMAIL>",
										    "<TEL>",
										    "<DATE>",
										    "<PRICE>",
										    "<BLOCK>",
										    "<KIGOU>",
										    "<U2000U2BFF>",
										    "<|emoji1|>",
										    "<unk>",
										    "<|startoftext|>",
										    "<|endoftext|>",
										]
										__magic_name__				:					List[str]						       =       {"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
										__magic_name__				:					str						       =       {"unk_token": "<unk>"}

										__magic_name__				:					Dict						       =       os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES["vocab_file"]     )
										__magic_name__				:					Dict						       =       os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES["emoji_file"]     )
										with open(self.vocab_file      ,							"w"      ,							encoding="utf-8"     ) as vocab_writer:
														vocab_writer.write("".join([x + "\n" for x in vocab_tokens]     )     )
										with open(self.emoji_file      ,							"w"     ) as emoji_writer:
														emoji_writer.write(json.dumps(__UpperCamelCase     )     )







						def lowerCAmelCase__ (  self:	Tuple      ,							**__UpperCamelCase:	str     )							->	Union[str, Any]:
										kwargs.update(self.special_tokens_map     )
										return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname      ,							**__UpperCamelCase     )







						def lowerCAmelCase__ (  self:	List[str]      ,							__UpperCamelCase:	str     )							->	List[str]:
										__magic_name__				:					str						       =       "こんにちは、世界。 \nこんばんは、㔺界。😀"
										__magic_name__				:					str						       =       "こんにちは、世界。 \nこんばんは、世界。😀"
										return input_text, output_text







						def lowerCAmelCase__ (  self:	Union[str, Any]      ,							__UpperCamelCase:	List[Any]     )							->	Dict:
										__magic_name__       ,     __magic_name__				:					Union[str, Any]						       =       self.get_input_output_texts(__UpperCamelCase     )
										__magic_name__				:					List[Any]						       =       tokenizer.encode(__UpperCamelCase      ,							add_special_tokens=__UpperCamelCase     )
										__magic_name__				:					Optional[Any]						       =       tokenizer.decode(__UpperCamelCase      ,							clean_up_tokenization_spaces=__UpperCamelCase     )
										return text, ids







						def lowerCAmelCase__ (  self:	Optional[int]     )							->	Union[str, Any]:
										pass  # TODO add if relevant







						def lowerCAmelCase__ (  self:	str     )							->	List[str]:
										pass  # TODO add if relevant







						def lowerCAmelCase__ (  self:	str     )							->	str:
										pass  # TODO add if relevant







						def lowerCAmelCase__ (  self:	Union[str, Any]     )							->	Dict:
										__magic_name__				:					List[str]						       =       self.get_tokenizer()

										# Testing tokenization
										__magic_name__				:					Any						       =       "こんにちは、世界。　こんばんは、㔺界。"
										__magic_name__				:					Optional[int]						       =       ["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
										__magic_name__				:					List[Any]						       =       tokenizer.tokenize(__UpperCamelCase     )
										self.assertListEqual(__UpperCamelCase      ,							__UpperCamelCase     )

										# Testing conversion to ids without special tokens
										__magic_name__				:					int						       =       [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
										__magic_name__				:					str						       =       tokenizer.convert_tokens_to_ids(__UpperCamelCase     )
										self.assertListEqual(__UpperCamelCase      ,							__UpperCamelCase     )

										# Testing conversion to ids with special tokens
										__magic_name__				:					str						       =       tokens + [tokenizer.unk_token]
										__magic_name__				:					Tuple						       =       [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
										__magic_name__				:					Dict						       =       tokenizer.convert_tokens_to_ids(__UpperCamelCase     )
										self.assertListEqual(__UpperCamelCase      ,							__UpperCamelCase     )







						@slow
						def lowerCAmelCase__ (  self:	List[Any]     )							->	Union[str, Any]:
										__magic_name__				:					List[str]						       =       self.tokenizer_class.from_pretrained("abeja/gpt-neox-japanese-2.7b"     )

										__magic_name__				:					List[Any]						       =       tokenizer.encode("ありがとう。"      ,							add_special_tokens=__UpperCamelCase     )
										__magic_name__				:					Optional[Any]						       =       tokenizer.encode("どういたしまして。"      ,							add_special_tokens=__UpperCamelCase     )

										__magic_name__				:					List[Any]						       =       tokenizer.build_inputs_with_special_tokens(__UpperCamelCase     )
										__magic_name__				:					str						       =       tokenizer.build_inputs_with_special_tokens(__UpperCamelCase      ,							__UpperCamelCase     )

										assert encoded_sentence == ids_a
										assert encoded_pair == ids_a + ids_a







						def lowerCAmelCase__ (  self:	Tuple     )							->	Dict:
										# Intentionally convert some words to accommodate character fluctuations unique to Japanese
										pass







						def lowerCAmelCase__ (  self:	Dict     )							->	Optional[int]:
										# tokenizer has no padding token
										pass



'''simple docstring'''







import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 		_snake_case    (				snake_case_  ,  unittest.TestCase		):






						'''simple docstring'''







						__snake_case   =      GPTNeoXJapaneseTokenizer
						__snake_case   =      False
						__snake_case   =      {"do_clean_text": False, "add_prefix_space": False}







						def lowerCAmelCase__ (  self:	Any     )							->	List[str]:
										super().setUp()

										__magic_name__				:					Union[str, Any]						       =       [
										    "こん",
										    "こんに",
										    "にちは",
										    "ばんは",
										    "世界,㔺界",
										    "、",
										    "。",
										    "<BR>",
										    "<SP>",
										    "<TAB>",
										    "<URL>",
										    "<EMAIL>",
										    "<TEL>",
										    "<DATE>",
										    "<PRICE>",
										    "<BLOCK>",
										    "<KIGOU>",
										    "<U2000U2BFF>",
										    "<|emoji1|>",
										    "<unk>",
										    "<|startoftext|>",
										    "<|endoftext|>",
										]
										__magic_name__				:					List[str]						       =       {"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
										__magic_name__				:					str						       =       {"unk_token": "<unk>"}

										__magic_name__				:					Dict						       =       os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES["vocab_file"]     )
										__magic_name__				:					Dict						       =       os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES["emoji_file"]     )
										with open(self.vocab_file      ,							"w"      ,							encoding="utf-8"     ) as vocab_writer:
														vocab_writer.write("".join([x + "\n" for x in vocab_tokens]     )     )
										with open(self.emoji_file      ,							"w"     ) as emoji_writer:
														emoji_writer.write(json.dumps(__UpperCamelCase     )     )







						def lowerCAmelCase__ (  self:	Tuple      ,							**__UpperCamelCase:	str     )							->	Union[str, Any]:
										kwargs.update(self.special_tokens_map     )
										return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname      ,							**__UpperCamelCase     )







						def lowerCAmelCase__ (  self:	List[str]      ,							__UpperCamelCase:	str     )							->	List[str]:
										__magic_name__				:					str						       =       "こんにちは、世界。 \nこんばんは、㔺界。😀"
										__magic_name__				:					str						       =       "こんにちは、世界。 \nこんばんは、世界。😀"
										return input_text, output_text







						def lowerCAmelCase__ (  self:	Union[str, Any]      ,							__UpperCamelCase:	List[Any]     )							->	Dict:
										__magic_name__       ,     __magic_name__				:					Union[str, Any]						       =       self.get_input_output_texts(__UpperCamelCase     )
										__magic_name__				:					List[Any]						       =       tokenizer.encode(__UpperCamelCase      ,							add_special_tokens=__UpperCamelCase     )
										__magic_name__				:					Optional[Any]						       =       tokenizer.decode(__UpperCamelCase      ,							clean_up_tokenization_spaces=__UpperCamelCase     )
										return text, ids







						def lowerCAmelCase__ (  self:	Optional[int]     )							->	Union[str, Any]:
										pass  # TODO add if relevant







						def lowerCAmelCase__ (  self:	str     )							->	List[str]:
										pass  # TODO add if relevant







						def lowerCAmelCase__ (  self:	str     )							->	str:
										pass  # TODO add if relevant







						def lowerCAmelCase__ (  self:	Union[str, Any]     )							->	Dict:
										__magic_name__				:					List[str]						       =       self.get_tokenizer()

										# Testing tokenization
										__magic_name__				:					Any						       =       "こんにちは、世界。　こんばんは、㔺界。"
										__magic_name__				:					Optional[int]						       =       ["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
										__magic_name__				:					List[Any]						       =       tokenizer.tokenize(__UpperCamelCase     )
										self.assertListEqual(__UpperCamelCase      ,							__UpperCamelCase     )

										# Testing conversion to ids without special tokens
										__magic_name__				:					int						       =       [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
										__magic_name__				:					str						       =       tokenizer.convert_tokens_to_ids(__UpperCamelCase     )
										self.assertListEqual(__UpperCamelCase      ,							__UpperCamelCase     )

										# Testing conversion to ids with special tokens
										__magic_name__				:					str						       =       tokens + [tokenizer.unk_token]
										__magic_name__				:					Tuple						       =       [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
										__magic_name__				:					Dict						       =       tokenizer.convert_tokens_to_ids(__UpperCamelCase     )
										self.assertListEqual(__UpperCamelCase      ,							__UpperCamelCase     )







						@slow
						def lowerCAmelCase__ (  self:	List[Any]     )							->	Union[str, Any]:
										__magic_name__				:					List[str]						       =       self.tokenizer_class.from_pretrained("abeja/gpt-neox-japanese-2.7b"     )

										__magic_name__				:					List[Any]						       =       tokenizer.encode("ありがとう。"      ,							add_special_tokens=__UpperCamelCase     )
										__magic_name__				:					Optional[Any]						       =       tokenizer.encode("どういたしまして。"      ,							add_special_tokens=__UpperCamelCase     )

										__magic_name__				:					List[Any]						       =       tokenizer.build_inputs_with_special_tokens(__UpperCamelCase     )
										__magic_name__				:					str						       =       tokenizer.build_inputs_with_special_tokens(__UpperCamelCase      ,							__UpperCamelCase     )

										assert encoded_sentence == ids_a
										assert encoded_pair == ids_a + ids_a







						def lowerCAmelCase__ (  self:	Tuple     )							->	Dict:
										# Intentionally convert some words to accommodate character fluctuations unique to Japanese
										pass







						def lowerCAmelCase__ (  self:	Dict     )							->	Optional[int]:
										# tokenizer has no padding token
										pass