

import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 							A_				(	snake_case__							,			unittest.TestCase      ):
     _lowercase    :  Optional[Any]          =			GPTNeoXJapaneseTokenizer
     _lowercase    :  Any          =			False
     _lowercase    :  List[Any]          =			{'do_clean_text': False, 'add_prefix_space': False}


     def 						UpperCAmelCase						(							self    :	str      )							->			str:
      super().setUp()

      __lowerCAmelCase:     Optional[int]     			=   [
          'こん',
          'こんに',
          'にちは',
          'ばんは',
          '世界,㔺界',
          '、',
          '。',
          '<BR>',
          '<SP>',
          '<TAB>',
          '<URL>',
          '<EMAIL>',
          '<TEL>',
          '<DATE>',
          '<PRICE>',
          '<BLOCK>',
          '<KIGOU>',
          '<U2000U2BFF>',
          '<|emoji1|>',
          '<unk>',
          '<|startoftext|>',
          '<|endoftext|>',
      ]
      __lowerCAmelCase:     Union[str, Any]     			=   {'emoji': {'\ud83d\ude00': '<|emoji1|>'}, 'emoji_inv': {'<|emoji1|>': '\ud83d\ude00'}}  # 😀
      __lowerCAmelCase:     Optional[Any]     			=   {'unk_token': '<unk>'}

      __lowerCAmelCase:     Optional[Any]     			=   os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES['vocab_file']      )
      __lowerCAmelCase:     str     			=   os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES['emoji_file']      )
      with open(self.vocab_file    ,						'w'    ,						encoding='utf-8'      ) as vocab_writer:
       vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]      )      )
      with open(self.emoji_file    ,						'w'      ) as emoji_writer:
       emoji_writer.write(json.dumps(UpperCAmelCase      )      )


     def 						UpperCAmelCase						(							self    :	Tuple    ,						**UpperCAmelCase    :	List[str]      )							->			List[Any]:
      kwargs.update(self.special_tokens_map      )
      return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname    ,						**UpperCAmelCase      )


     def 						UpperCAmelCase						(							self    :	int    ,						UpperCAmelCase    :	List[Any]      )							->			Tuple:
      __lowerCAmelCase:     Tuple     			=   'こんにちは、世界。 \nこんばんは、㔺界。😀'
      __lowerCAmelCase:     Dict     			=   'こんにちは、世界。 \nこんばんは、世界。😀'
      return input_text, output_text


     def 						UpperCAmelCase						(							self    :	Union[str, Any]    ,						UpperCAmelCase    :	Any      )							->			Any:
      __lowerCAmelCase			,							__lowerCAmelCase:     Optional[int]     			=   self.get_input_output_texts(UpperCAmelCase      )
      __lowerCAmelCase:     Union[str, Any]     			=   tokenizer.encode(UpperCAmelCase    ,						add_special_tokens=UpperCAmelCase      )
      __lowerCAmelCase:     Tuple     			=   tokenizer.decode(UpperCAmelCase    ,						clean_up_tokenization_spaces=UpperCAmelCase      )
      return text, ids


     def 						UpperCAmelCase						(							self    :	int      )							->			str:
      pass  # TODO add if relevant


     def 						UpperCAmelCase						(							self    :	List[str]      )							->			str:
      pass  # TODO add if relevant


     def 						UpperCAmelCase						(							self    :	Optional[Any]      )							->			Optional[int]:
      pass  # TODO add if relevant


     def 						UpperCAmelCase						(							self    :	Optional[int]      )							->			Any:
      __lowerCAmelCase:     Union[str, Any]     			=   self.get_tokenizer()

      # Testing tokenization
      __lowerCAmelCase:     Union[str, Any]     			=   'こんにちは、世界。　こんばんは、㔺界。'
      __lowerCAmelCase:     Optional[Any]     			=   ['こん', 'にちは', '、', '世界', '。', '<SP>', 'こん', 'ばんは', '、', '㔺界', '。']
      __lowerCAmelCase:     Dict     			=   tokenizer.tokenize(UpperCAmelCase      )
      self.assertListEqual(UpperCAmelCase    ,						UpperCAmelCase      )

      # Testing conversion to ids without special tokens
      __lowerCAmelCase:     Optional[Any]     			=   [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
      __lowerCAmelCase:     Optional[int]     			=   tokenizer.convert_tokens_to_ids(UpperCAmelCase      )
      self.assertListEqual(UpperCAmelCase    ,						UpperCAmelCase      )

      # Testing conversion to ids with special tokens
      __lowerCAmelCase:     Tuple     			=   tokens + [tokenizer.unk_token]
      __lowerCAmelCase:     Dict     			=   [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
      __lowerCAmelCase:     Optional[int]     			=   tokenizer.convert_tokens_to_ids(UpperCAmelCase      )
      self.assertListEqual(UpperCAmelCase    ,						UpperCAmelCase      )


     @slow
     def 						UpperCAmelCase						(							self    :	int      )							->			List[str]:
      __lowerCAmelCase:     Union[str, Any]     			=   self.tokenizer_class.from_pretrained('abeja/gpt-neox-japanese-2.7b'      )

      __lowerCAmelCase:     List[str]     			=   tokenizer.encode('ありがとう。'    ,						add_special_tokens=UpperCAmelCase      )
      __lowerCAmelCase:     Optional[int]     			=   tokenizer.encode('どういたしまして。'    ,						add_special_tokens=UpperCAmelCase      )

      __lowerCAmelCase:     Optional[Any]     			=   tokenizer.build_inputs_with_special_tokens(UpperCAmelCase      )
      __lowerCAmelCase:     List[str]     			=   tokenizer.build_inputs_with_special_tokens(UpperCAmelCase    ,						UpperCAmelCase      )

      assert encoded_sentence == ids_a
      assert encoded_pair == ids_a + ids_a


     def 						UpperCAmelCase						(							self    :	Tuple      )							->			Any:
      # Intentionally convert some words to accommodate character fluctuations unique to Japanese
      pass


     def 						UpperCAmelCase						(							self    :	List[str]      )							->			Optional[int]:
      # tokenizer has no padding token
      pass








from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_keras_nlp_available,
    is_tensorflow_text_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_a	 =     {
    '''configuration_gpt2''': ['''GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''GPT2Config''', '''GPT2OnnxConfig'''],
    '''tokenization_gpt2''': ['''GPT2Tokenizer'''],
}

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _a	 =     ['''GPT2TokenizerFast''']

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _a	 =     [
      '''GPT2_PRETRAINED_MODEL_ARCHIVE_LIST''',
      '''GPT2DoubleHeadsModel''',
      '''GPT2ForQuestionAnswering''',
      '''GPT2ForSequenceClassification''',
      '''GPT2ForTokenClassification''',
      '''GPT2LMHeadModel''',
      '''GPT2Model''',
      '''GPT2PreTrainedModel''',
      '''load_tf_weights_in_gpt2''',
  ]

try:
  if not is_tf_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _a	 =     [
      '''TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST''',
      '''TFGPT2DoubleHeadsModel''',
      '''TFGPT2ForSequenceClassification''',
      '''TFGPT2LMHeadModel''',
      '''TFGPT2MainLayer''',
      '''TFGPT2Model''',
      '''TFGPT2PreTrainedModel''',
  ]

try:
  if not is_keras_nlp_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _a	 =     ['''TFGPT2Tokenizer''']

try:
  if not is_flax_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  _a	 =     ['''FlaxGPT2LMHeadModel''', '''FlaxGPT2Model''', '''FlaxGPT2PreTrainedModel''']

if TYPE_CHECKING:
  from .configuration_gpta import GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTaConfig, GPTaOnnxConfig
  from .tokenization_gpta import GPTaTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_gpta_fast import GPTaTokenizerFast

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_gpta import (
        GPT2_PRETRAINED_MODEL_ARCHIVE_LIST,
        GPTaDoubleHeadsModel,
        GPTaForQuestionAnswering,
        GPTaForSequenceClassification,
        GPTaForTokenClassification,
        GPTaLMHeadModel,
        GPTaModel,
        GPTaPreTrainedModel,
        load_tf_weights_in_gpta,
    )

  try:
    if not is_tf_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_tf_gpta import (
        TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFGPTaDoubleHeadsModel,
        TFGPTaForSequenceClassification,
        TFGPTaLMHeadModel,
        TFGPTaMainLayer,
        TFGPTaModel,
        TFGPTaPreTrainedModel,
    )

  try:
    if not is_keras_nlp_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .tokenization_gpta_tf import TFGPTaTokenizer

  try:
    if not is_flax_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_flax_gpta import FlaxGPTaLMHeadModel, FlaxGPTaModel, FlaxGPTaPreTrainedModel

else:
  import sys

  _a	 =     _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





