





"""simple docstring"""
import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class lowerCamelCase__  (    __UpperCamelCase  ,				unittest.TestCase				):
   __UpperCAmelCase        =					GPTNeoXJapaneseTokenizer
   __UpperCAmelCase        =					False
   __UpperCAmelCase        =					{"""do_clean_text""": False, """add_prefix_space""": False}



   def    _UpperCAmelCase	(       self	) ->    List[Any]:







      """simple docstring"""
      super().setUp()

      lowercase    :			Optional[Any]  =       [
          """こん""",
          """こんに""",
          """にちは""",
          """ばんは""",
          """世界,㔺界""",
          """、""",
          """。""",
          """<BR>""",
          """<SP>""",
          """<TAB>""",
          """<URL>""",
          """<EMAIL>""",
          """<TEL>""",
          """<DATE>""",
          """<PRICE>""",
          """<BLOCK>""",
          """<KIGOU>""",
          """<U2000U2BFF>""",
          """<|emoji1|>""",
          """<unk>""",
          """<|startoftext|>""",
          """<|endoftext|>""",
      ]
      lowercase    :			Optional[Any]  =       {"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
      lowercase    :			Tuple  =       {"""unk_token""": """<unk>"""}

      lowercase    :			Optional[int]  =       os.path.join(self.tmpdirname      ,   VOCAB_FILES_NAMES["""vocab_file"""]	)
      lowercase    :			Dict  =       os.path.join(self.tmpdirname      ,   VOCAB_FILES_NAMES["""emoji_file"""]	)
      with open(self.vocab_file      ,   """w"""      ,   encoding="""utf-8"""	) as vocab_writer:
         vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]	)	)
      with open(self.emoji_file      ,   """w"""	) as emoji_writer:
         emoji_writer.write(json.dumps(snake_case	)	)



   def    _UpperCAmelCase	(       self      ,   **snake_case	) ->    Optional[Any]:







      """simple docstring"""
      kwargs.update(self.special_tokens_map	)
      return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname      ,   **snake_case	)



   def    _UpperCAmelCase	(       self      ,   snake_case	) ->    Optional[int]:







      """simple docstring"""
      lowercase    :			Any  =       """こんにちは、世界。 \nこんばんは、㔺界。😀"""
      lowercase    :			str  =       """こんにちは、世界。 \nこんばんは、世界。😀"""
      return input_text, output_text



   def    _UpperCAmelCase	(       self      ,   snake_case	) ->    Dict:







      """simple docstring"""
      lowercase  ,					lowercase    :			Union[str, Any]  =       self.get_input_output_texts(snake_case	)
      lowercase    :			Union[str, Any]  =       tokenizer.encode(snake_case      ,   add_special_tokens=snake_case	)
      lowercase    :			List[Any]  =       tokenizer.decode(snake_case      ,   clean_up_tokenization_spaces=snake_case	)
      return text, ids



   def    _UpperCAmelCase	(       self	) ->    Tuple:







      """simple docstring"""
      pass  # TODO add if relevant



   def    _UpperCAmelCase	(       self	) ->    Any:







      """simple docstring"""
      pass  # TODO add if relevant



   def    _UpperCAmelCase	(       self	) ->    str:







      """simple docstring"""
      pass  # TODO add if relevant



   def    _UpperCAmelCase	(       self	) ->    Optional[Any]:







      """simple docstring"""
      lowercase    :			Optional[Any]  =       self.get_tokenizer()

      # Testing tokenization
      lowercase    :			int  =       """こんにちは、世界。　こんばんは、㔺界。"""
      lowercase    :			Optional[Any]  =       ["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
      lowercase    :			List[str]  =       tokenizer.tokenize(snake_case	)
      self.assertListEqual(snake_case      ,   snake_case	)

      # Testing conversion to ids without special tokens
      lowercase    :			str  =       [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
      lowercase    :			Dict  =       tokenizer.convert_tokens_to_ids(snake_case	)
      self.assertListEqual(snake_case      ,   snake_case	)

      # Testing conversion to ids with special tokens
      lowercase    :			Optional[int]  =       tokens + [tokenizer.unk_token]
      lowercase    :			Any  =       [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 1_9]
      lowercase    :			Dict  =       tokenizer.convert_tokens_to_ids(snake_case	)
      self.assertListEqual(snake_case      ,   snake_case	)



   @slow
   def    _UpperCAmelCase	(       self	) ->    int:







      """simple docstring"""
      lowercase    :			List[str]  =       self.tokenizer_class.from_pretrained("""abeja/gpt-neox-japanese-2.7b"""	)

      lowercase    :			List[Any]  =       tokenizer.encode("""ありがとう。"""      ,   add_special_tokens=snake_case	)
      lowercase    :			Tuple  =       tokenizer.encode("""どういたしまして。"""      ,   add_special_tokens=snake_case	)

      lowercase    :			List[Any]  =       tokenizer.build_inputs_with_special_tokens(snake_case	)
      lowercase    :			Union[str, Any]  =       tokenizer.build_inputs_with_special_tokens(snake_case      ,   snake_case	)

      assert encoded_sentence == ids_a
      assert encoded_pair == ids_a + ids_a



   def    _UpperCAmelCase	(       self	) ->    List[str]:







      """simple docstring"""
      # Intentionally convert some words to accommodate character fluctuations unique to Japanese
      pass





   def    _UpperCAmelCase	(       self	) ->    Optional[Any]:







      """simple docstring"""
      # tokenizer has no padding token
      pass









"""simple docstring"""
import torch.nn as nn

from .imports import is_fpa_available


if is_fpa_available():
   import transformer_engine.pytorch as te




def 			__snake_case						(      __A ,__A=True ,__A=True ,__A=True    )		->				Union[str, Any]:
   if not is_fpa_available():
      raise ImportError("""Using `convert_model` requires transformer_engine to be installed."""    )
   for name, module in model.named_children():
      if isinstance(__A ,nn.Linear    ) and to_transformer_engine and _convert_linear:
         # Return early if the linear layer weights are not multiples of 16
         if any(p % 16 != 0 for p in module.weight.shape    ):
            return
         lowercase    :			Optional[int]  =       module.bias is not None
         lowercase    :			Union[str, Any]  =       te.Linear(
             module.in_features ,module.out_features ,bias=__A ,params_dtype=module.weight.dtype    )
         lowercase    :			Union[str, Any]  =       module.weight.data.clone()
         if has_bias:
            lowercase    :			Dict  =       module.bias.data.clone()

         setattr(__A ,__A ,__A    )
      elif isinstance(__A ,nn.LayerNorm    ) and to_transformer_engine and _convert_ln:
         lowercase    :			List[str]  =       te.LayerNorm(module.normalized_shape[0] ,eps=module.eps ,params_dtype=module.weight.dtype    )
         lowercase    :			Optional[Any]  =       module.weight.data.clone()
         lowercase    :			Any  =       module.bias.data.clone()

         setattr(__A ,__A ,__A    )
      elif isinstance(__A ,te.Linear    ) and not to_transformer_engine and _convert_linear:
         lowercase    :			Any  =       module.bias is not None
         lowercase    :			List[Any]  =       nn.Linear(
             module.in_features ,module.out_features ,bias=__A ,params_dtype=module.weight.dtype    )
         lowercase    :			Union[str, Any]  =       module.weight.data.clone()
         if has_bias:
            lowercase    :			Optional[Any]  =       module.bias.data.clone()

         setattr(__A ,__A ,__A    )
      elif isinstance(__A ,te.LayerNorm    ) and not to_transformer_engine and _convert_ln:
         lowercase    :			Any  =       nn.LayerNorm(module.normalized_shape[0] ,eps=module.eps ,params_dtype=module.weight.dtype    )
         lowercase    :			str  =       module.weight.data.clone()
         lowercase    :			List[str]  =       module.bias.data.clone()

         setattr(__A ,__A ,__A    )
      else:
         convert_model(
             __A ,to_transformer_engine=__A ,_convert_linear=_convert_linear ,_convert_ln=_convert_ln ,)







def 			__snake_case						(      __A    )		->				Union[str, Any]:
   if not is_fpa_available():
      raise ImportError("""Using `has_transformer_engine_layers` requires transformer_engine to be installed."""    )
   for m in model.modules():
      if isinstance(__A ,(te.LayerNorm, te.Linear)    ):
         return True
   return False


