






from __future__ import annotations

import unittest

from transformers import DebertaConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
   import tensorflow as tf

   from transformers import (
       TFDebertaForMaskedLM,
       TFDebertaForQuestionAnswering,
       TFDebertaForSequenceClassification,
       TFDebertaForTokenClassification,
       TFDebertaModel,
   )




class        _lowerCAmelCase:





       """simple docstring"""







       def __init__(      self						,    _lowerCamelCase						,    _lowerCamelCase=1_3						,    _lowerCamelCase=7						,    _lowerCamelCase=True						,    _lowerCamelCase=True						,    _lowerCamelCase=True						,    _lowerCamelCase=True						,    _lowerCamelCase=9_9						,    _lowerCamelCase=3_2						,    _lowerCamelCase=2						,    _lowerCamelCase=4						,    _lowerCamelCase=3_7						,    _lowerCamelCase="gelu"						,    _lowerCamelCase=0.1						,    _lowerCamelCase=0.1						,    _lowerCamelCase=5_1_2						,    _lowerCamelCase=1_6						,    _lowerCamelCase=2						,    _lowerCamelCase=0.0_2						,    _lowerCamelCase=3						,    _lowerCamelCase=4						,    _lowerCamelCase=None						,    ):
         UpperCamelCase_:      Any					   =       parent
         UpperCamelCase_:      Union[str, Any]					   =       1_3
         UpperCamelCase_:      Optional[int]					   =       7
         UpperCamelCase_:      List[Any]					   =       True
         UpperCamelCase_:      Optional[int]					   =       True
         UpperCamelCase_:      Optional[int]					   =       True
         UpperCamelCase_:      Any					   =       True
         UpperCamelCase_:      Optional[Any]					   =       9_9
         UpperCamelCase_:      Any					   =       3_2
         UpperCamelCase_:      Optional[int]					   =       2
         UpperCamelCase_:      Optional[int]					   =       4
         UpperCamelCase_:      List[Any]					   =       3_7
         UpperCamelCase_:      List[Any]					   =       'gelu'
         UpperCamelCase_:      int					   =       0.1
         UpperCamelCase_:      Tuple					   =       0.1
         UpperCamelCase_:      Tuple					   =       5_1_2
         UpperCamelCase_:      Dict					   =       1_6
         UpperCamelCase_:      Tuple					   =       False
         UpperCamelCase_:      Optional[int]					   =       -1
         UpperCamelCase_:      List[str]					   =       True
         UpperCamelCase_:      List[Any]					   =       2
         UpperCamelCase_:      List[str]					   =       0.0_2
         UpperCamelCase_:      Optional[Any]					   =       3
         UpperCamelCase_:      Any					   =       4
         UpperCamelCase_:      List[Any]					   =       None





       def     _a					(      self							):
         UpperCamelCase_:      int					   =       ids_tensor([self.batch_size, self.seq_length]						,    self.vocab_size							)

         UpperCamelCase_:      str					   =       None
         if self.use_input_mask:
           UpperCamelCase_:      int					   =       random_attention_mask([self.batch_size, self.seq_length]							)

         UpperCamelCase_:      Union[str, Any]					   =       None
         if self.use_token_type_ids:
           UpperCamelCase_:      Union[str, Any]					   =       ids_tensor([self.batch_size, self.seq_length]						,    self.type_vocab_size							)

         UpperCamelCase_:      List[str]					   =       None
         UpperCamelCase_:      Optional[Any]					   =       None
         UpperCamelCase_:      Tuple					   =       None
         if self.use_labels:
           UpperCamelCase_:      Union[str, Any]					   =       ids_tensor([self.batch_size]						,    self.type_sequence_label_size							)
           UpperCamelCase_:      Union[str, Any]					   =       ids_tensor([self.batch_size, self.seq_length]						,    self.num_labels							)

         UpperCamelCase_:      Tuple					   =       DebertaConfig(
             vocab_size=self.vocab_size						,    hidden_size=self.hidden_size						,    num_hidden_layers=self.num_hidden_layers						,    num_attention_heads=self.num_attention_heads						,    intermediate_size=self.intermediate_size						,    hidden_act=self.hidden_act						,    hidden_dropout_prob=self.hidden_dropout_prob						,    attention_probs_dropout_prob=self.attention_probs_dropout_prob						,    max_position_embeddings=self.max_position_embeddings						,    type_vocab_size=self.type_vocab_size						,    relative_attention=self.relative_attention						,    max_relative_positions=self.max_relative_positions						,    position_biased_input=self.position_biased_input						,    initializer_range=self.initializer_range						,    return_dict=_lowerCamelCase						,    )

         return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





       def     _a					(      self						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase							):
         UpperCamelCase_:      Union[str, Any]					   =       TFDebertaModel(config=_lowerCamelCase							)
         UpperCamelCase_:      Optional[int]					   =       {'input_ids': input_ids, 'attention_mask': input_mask, 'token_type_ids': token_type_ids}

         UpperCamelCase_:      Any					   =       [input_ids, input_mask]
         UpperCamelCase_:      int					   =       model(_lowerCamelCase							)

         UpperCamelCase_:      List[Any]					   =       model(_lowerCamelCase							)

         self.parent.assertEqual(result.last_hidden_state.shape						,    (self.batch_size, self.seq_length, self.hidden_size)							)





       def     _a					(      self						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase							):
         UpperCamelCase_:      Tuple					   =       TFDebertaForMaskedLM(config=_lowerCamelCase							)
         UpperCamelCase_:      Any					   =       {
             'input_ids': input_ids,
             'attention_mask': input_mask,
             'token_type_ids': token_type_ids,
         }
         UpperCamelCase_:      Optional[Any]					   =       model(_lowerCamelCase							)
         self.parent.assertEqual(result.logits.shape						,    (self.batch_size, self.seq_length, self.vocab_size)							)





       def     _a					(      self						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase							):
         UpperCamelCase_:      Optional[int]					   =       self.num_labels
         UpperCamelCase_:      List[str]					   =       TFDebertaForSequenceClassification(config=_lowerCamelCase							)
         UpperCamelCase_:      Dict					   =       {
             'input_ids': input_ids,
             'attention_mask': input_mask,
             'token_type_ids': token_type_ids,
         }

         UpperCamelCase_:      Tuple					   =       model(_lowerCamelCase							)
         self.parent.assertEqual(result.logits.shape						,    (self.batch_size, self.num_labels)							)





       def     _a					(      self						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase							):
         UpperCamelCase_:      Optional[Any]					   =       self.num_labels
         UpperCamelCase_:      List[Any]					   =       TFDebertaForTokenClassification(config=_lowerCamelCase							)
         UpperCamelCase_:      Any					   =       {
             'input_ids': input_ids,
             'attention_mask': input_mask,
             'token_type_ids': token_type_ids,
         }
         UpperCamelCase_:      Optional[int]					   =       model(_lowerCamelCase							)
         self.parent.assertEqual(result.logits.shape						,    (self.batch_size, self.seq_length, self.num_labels)							)





       def     _a					(      self						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase						,    _lowerCamelCase							):
         UpperCamelCase_:      str					   =       TFDebertaForQuestionAnswering(config=_lowerCamelCase							)
         UpperCamelCase_:      Optional[int]					   =       {
             'input_ids': input_ids,
             'attention_mask': input_mask,
             'token_type_ids': token_type_ids,
         }

         UpperCamelCase_:      str					   =       model(_lowerCamelCase							)
         self.parent.assertEqual(result.start_logits.shape						,    (self.batch_size, self.seq_length)							)
         self.parent.assertEqual(result.end_logits.shape						,    (self.batch_size, self.seq_length)							)





       def     _a					(      self							):
         UpperCamelCase_:      str					   =       self.prepare_config_and_inputs()
         (
             (
             UpperCamelCase_
         )   ,(
             UpperCamelCase_
         )   ,(
             UpperCamelCase_
         )   ,(
             UpperCamelCase_
         )   ,(
             UpperCamelCase_
         )   ,(
             UpperCamelCase_
         )   ,(
             UpperCamelCase_
         )   ,
         ):      Optional[int]					   =       config_and_inputs
         UpperCamelCase_:      Any					   =       {'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': input_mask}
         return config, inputs_dict







@require_tf
class        _lowerCAmelCase( UpperCAmelCase_							,			UpperCAmelCase_							,			unittest.TestCase      ):





       """simple docstring"""







       a       :	Union[str, Any]								=(
           (
               TFDebertaModel,
               TFDebertaForMaskedLM,
               TFDebertaForQuestionAnswering,
               TFDebertaForSequenceClassification,
               TFDebertaForTokenClassification,
           )
           if is_tf_available()
           else ()
       )
       a       :	int								=(
           {
               '''feature-extraction''': TFDebertaModel,
               '''fill-mask''': TFDebertaForMaskedLM,
               '''question-answering''': TFDebertaForQuestionAnswering,
               '''text-classification''': TFDebertaForSequenceClassification,
               '''token-classification''': TFDebertaForTokenClassification,
               '''zero-shot''': TFDebertaForSequenceClassification,
           }
           if is_tf_available()
           else {}
       )

       a       :	int								=False
       a       :	Optional[Any]								=False





       def     _a					(      self							):
         UpperCamelCase_:      Union[str, Any]					   =       TFDebertaModelTester(self							)
         UpperCamelCase_:      Tuple					   =       ConfigTester(self						,    config_class=_lowerCamelCase						,    hidden_size=3_7							)





       def     _a					(      self							):
         self.config_tester.run_common_tests()





       def     _a					(      self							):
         UpperCamelCase_:      Optional[int]					   =       self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_model(*_lowerCamelCase							)





       def     _a					(      self							):
         UpperCamelCase_:      Optional[Any]					   =       self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_masked_lm(*_lowerCamelCase							)





       def     _a					(      self							):
         UpperCamelCase_:      List[str]					   =       self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_question_answering(*_lowerCamelCase							)





       def     _a					(      self							):
         UpperCamelCase_:      List[str]					   =       self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_sequence_classification(*_lowerCamelCase							)





       def     _a					(      self							):
         UpperCamelCase_:      Dict					   =       self.model_tester.prepare_config_and_inputs()
         self.model_tester.create_and_check_for_token_classification(*_lowerCamelCase							)





       @slow
       def     _a					(      self							):
         UpperCamelCase_:      List[str]					   =       TFDebertaModel.from_pretrained('kamalkraj/deberta-base'							)
         self.assertIsNotNone(_lowerCamelCase							)







@require_tf
class        _lowerCAmelCase( unittest.TestCase      ):





       """simple docstring"""







       @unittest.skip(reason='Model not available yet'							)
       def     _a					(      self							):
         pass





       @slow
       def     _a					(      self							):
         UpperCamelCase_:      int					   =       TFDebertaModel.from_pretrained('kamalkraj/deberta-base'							)
         UpperCamelCase_:      Union[str, Any]					   =       tf.constant([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]]							)
         UpperCamelCase_:      Optional[int]					   =       tf.constant([[0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]							)
         UpperCamelCase_:      Union[str, Any]					   =       model(_lowerCamelCase						,    attention_mask=_lowerCamelCase							)[0]

         UpperCamelCase_:      Union[str, Any]					   =       tf.constant(
             [
                 [
                     [-0.5_9_8_5_5_8_9_6, -0.8_0_5_5_2_5_6_6, -0.8_4_6_2_1_3_5],
                     [1.4_4_8_4_0_2_5, -0.9_3_4_8_3_7_9_4, -0.8_0_5_9_3_0_8_5],
                     [0.3_1_2_2_7_4_1, 0.0_0_3_1_6_0_5_9, -1.4_1_3_1_3_7_7],
                 ]
             ]							)
         tf.debugging.assert_near(output[:, 1:4, 1:4]						,    _lowerCamelCase						,    atol=1e-4							)







import os
from shutil import copyfile
from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
   from .tokenization_reformer import ReformerTokenizer
else:
   A_	:	Union[str, Any]				=				None


A_	:	List[Any]				=				logging.get_logger(__name__)


A_	:	List[str]				=				'▁'

A_	:	int				=				{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

A_	:	Optional[Any]				=				{
    'vocab_file': {
        'google/reformer-crime-and-punishment': (
            'https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model'
        )
    },
    'tokenizer_file': {
        'google/reformer-crime-and-punishment': (
            'https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/tokenizer.json'
        )
    },
}

A_	:	List[Any]				=				{
    'google/reformer-crime-and-punishment': 524288,
}




class        _lowerCAmelCase( UpperCAmelCase_      ):





       """simple docstring"""







       a       :	int								=VOCAB_FILES_NAMES
       a       :	Optional[Any]								=PRETRAINED_VOCAB_FILES_MAP
       a       :	List[Any]								=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       a       :	Tuple								=['''input_ids''', '''attention_mask''']
       a       :	Optional[Any]								=ReformerTokenizer





       def __init__(      self						,    _lowerCamelCase=None						,    _lowerCamelCase=None						,    _lowerCamelCase="</s>"						,    _lowerCamelCase="<unk>"						,    _lowerCamelCase=[]						,    **_lowerCamelCase						,    ):
         super().__init__(
             _lowerCamelCase						,    tokenizer_file=_lowerCamelCase						,    eos_token=_lowerCamelCase						,    unk_token=_lowerCamelCase						,    additional_special_tokens=_lowerCamelCase						,    **_lowerCamelCase						,    )

         UpperCamelCase_:      Any					   =       vocab_file
         UpperCamelCase_:      str					   =       False if not self.vocab_file else True





       def     _a					(      self						,    _lowerCamelCase						,    _lowerCamelCase = None							):
         if not self.can_save_slow_tokenizer:
           raise ValueError(
               'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
               'tokenizer.'							)

         if not os.path.isdir(_lowerCamelCase							):
           logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''							)
           return
         UpperCamelCase_:      str					   =       os.path.join(
             _lowerCamelCase						,    (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']							)

         if os.path.abspath(self.vocab_file							) != os.path.abspath(_lowerCamelCase							):
           copyfile(self.vocab_file						,    _lowerCamelCase							)

         return (out_vocab_file,)