




'''simple docstring'''





import json
import os
import unittest

from transformers import HerbertTokenizer, HerbertTokenizerFast
from transformers.models.herbert.tokenization_herbert import VOCAB_FILES_NAMES
from transformers.testing_utils import get_tests_dir, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin





@require_tokenizers
class  lowerCamelCase_							(   _SCREAMING_SNAKE_CASE ,     unittest.TestCase      ):


 '''simple docstring'''







 lowercase_       					=					HerbertTokenizer
 lowercase_       					=					HerbertTokenizerFast
 lowercase_       					=					True



 def    lowerCAmelCase_						(							self						:	Optional[Any]		):
       super().setUp()

       # Use a simpler test file without japanese/chinese characters
       with open(F"{get_tests_dir()}/fixtures/sample_text_no_unicode.txt"       ,       encoding='utf-8'		) as f_data:
             SCREAMING_SNAKE_CASE_					=    f_data.read().replace('\n\n'       ,       '\n'		).strip()

       SCREAMING_SNAKE_CASE_					=    [
           '<s>',
           '</s>',
           'l',
           'o',
           'w',
           'e',
           'r',
           's',
           't',
           'i',
           'd',
           'n',
           'w</w>',
           'r</w>',
           't</w>',
           'lo',
           'low',
           'er</w>',
           'low</w>',
           'lowest</w>',
           'newer</w>',
           'wider</w>',
           ',</w>',
           '<unk>',
       ]
       SCREAMING_SNAKE_CASE_					=    dict(zip(_lowerCAmelCase       ,       range(len(_lowerCAmelCase		)		)		)		)
       SCREAMING_SNAKE_CASE_					=    ['l o 123', 'lo w 1456', 'e r</w> 1789', '']

       SCREAMING_SNAKE_CASE_					=    os.path.join(self.tmpdirname       ,       VOCAB_FILES_NAMES['vocab_file']		)
       SCREAMING_SNAKE_CASE_					=    os.path.join(self.tmpdirname       ,       VOCAB_FILES_NAMES['merges_file']		)
       with open(self.vocab_file       ,       'w'		) as fp:
             fp.write(json.dumps(_lowerCAmelCase		)		)
       with open(self.merges_file       ,       'w'		) as fp:
             fp.write('\n'.join(_lowerCAmelCase		)		)



 def    lowerCAmelCase_						(							self						:	Union[str, Any]       ,       _lowerCAmelCase						:	int		):
       SCREAMING_SNAKE_CASE_					=    'lower newer'
       SCREAMING_SNAKE_CASE_					=    'lower newer'
       return input_text, output_text



 def    lowerCAmelCase_						(							self						:	Tuple		):
       SCREAMING_SNAKE_CASE_					=    self.tokenizer_class(vocab_file=self.vocab_file       ,       merges_file=self.merges_file		)

       SCREAMING_SNAKE_CASE_					=    'lower'
       SCREAMING_SNAKE_CASE_					=    ['low', 'er</w>']
       SCREAMING_SNAKE_CASE_					=    tokenizer.tokenize(_lowerCAmelCase		)
       self.assertListEqual(_lowerCAmelCase       ,       _lowerCAmelCase		)

       SCREAMING_SNAKE_CASE_					=    tokens + ['<unk>']
       SCREAMING_SNAKE_CASE_					=    [16, 17, 23]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase		)       ,       _lowerCAmelCase		)



 def    lowerCAmelCase_						(							self						:	Optional[int]		):
       if not self.test_rust_tokenizer:
             return

       SCREAMING_SNAKE_CASE_					=    self.get_tokenizer()
       SCREAMING_SNAKE_CASE_					=    self.get_rust_tokenizer()

       SCREAMING_SNAKE_CASE_					=    'lower,newer'

       SCREAMING_SNAKE_CASE_					=    tokenizer.tokenize(_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_					=    rust_tokenizer.tokenize(_lowerCAmelCase		)
       self.assertListEqual(_lowerCAmelCase       ,       _lowerCAmelCase		)

       SCREAMING_SNAKE_CASE_					=    tokenizer.encode(_lowerCAmelCase       ,       add_special_tokens=_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_					=    rust_tokenizer.encode(_lowerCAmelCase       ,       add_special_tokens=_lowerCAmelCase		)
       self.assertListEqual(_lowerCAmelCase       ,       _lowerCAmelCase		)

       SCREAMING_SNAKE_CASE_					=    self.get_rust_tokenizer()
       SCREAMING_SNAKE_CASE_					=    tokenizer.encode(_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_					=    rust_tokenizer.encode(_lowerCAmelCase		)
       self.assertListEqual(_lowerCAmelCase       ,       _lowerCAmelCase		)




 @slow
 def    lowerCAmelCase_						(							self						:	int		):
       SCREAMING_SNAKE_CASE_					=    self.tokenizer_class.from_pretrained('allegro/herbert-base-cased'		)

       SCREAMING_SNAKE_CASE_					=    tokenizer.encode('konstruowanie sekwencji'       ,       add_special_tokens=_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_					=    tokenizer.encode('konstruowanie wielu sekwencji'       ,       add_special_tokens=_lowerCAmelCase		)

       SCREAMING_SNAKE_CASE_					=    tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_					=    tokenizer.build_inputs_with_special_tokens(_lowerCAmelCase       ,       _lowerCAmelCase		)

       assert encoded_sentence == [0] + text + [2]
       assert encoded_pair == [0] + text + [2] + text_a + [2]





def       UpperCAmelCase_							(   __UpperCAmelCase		:			float    ,    __UpperCAmelCase		:			int = 2							)      ->				float:
      return round((float(__UpperCAmelCase							) * 9 / 5) + 32    ,    __UpperCAmelCase							)

def       UpperCAmelCase_							(   __UpperCAmelCase		:			float    ,    __UpperCAmelCase		:			int = 2							)      ->				float:
      return round(float(__UpperCAmelCase							) + 2_7_3.1_5    ,    __UpperCAmelCase							)

def       UpperCAmelCase_							(   __UpperCAmelCase		:			float    ,    __UpperCAmelCase		:			int = 2							)      ->				float:
      return round((float(__UpperCAmelCase							) * 9 / 5) + 4_9_1.6_7    ,    __UpperCAmelCase							)

def       UpperCAmelCase_							(   __UpperCAmelCase		:			float    ,    __UpperCAmelCase		:			int = 2							)      ->				float:
      return round((float(__UpperCAmelCase							) - 32) * 5 / 9    ,    __UpperCAmelCase							)

def       UpperCAmelCase_							(   __UpperCAmelCase		:			float    ,    __UpperCAmelCase		:			int = 2							)      ->				float:
      return round(((float(__UpperCAmelCase							) - 32) * 5 / 9) + 2_7_3.1_5    ,    __UpperCAmelCase							)

def       UpperCAmelCase_							(   __UpperCAmelCase		:			float    ,    __UpperCAmelCase		:			int = 2							)      ->				float:
      return round(float(__UpperCAmelCase							) + 4_5_9.6_7    ,    __UpperCAmelCase							)

def       UpperCAmelCase_							(   __UpperCAmelCase		:			float    ,    __UpperCAmelCase		:			int = 2							)      ->				float:
      return round(float(__UpperCAmelCase							) - 2_7_3.1_5    ,    __UpperCAmelCase							)

def       UpperCAmelCase_							(   __UpperCAmelCase		:			float    ,    __UpperCAmelCase		:			int = 2							)      ->				float:
      return round(((float(__UpperCAmelCase							) - 2_7_3.1_5) * 9 / 5) + 32    ,    __UpperCAmelCase							)

def       UpperCAmelCase_							(   __UpperCAmelCase		:			float    ,    __UpperCAmelCase		:			int = 2							)      ->				float:
      return round((float(__UpperCAmelCase							) * 9 / 5)    ,    __UpperCAmelCase							)

def       UpperCAmelCase_							(   __UpperCAmelCase		:			float    ,    __UpperCAmelCase		:			int = 2							)      ->				float:
      return round((float(__UpperCAmelCase							) - 4_9_1.6_7) * 5 / 9    ,    __UpperCAmelCase							)

def       UpperCAmelCase_							(   __UpperCAmelCase		:			float    ,    __UpperCAmelCase		:			int = 2							)      ->				float:
      return round(float(__UpperCAmelCase							) - 4_5_9.6_7    ,    __UpperCAmelCase							)

def       UpperCAmelCase_							(   __UpperCAmelCase		:			float    ,    __UpperCAmelCase		:			int = 2							)      ->				float:
      return round((float(__UpperCAmelCase							) * 5 / 9)    ,    __UpperCAmelCase							)

def       UpperCAmelCase_							(   __UpperCAmelCase		:			float    ,    __UpperCAmelCase		:			int = 2							)      ->				float:
      return round((float(__UpperCAmelCase							) * 1.2_5 + 2_7_3.1_5)    ,    __UpperCAmelCase							)

def       UpperCAmelCase_							(   __UpperCAmelCase		:			float    ,    __UpperCAmelCase		:			int = 2							)      ->				float:
      return round((float(__UpperCAmelCase							) * 2.2_5 + 32)    ,    __UpperCAmelCase							)

def       UpperCAmelCase_							(   __UpperCAmelCase		:			float    ,    __UpperCAmelCase		:			int = 2							)      ->				float:
      return round((float(__UpperCAmelCase							) * 1.2_5)    ,    __UpperCAmelCase							)

def       UpperCAmelCase_							(   __UpperCAmelCase		:			float    ,    __UpperCAmelCase		:			int = 2							)      ->				float:
      return round((float(__UpperCAmelCase							) * 2.2_5 + 32 + 4_5_9.6_7)    ,    __UpperCAmelCase							)


if __name__ == "__main__":
   import doctest

   doctest.testmod()