





from transformers import (
    DPRContextEncoderTokenizer,
    DPRContextEncoderTokenizerFast,
    DPRQuestionEncoderTokenizer,
    DPRQuestionEncoderTokenizerFast,
    DPRReaderOutput,
    DPRReaderTokenizer,
    DPRReaderTokenizerFast,
)
from transformers.testing_utils import require_tokenizers, slow
from transformers.tokenization_utils_base import BatchEncoding

from ..bert.test_tokenization_bert import BertTokenizationTest







@require_tokenizers
class     lowerCamelCase(	__snake_case  ):
     '''simple docstring'''


     __magic_name__        = DPRContextEncoderTokenizer
     __magic_name__        = DPRContextEncoderTokenizerFast
     __magic_name__        = True







@require_tokenizers
class     lowerCamelCase(	__snake_case  ):
     '''simple docstring'''


     __magic_name__        = DPRQuestionEncoderTokenizer
     __magic_name__        = DPRQuestionEncoderTokenizerFast
     __magic_name__        = True







@require_tokenizers
class     lowerCamelCase(	__snake_case  ):
     '''simple docstring'''


     __magic_name__        = DPRReaderTokenizer
     __magic_name__        = DPRReaderTokenizerFast
     __magic_name__        = True




     @slow
     def 					lowerCAmelCase__		(						self       ):
         _A    =       self.tokenizer_class.from_pretrained('bert-base-uncased'       )

         _A    =       tokenizer.encode('question sequence'							,    add_special_tokens=snake_case_       )
         _A    =       tokenizer.encode('title sequence'							,    add_special_tokens=snake_case_       )
         _A    =       tokenizer.encode('text sequence ' * 4							,    add_special_tokens=snake_case_       )
         _A    =       [[101] + text_a + [102] + text_a + [102] + text_a]
         _A    =       BatchEncoding({'input_ids': input_ids}       )

         _A    =       [[0] * len(input_ids[0]       )]
         _A    =       [[0] * len(input_ids[0]       )]
         _A    =       [0]
         _A    =       DPRReaderOutput(snake_case_							,    snake_case_							,    snake_case_       )

         _A, _A    =       8, 9
         _A    =       10
         _A    =       10
         _A    =       tokenizer.decode_best_spans(snake_case_							,    snake_case_       )
         self.assertEqual(predicted_spans[0].start_index							,    snake_case_       )
         self.assertEqual(predicted_spans[0].end_index							,    snake_case_       )
         self.assertEqual(predicted_spans[0].doc_id							,    0       )




     @slow
     def 					lowerCAmelCase__		(						self       ):
         _A    =       self.tokenizer_class.from_pretrained('bert-base-uncased'       )

         _A    =       tokenizer.encode('question sequence'							,    add_special_tokens=snake_case_       )
         _A    =       tokenizer.encode('title sequence'							,    add_special_tokens=snake_case_       )
         _A    =       tokenizer.encode('text sequence'							,    add_special_tokens=snake_case_       )
         _A    =       [101] + text_a + [102] + text_a + [102] + text_a
         _A    =       tokenizer(questions=['question sequence']							,    titles=['title sequence']							,    texts=['text sequence']       )
         self.assertIn('input_ids'							,    snake_case_       )
         self.assertIn('attention_mask'							,    snake_case_       )
         self.assertListEqual(encoded_input['input_ids'][0]							,    snake_case_       )












from transformers import SqueezeBertTokenizer, SqueezeBertTokenizerFast
from transformers.testing_utils import require_tokenizers, slow

from ..bert.test_tokenization_bert import BertTokenizationTest







@require_tokenizers
class     lowerCamelCase(	__snake_case  ):
     '''simple docstring'''


     __magic_name__        = SqueezeBertTokenizer
     __magic_name__        = SqueezeBertTokenizerFast
     __magic_name__        = True




     def 					lowerCAmelCase__		(						self							,    **snake_case_       ):
         return SqueezeBertTokenizerFast.from_pretrained(self.tmpdirname							,    **snake_case_       )




     @slow
     def 					lowerCAmelCase__		(						self       ):
         _A    =       SqueezeBertTokenizer.from_pretrained('squeezebert/squeezebert-mnli-headless'       )

         _A    =       tokenizer.encode('sequence builders'							,    add_special_tokens=snake_case_       )
         _A    =       tokenizer.encode('multi-sequence build'							,    add_special_tokens=snake_case_       )

         _A    =       tokenizer.build_inputs_with_special_tokens(snake_case_       )
         _A    =       tokenizer.build_inputs_with_special_tokens(snake_case_							,    snake_case_       )

         assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
         assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
             tokenizer.sep_token_id
         ]





