import os
import unittest

from transformers import RealmTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.models.realm.tokenization_realm import RealmTokenizer
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class        a_				(		SCREAMING_SNAKE_CASE ,  unittest.TestCase		):






	'''simple docstring'''






	UpperCAmelCase_							=      RealmTokenizer
	UpperCAmelCase_							=      RealmTokenizerFast
	UpperCAmelCase_							=      True
	UpperCAmelCase_							=      True
	UpperCAmelCase_							=      filter_non_english




	def 	__snake_case		(    self      :						Tuple):







								'''simple docstring'''



								super().setUp()

								lowerCAmelCase__  			=      [
								    '[UNK]',
								    '[CLS]',
								    '[SEP]',
								    '[PAD]',
								    '[MASK]',
								    'want',
								    '##want',
								    '##ed',
								    'wa',
								    'un',
								    'runn',
								    '##ing',
								    ',',
								    'low',
								    'lowest',
								]
								lowerCAmelCase__  			=      os.path.join(self.tmpdirname    ,					VOCAB_FILES_NAMES['vocab_file'])
								with open(self.vocab_file    ,					'w'    ,					encoding='utf-8') as vocab_writer:
															vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]))




	def 	__snake_case		(    self      :						Optional[int]    ,					lowercase__      :						List[str]):







								'''simple docstring'''



								lowerCAmelCase__  			=      'UNwant\u00E9d,running'
								lowerCAmelCase__  			=      'unwanted, running'
								return input_text, output_text




	def 	__snake_case		(    self      :						Optional[Any]):







								'''simple docstring'''



								lowerCAmelCase__  			=      self.tokenizer_class(self.vocab_file)

								lowerCAmelCase__  			=      tokenizer.tokenize('UNwant\u00E9d,running')
								self.assertListEqual(a_    ,					['un', '##want', '##ed', ',', 'runn', '##ing'])
								self.assertListEqual(tokenizer.convert_tokens_to_ids(a_)    ,					[9, 6, 7, 12, 10, 11])




	def 	__snake_case		(    self      :						int):







								'''simple docstring'''



								if not self.test_rust_tokenizer:
															return

								lowerCAmelCase__  			=      self.get_tokenizer()
								lowerCAmelCase__  			=      self.get_rust_tokenizer()

								lowerCAmelCase__  			=      'UNwant\u00E9d,running'

								lowerCAmelCase__  			=      tokenizer.tokenize(a_)
								lowerCAmelCase__  			=      rust_tokenizer.tokenize(a_)
								self.assertListEqual(a_    ,					a_)

								lowerCAmelCase__  			=      tokenizer.encode(a_    ,					add_special_tokens=a_)
								lowerCAmelCase__  			=      rust_tokenizer.encode(a_    ,					add_special_tokens=a_)
								self.assertListEqual(a_    ,					a_)

								lowerCAmelCase__  			=      self.get_rust_tokenizer()
								lowerCAmelCase__  			=      tokenizer.encode(a_)
								lowerCAmelCase__  			=      rust_tokenizer.encode(a_)
								self.assertListEqual(a_    ,					a_)

								# With lower casing
								lowerCAmelCase__  			=      self.get_tokenizer(do_lower_case=a_)
								lowerCAmelCase__  			=      self.get_rust_tokenizer(do_lower_case=a_)

								lowerCAmelCase__  			=      'UNwant\u00E9d,running'

								lowerCAmelCase__  			=      tokenizer.tokenize(a_)
								lowerCAmelCase__  			=      rust_tokenizer.tokenize(a_)
								self.assertListEqual(a_    ,					a_)

								lowerCAmelCase__  			=      tokenizer.encode(a_    ,					add_special_tokens=a_)
								lowerCAmelCase__  			=      rust_tokenizer.encode(a_    ,					add_special_tokens=a_)
								self.assertListEqual(a_    ,					a_)

								lowerCAmelCase__  			=      self.get_rust_tokenizer()
								lowerCAmelCase__  			=      tokenizer.encode(a_)
								lowerCAmelCase__  			=      rust_tokenizer.encode(a_)
								self.assertListEqual(a_    ,					a_)




	def 	__snake_case		(    self      :						Dict):







								'''simple docstring'''



								lowerCAmelCase__  			=      BasicTokenizer()

								self.assertListEqual(tokenizer.tokenize('ah\u535A\u63A8zz')    ,					['ah', '\u535A', '\u63A8', 'zz'])




	def 	__snake_case		(    self      :						int):







								'''simple docstring'''



								lowerCAmelCase__  			=      BasicTokenizer(do_lower_case=a_)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')    ,					['hello', '!', 'how', 'are', 'you', '?'])
								self.assertListEqual(tokenizer.tokenize('H\u00E9llo')    ,					['hello'])




	def 	__snake_case		(    self      :						List[Any]):







								'''simple docstring'''



								lowerCAmelCase__  			=      BasicTokenizer(do_lower_case=a_    ,					strip_accents=a_)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')    ,					['hällo', '!', 'how', 'are', 'you', '?'])
								self.assertListEqual(tokenizer.tokenize('H\u00E9llo')    ,					['h\u00E9llo'])




	def 	__snake_case		(    self      :						Dict):







								'''simple docstring'''



								lowerCAmelCase__  			=      BasicTokenizer(do_lower_case=a_    ,					strip_accents=a_)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')    ,					['hallo', '!', 'how', 'are', 'you', '?'])
								self.assertListEqual(tokenizer.tokenize('H\u00E9llo')    ,					['hello'])




	def 	__snake_case		(    self      :						Optional[int]):







								'''simple docstring'''



								lowerCAmelCase__  			=      BasicTokenizer(do_lower_case=a_)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')    ,					['hallo', '!', 'how', 'are', 'you', '?'])
								self.assertListEqual(tokenizer.tokenize('H\u00E9llo')    ,					['hello'])




	def 	__snake_case		(    self      :						Tuple):







								'''simple docstring'''



								lowerCAmelCase__  			=      BasicTokenizer(do_lower_case=a_)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU?  ')    ,					['HeLLo', '!', 'how', 'Are', 'yoU', '?'])




	def 	__snake_case		(    self      :						Optional[Any]):







								'''simple docstring'''



								lowerCAmelCase__  			=      BasicTokenizer(do_lower_case=a_    ,					strip_accents=a_)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')    ,					['HäLLo', '!', 'how', 'Are', 'yoU', '?'])




	def 	__snake_case		(    self      :						Union[str, Any]):







								'''simple docstring'''



								lowerCAmelCase__  			=      BasicTokenizer(do_lower_case=a_    ,					strip_accents=a_)

								self.assertListEqual(
								    tokenizer.tokenize(' \tHäLLo!how  \n Are yoU?  ')    ,					['HaLLo', '!', 'how', 'Are', 'yoU', '?'])




	def 	__snake_case		(    self      :						List[Any]):







								'''simple docstring'''



								lowerCAmelCase__  			=      BasicTokenizer(do_lower_case=a_    ,					never_split=['[UNK]'])

								self.assertListEqual(
								    tokenizer.tokenize(' \tHeLLo!how  \n Are yoU? [UNK]')    ,					['HeLLo', '!', 'how', 'Are', 'yoU', '?', '[UNK]'])




	def 	__snake_case		(    self      :						str):







								'''simple docstring'''



								lowerCAmelCase__  			=      ['[UNK]', '[CLS]', '[SEP]', 'want', '##want', '##ed', 'wa', 'un', 'runn', '##ing']

								lowerCAmelCase__  			=      {}
								for i, token in enumerate(a_):
															lowerCAmelCase__  			=      i
								lowerCAmelCase__  			=      WordpieceTokenizer(vocab=a_    ,					unk_token='[UNK]')

								self.assertListEqual(tokenizer.tokenize('')    ,					[])

								self.assertListEqual(tokenizer.tokenize('unwanted running')    ,					['un', '##want', '##ed', 'runn', '##ing'])

								self.assertListEqual(tokenizer.tokenize('unwantedX running')    ,					['[UNK]', 'runn', '##ing'])




	def 	__snake_case		(    self      :						Any):







								'''simple docstring'''



								self.assertTrue(_is_whitespace(' '))
								self.assertTrue(_is_whitespace('\t'))
								self.assertTrue(_is_whitespace('\r'))
								self.assertTrue(_is_whitespace('\n'))
								self.assertTrue(_is_whitespace('\u00A0'))

								self.assertFalse(_is_whitespace('A'))
								self.assertFalse(_is_whitespace('-'))




	def 	__snake_case		(    self      :						Optional[int]):







								'''simple docstring'''



								self.assertTrue(_is_control('\u0005'))

								self.assertFalse(_is_control('A'))
								self.assertFalse(_is_control(' '))
								self.assertFalse(_is_control('\t'))
								self.assertFalse(_is_control('\r'))




	def 	__snake_case		(    self      :						str):







								'''simple docstring'''



								self.assertTrue(_is_punctuation('-'))
								self.assertTrue(_is_punctuation('$'))
								self.assertTrue(_is_punctuation('`'))
								self.assertTrue(_is_punctuation('.'))

								self.assertFalse(_is_punctuation('A'))
								self.assertFalse(_is_punctuation(' '))




	def 	__snake_case		(    self      :						Union[str, Any]):







								'''simple docstring'''



								lowerCAmelCase__  			=      self.get_tokenizer()

								# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
								self.assertListEqual([tokenizer.tokenize(a_) for t in ['Test', '\xad', 'test']]    ,					[['[UNK]'], [], ['[UNK]']])

								if self.test_rust_tokenizer:
															lowerCAmelCase__  			=      self.get_rust_tokenizer()
															self.assertListEqual(
															    [rust_tokenizer.tokenize(a_) for t in ['Test', '\xad', 'test']]    ,					[['[UNK]'], [], ['[UNK]']])




	@slow
	def 	__snake_case		(    self      :						Any):







								'''simple docstring'''



								lowerCAmelCase__  			=      self.tokenizer_class.from_pretrained('bert-base-uncased')

								lowerCAmelCase__  			=      tokenizer.encode('sequence builders'    ,					add_special_tokens=a_)
								lowerCAmelCase__  			=      tokenizer.encode('multi-sequence build'    ,					add_special_tokens=a_)

								lowerCAmelCase__  			=      tokenizer.build_inputs_with_special_tokens(a_)
								lowerCAmelCase__  			=      tokenizer.build_inputs_with_special_tokens(a_    ,					a_)

								assert encoded_sentence == [101] + text + [102]
								assert encoded_pair == [101] + text + [102] + text_a + [102]




	def 	__snake_case		(    self      :						str):







								'''simple docstring'''



								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
																						lowerCAmelCase__  			=      self.rust_tokenizer_class.from_pretrained(a_    ,					**a_)

																						lowerCAmelCase__  			=      F"""A, naïve {tokenizer_r.mask_token} AllenNLP sentence."""
																						lowerCAmelCase__  			=      tokenizer_r.encode_plus(
																						    a_    ,					return_attention_mask=a_    ,					return_token_type_ids=a_    ,					return_offsets_mapping=a_    ,					add_special_tokens=a_    ,					)

																						lowerCAmelCase__  			=      tokenizer_r.do_lower_case if hasattr(a_    ,					'do_lower_case') else False
																						lowerCAmelCase__  			=      (
																						    [
																						        ((0, 0), tokenizer_r.cls_token),
																						        ((0, 1), 'A'),
																						        ((1, 2), ','),
																						        ((3, 5), 'na'),
																						        ((5, 6), '##ï'),
																						        ((6, 8), '##ve'),
																						        ((9, 15), tokenizer_r.mask_token),
																						        ((16, 21), 'Allen'),
																						        ((21, 23), '##NL'),
																						        ((23, 24), '##P'),
																						        ((25, 33), 'sentence'),
																						        ((33, 34), '.'),
																						        ((0, 0), tokenizer_r.sep_token),
																						    ]
																						    if not do_lower_case
																						    else [
																						        ((0, 0), tokenizer_r.cls_token),
																						        ((0, 1), 'a'),
																						        ((1, 2), ','),
																						        ((3, 8), 'naive'),
																						        ((9, 15), tokenizer_r.mask_token),
																						        ((16, 21), 'allen'),
																						        ((21, 23), '##nl'),
																						        ((23, 24), '##p'),
																						        ((25, 33), 'sentence'),
																						        ((33, 34), '.'),
																						        ((0, 0), tokenizer_r.sep_token),
																						    ]
																						)

																						self.assertEqual(
																						    [e[1] for e in expected_results]    ,					tokenizer_r.convert_ids_to_tokens(tokens['input_ids']))
																						self.assertEqual([e[0] for e in expected_results]    ,					tokens['offset_mapping'])




	@slow
	def 	__snake_case		(    self      :						int):







								'''simple docstring'''



								for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
															with self.subTest(F"""{tokenizer.__class__.__name__} ({pretrained_name})"""):
																						lowerCAmelCase__  			=      self.rust_tokenizer_class.from_pretrained(a_    ,					**a_)
																						lowerCAmelCase__  			=      self.tokenizer_class.from_pretrained(a_    ,					**a_)
																						lowerCAmelCase__  			=      [['Hello world!', 'Nice to meet you!'], ['The cute cat.', 'The adorable dog.']]

																						lowerCAmelCase__  			=      tokenizer_r.batch_encode_candidates(a_    ,					max_length=10    ,					return_tensors='np')
																						lowerCAmelCase__  			=      tokenizer_p.batch_encode_candidates(a_    ,					max_length=10    ,					return_tensors='np')

																						lowerCAmelCase__  			=      (2, 2, 10)

																						self.assertEqual(encoded_sentence_r['input_ids'].shape    ,					a_)
																						self.assertEqual(encoded_sentence_r['attention_mask'].shape    ,					a_)
																						self.assertEqual(encoded_sentence_r['token_type_ids'].shape    ,					a_)

																						self.assertEqual(encoded_sentence_p['input_ids'].shape    ,					a_)
																						self.assertEqual(encoded_sentence_p['attention_mask'].shape    ,					a_)
																						self.assertEqual(encoded_sentence_p['token_type_ids'].shape    ,					a_)

from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowerCAmelCase__					=				logging.get_logger(__name__)

lowerCAmelCase__					=				{
    'ZinengTang/tvlt-base': 'https://huggingface.co/ZinengTang/tvlt-base/blob/main/config.json',
}
class        a_				(		SCREAMING_SNAKE_CASE		):






 '''simple docstring'''






 UpperCAmelCase_							=      'tvlt'




 def __init__(    self      :						Dict    ,					lowercase__      :						str=224    ,					lowercase__      :						int=2_048    ,					lowercase__      :						int=128    ,					lowercase__      :						Optional[Any]=[16, 16]    ,					lowercase__      :						List[str]=[16, 16]    ,					lowercase__      :						Optional[int]=3    ,					lowercase__      :						Tuple=1    ,					lowercase__      :						Union[str, Any]=8    ,					lowercase__      :						int=768    ,					lowercase__      :						Tuple=12    ,					lowercase__      :						List[str]=12    ,					lowercase__      :						Union[str, Any]=3_072    ,					lowercase__      :						Union[str, Any]="gelu"    ,					lowercase__      :						Dict=0.0    ,					lowercase__      :						Tuple=0.0    ,					lowercase__      :						Union[str, Any]=0.02    ,					lowercase__      :						Any=1e-6    ,					lowercase__      :						Dict=True    ,					lowercase__      :						int=False    ,					lowercase__      :						Tuple=16    ,					lowercase__      :						Optional[int]=512    ,					lowercase__      :						int=8    ,					lowercase__      :						List[Any]=2_048    ,					lowercase__      :						Optional[int]=0.75    ,					lowercase__      :						Union[str, Any]=0.15    ,					lowercase__      :						List[str]="frame-level"    ,					lowercase__      :						Dict=True    ,					lowercase__      :						Tuple=True    ,					lowercase__      :						Any="classification"    ,					**lowercase__      :						Union[str, Any]    ,					):







        '''simple docstring'''



        super().__init__(**lowercase__)

        if audio_mask_type not in ("frame-level", "patch_level"):
               raise ValueError(
                   'audio_mask_type must be one of two acceptable strategies - {\'frame_level\', \'patch-level\') '
                   F"""got {audio_mask_type}""")

        lowerCAmelCase__  			=      image_size
        lowerCAmelCase__  			=      spectrogram_length
        lowerCAmelCase__  			=      frequency_length
        lowerCAmelCase__  			=      image_patch_size
        lowerCAmelCase__  			=      audio_patch_size
        lowerCAmelCase__  			=      num_image_channels
        lowerCAmelCase__  			=      num_audio_channels
        lowerCAmelCase__  			=      num_frames

        lowerCAmelCase__  			=      hidden_size
        lowerCAmelCase__  			=      num_hidden_layers
        lowerCAmelCase__  			=      num_attention_heads
        lowerCAmelCase__  			=      intermediate_size
        lowerCAmelCase__  			=      hidden_act
        lowerCAmelCase__  			=      hidden_dropout_prob
        lowerCAmelCase__  			=      attention_probs_dropout_prob
        lowerCAmelCase__  			=      initializer_range
        lowerCAmelCase__  			=      layer_norm_eps
        lowerCAmelCase__  			=      qkv_bias
        lowerCAmelCase__  			=      use_mean_pooling

        lowerCAmelCase__  			=      decoder_num_attention_heads
        lowerCAmelCase__  			=      decoder_hidden_size
        lowerCAmelCase__  			=      decoder_num_hidden_layers
        lowerCAmelCase__  			=      decoder_intermediate_size
        lowerCAmelCase__  			=      pixel_mask_ratio
        lowerCAmelCase__  			=      audio_mask_ratio
        lowerCAmelCase__  			=      audio_mask_type

        lowerCAmelCase__  			=      task_matching
        lowerCAmelCase__  			=      task_mae
        lowerCAmelCase__  			=      loss_type
