





"""simple docstring"""







import unittest

from transformers import MarkupLMConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
  import torch

  from transformers import (
      MarkupLMForQuestionAnswering,
      MarkupLMForSequenceClassification,
      MarkupLMForTokenClassification,
      MarkupLMModel,
  )

# TODO check dependencies
from transformers import MarkupLMFeatureExtractor, MarkupLMProcessor, MarkupLMTokenizer

class 							snake_case_ :







   """simple docstring"""




   def __init__(		self					,				lowerCamelCase_					,				lowerCamelCase_=1_3					,				lowerCamelCase_=7					,				lowerCamelCase_=True					,				lowerCamelCase_=True					,				lowerCamelCase_=True					,				lowerCamelCase_=True					,				lowerCamelCase_=9_9					,				lowerCamelCase_=3_2					,				lowerCamelCase_=5					,				lowerCamelCase_=4					,				lowerCamelCase_=3_7					,				lowerCamelCase_="gelu"					,				lowerCamelCase_=0.1					,				lowerCamelCase_=0.1					,				lowerCamelCase_=5_1_2					,				lowerCamelCase_=1_6					,				lowerCamelCase_=2					,				lowerCamelCase_=0.02					,				lowerCamelCase_=3					,				lowerCamelCase_=None					,				lowerCamelCase_=2_0					,				lowerCamelCase_=3_0					,				lowerCamelCase_=2					,				lowerCamelCase_=2					,				lowerCamelCase_=1_0					,				)							->      Union[str, Any]:
       UpperCamelCase										=  parent
       UpperCamelCase										=  batch_size
       UpperCamelCase										=  seq_length
       UpperCamelCase										=  is_training
       UpperCamelCase										=  use_input_mask
       UpperCamelCase										=  use_token_type_ids
       UpperCamelCase										=  use_labels
       UpperCamelCase										=  vocab_size
       UpperCamelCase										=  hidden_size
       UpperCamelCase										=  num_hidden_layers
       UpperCamelCase										=  num_attention_heads
       UpperCamelCase										=  intermediate_size
       UpperCamelCase										=  hidden_act
       UpperCamelCase										=  hidden_dropout_prob
       UpperCamelCase										=  attention_probs_dropout_prob
       UpperCamelCase										=  max_position_embeddings
       UpperCamelCase										=  type_vocab_size
       UpperCamelCase										=  type_sequence_label_size
       UpperCamelCase										=  initializer_range
       UpperCamelCase										=  num_labels
       UpperCamelCase										=  scope
       UpperCamelCase										=  max_xpath_tag_unit_embeddings
       UpperCamelCase										=  max_xpath_subs_unit_embeddings
       UpperCamelCase										=  tag_pad_id
       UpperCamelCase										=  subs_pad_id
       UpperCamelCase										=  max_depth







   def      UpperCAmelCase__ (		self)							->      List[str]:
       UpperCamelCase										=  ids_tensor([self.batch_size, self.seq_length]					,				self.vocab_size)

       UpperCamelCase										=  ids_tensor(
           [self.batch_size, self.seq_length, self.max_depth]					,				self.max_xpath_tag_unit_embeddings)

       UpperCamelCase										=  ids_tensor(
           [self.batch_size, self.seq_length, self.max_depth]					,				self.max_xpath_subs_unit_embeddings)

       UpperCamelCase										=  None
       if self.use_input_mask:
           UpperCamelCase										=  ids_tensor([self.batch_size, self.seq_length]					,				vocab_size=2)

       UpperCamelCase										=  None
       if self.use_token_type_ids:
           UpperCamelCase										=  ids_tensor([self.batch_size, self.seq_length]					,				self.type_vocab_size)

       UpperCamelCase										=  None
       UpperCamelCase										=  None
       if self.use_labels:
           UpperCamelCase										=  ids_tensor([self.batch_size]					,				self.type_sequence_label_size)
           UpperCamelCase										=  ids_tensor([self.batch_size, self.seq_length]					,				self.num_labels)

       UpperCamelCase										=  self.get_config()

       return (
           config,
           input_ids,
           xpath_tags_seq,
           xpath_subs_seq,
           token_type_ids,
           input_mask,
           sequence_labels,
           token_labels,
       )







   def      UpperCAmelCase__ (		self)							->      List[str]:
       return MarkupLMConfig(
           vocab_size=self.vocab_size					,				hidden_size=self.hidden_size					,				num_hidden_layers=self.num_hidden_layers					,				num_attention_heads=self.num_attention_heads					,				intermediate_size=self.intermediate_size					,				hidden_act=self.hidden_act					,				hidden_dropout_prob=self.hidden_dropout_prob					,				attention_probs_dropout_prob=self.attention_probs_dropout_prob					,				max_position_embeddings=self.max_position_embeddings					,				type_vocab_size=self.type_vocab_size					,				initializer_range=self.initializer_range					,				max_xpath_tag_unit_embeddings=self.max_xpath_tag_unit_embeddings					,				max_xpath_subs_unit_embeddings=self.max_xpath_subs_unit_embeddings					,				tag_pad_id=self.tag_pad_id					,				subs_pad_id=self.subs_pad_id					,				max_depth=self.max_depth					,				)







   def      UpperCAmelCase__ (		self					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				)							->      List[str]:
       UpperCamelCase										=  MarkupLMModel(config=lowerCamelCase_)
       model.to(lowerCamelCase_)
       model.eval()
       print('''Configs:'''					,				model.config.tag_pad_id					,				model.config.subs_pad_id)
       UpperCamelCase										=  model(lowerCamelCase_					,				attention_mask=lowerCamelCase_					,				token_type_ids=lowerCamelCase_)
       UpperCamelCase										=  model(lowerCamelCase_					,				token_type_ids=lowerCamelCase_)
       UpperCamelCase										=  model(lowerCamelCase_)
       self.parent.assertEqual(result.last_hidden_state.shape					,				(self.batch_size, self.seq_length, self.hidden_size))
       self.parent.assertEqual(result.pooler_output.shape					,				(self.batch_size, self.hidden_size))







   def      UpperCAmelCase__ (		self					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				)							->      Optional[int]:
       UpperCamelCase										=  self.num_labels
       UpperCamelCase										=  MarkupLMForSequenceClassification(lowerCamelCase_)
       model.to(lowerCamelCase_)
       model.eval()
       UpperCamelCase										=  model(
           lowerCamelCase_					,				xpath_tags_seq=lowerCamelCase_					,				xpath_subs_seq=lowerCamelCase_					,				attention_mask=lowerCamelCase_					,				token_type_ids=lowerCamelCase_					,				labels=lowerCamelCase_					,				)
       self.parent.assertEqual(result.logits.shape					,				(self.batch_size, self.num_labels))







   def      UpperCAmelCase__ (		self					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				)							->      int:
       UpperCamelCase										=  self.num_labels
       UpperCamelCase										=  MarkupLMForTokenClassification(config=lowerCamelCase_)
       model.to(lowerCamelCase_)
       model.eval()
       UpperCamelCase										=  model(
           lowerCamelCase_					,				xpath_tags_seq=lowerCamelCase_					,				xpath_subs_seq=lowerCamelCase_					,				attention_mask=lowerCamelCase_					,				token_type_ids=lowerCamelCase_					,				labels=lowerCamelCase_					,				)
       self.parent.assertEqual(result.logits.shape					,				(self.batch_size, self.seq_length, self.num_labels))







   def      UpperCAmelCase__ (		self					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				)							->      str:
       UpperCamelCase										=  MarkupLMForQuestionAnswering(config=lowerCamelCase_)
       model.to(lowerCamelCase_)
       model.eval()
       UpperCamelCase										=  model(
           lowerCamelCase_					,				xpath_tags_seq=lowerCamelCase_					,				xpath_subs_seq=lowerCamelCase_					,				attention_mask=lowerCamelCase_					,				token_type_ids=lowerCamelCase_					,				start_positions=lowerCamelCase_					,				end_positions=lowerCamelCase_					,				)
       self.parent.assertEqual(result.start_logits.shape					,				(self.batch_size, self.seq_length))
       self.parent.assertEqual(result.end_logits.shape					,				(self.batch_size, self.seq_length))







   def      UpperCAmelCase__ (		self)							->      Tuple:
       UpperCamelCase										=  self.prepare_config_and_inputs()
       (
           (
           UpperCamelCase  
       )					,  (
           UpperCamelCase  
       )					,  (
           UpperCamelCase  
       )					,  (
           UpperCamelCase  
       )					,  (
           UpperCamelCase  
       )					,  (
           UpperCamelCase  
       )					,  (
           UpperCamelCase  
       )					,  (
           UpperCamelCase  
       )					,    
       )										=  config_and_inputs
       UpperCamelCase										=  {
           '''input_ids''': input_ids,
           '''xpath_tags_seq''': xpath_tags_seq,
           '''xpath_subs_seq''': xpath_subs_seq,
           '''token_type_ids''': token_type_ids,
           '''attention_mask''': input_mask,
       }
       return config, inputs_dict

@require_torch
class 							snake_case_ (				lowerCamelCase_  ,   lowerCamelCase_  ,   unittest.TestCase ):







   """simple docstring"""




   A_										=					(
       (
           MarkupLMModel,
           MarkupLMForSequenceClassification,
           MarkupLMForTokenClassification,
           MarkupLMForQuestionAnswering,
       )
       if is_torch_available()
       else None
   )
   A_										=					(
       {
           '''feature-extraction''': MarkupLMModel,
           '''question-answering''': MarkupLMForQuestionAnswering,
           '''text-classification''': MarkupLMForSequenceClassification,
           '''token-classification''': MarkupLMForTokenClassification,
           '''zero-shot''': MarkupLMForSequenceClassification,
       }
       if is_torch_available()
       else {}
   )







   def      UpperCAmelCase__ (		self					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_					,				lowerCamelCase_)							->      Tuple:
       # ValueError: Nodes must be of type `List[str]` (single pretokenized example), or `List[List[str]]`
       # (batch of pretokenized examples).
       return True







   def      UpperCAmelCase__ (		self)							->      List[str]:
       UpperCamelCase										=  MarkupLMModelTester(self)
       UpperCamelCase										=  ConfigTester(self					,				config_class=lowerCamelCase_					,				hidden_size=3_7)







   def      UpperCAmelCase__ (		self)							->      Optional[Any]:
       self.config_tester.run_common_tests()







   def      UpperCAmelCase__ (		self)							->      Optional[int]:
       UpperCamelCase										=  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*lowerCamelCase_)







   def      UpperCAmelCase__ (		self)							->      Optional[int]:
       UpperCamelCase										=  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_sequence_classification(*lowerCamelCase_)







   def      UpperCAmelCase__ (		self)							->      str:
       UpperCamelCase										=  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_token_classification(*lowerCamelCase_)







   def      UpperCAmelCase__ (		self)							->      List[Any]:
       UpperCamelCase										=  self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_for_question_answering(*lowerCamelCase_)





def     __snake_case   (							):


    """simple docstring"""







    UpperCamelCase										=  '''
    <!DOCTYPE html>
    <html>
    <head>
    <title>Page Title</title>
    </head>
    <body>

    <h1>This is a Heading</h1>
    <p>This is a paragraph.</p>

    </body>
    </html>
    '''

    return html_string


@require_torch
class 							snake_case_ (				unittest.TestCase ):







   """simple docstring"""




   @cached_property
   def      UpperCAmelCase__ (		self)							->      List[str]:
       # TODO use from_pretrained here
       UpperCamelCase										=  MarkupLMFeatureExtractor()
       UpperCamelCase										=  MarkupLMTokenizer.from_pretrained('''microsoft/markuplm-base''')

       return MarkupLMProcessor(lowerCamelCase_					,				lowerCamelCase_)







   @slow
   def      UpperCAmelCase__ (		self)							->      Optional[Any]:
       UpperCamelCase										=  MarkupLMModel.from_pretrained('''microsoft/markuplm-base''').to(lowerCamelCase_)

       UpperCamelCase										=  self.default_processor

       UpperCamelCase										=  processor(prepare_html_string()					,				return_tensors='''pt''')
       UpperCamelCase										=  inputs.to(lowerCamelCase_)

       # forward pass
       with torch.no_grad():
           UpperCamelCase										=  model(**lowerCamelCase_)

       # verify the last hidden states
       UpperCamelCase										=  torch.Size([1, 1_4, 7_6_8])
       self.assertEqual(outputs.last_hidden_state.shape					,				lowerCamelCase_)

       UpperCamelCase										=  torch.tensor(
           [[0.0675, -0.0052, 0.5001], [-0.2281, 0.0802, 0.2192], [-0.0583, -0.3311, 0.1185]]).to(lowerCamelCase_)

       self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]					,				lowerCamelCase_					,				atol=1e-4))





"""simple docstring"""







import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_video_inputs


if is_torch_available():
  import torch

if is_vision_available():
  from PIL import Image

  from transformers import VideoMAEImageProcessor

class 							snake_case_ (				unittest.TestCase ):







   """simple docstring"""




   def __init__(		self					,				lowerCamelCase_					,				lowerCamelCase_=7					,				lowerCamelCase_=3					,				lowerCamelCase_=1_0					,				lowerCamelCase_=1_8					,				lowerCamelCase_=3_0					,				lowerCamelCase_=4_0_0					,				lowerCamelCase_=True					,				lowerCamelCase_=None					,				lowerCamelCase_=True					,				lowerCamelCase_=[0.5, 0.5, 0.5]					,				lowerCamelCase_=[0.5, 0.5, 0.5]					,				lowerCamelCase_=None					,				)							->      Optional[Any]:
       UpperCamelCase										=  size if size is not None else {'''shortest_edge''': 1_8}
       UpperCamelCase										=  crop_size if crop_size is not None else {'''height''': 1_8, '''width''': 1_8}

       UpperCamelCase										=  parent
       UpperCamelCase										=  batch_size
       UpperCamelCase										=  num_channels
       UpperCamelCase										=  num_frames
       UpperCamelCase										=  image_size
       UpperCamelCase										=  min_resolution
       UpperCamelCase										=  max_resolution
       UpperCamelCase										=  do_resize
       UpperCamelCase										=  size
       UpperCamelCase										=  do_normalize
       UpperCamelCase										=  image_mean
       UpperCamelCase										=  image_std
       UpperCamelCase										=  crop_size







   def      UpperCAmelCase__ (		self)							->      Tuple:
       return {
           "image_mean": self.image_mean,
           "image_std": self.image_std,
           "do_normalize": self.do_normalize,
           "do_resize": self.do_resize,
           "size": self.size,
           "crop_size": self.crop_size,
       }

@require_torch
@require_vision
class 							snake_case_ (				lowerCamelCase_  ,   unittest.TestCase ):







   """simple docstring"""




   A_										=					VideoMAEImageProcessor if is_vision_available() else None







   def      UpperCAmelCase__ (		self)							->      Optional[Any]:
       UpperCamelCase										=  VideoMAEImageProcessingTester(self)







   @property
   def      UpperCAmelCase__ (		self)							->      Optional[int]:
       return self.image_processor_tester.prepare_image_processor_dict()







   def      UpperCAmelCase__ (		self)							->      List[Any]:
       UpperCamelCase										=  self.image_processing_class(**self.image_processor_dict)
       self.assertTrue(hasattr(lowerCamelCase_					,				'''image_mean'''))
       self.assertTrue(hasattr(lowerCamelCase_					,				'''image_std'''))
       self.assertTrue(hasattr(lowerCamelCase_					,				'''do_normalize'''))
       self.assertTrue(hasattr(lowerCamelCase_					,				'''do_resize'''))
       self.assertTrue(hasattr(lowerCamelCase_					,				'''do_center_crop'''))
       self.assertTrue(hasattr(lowerCamelCase_					,				'''size'''))







   def      UpperCAmelCase__ (		self)							->      Dict:
       UpperCamelCase										=  self.image_processing_class.from_dict(self.image_processor_dict)
       self.assertEqual(image_processor.size					,				{'''shortest_edge''': 1_8})
       self.assertEqual(image_processor.crop_size					,				{'''height''': 1_8, '''width''': 1_8})

       UpperCamelCase										=  self.image_processing_class.from_dict(self.image_processor_dict					,				size=4_2					,				crop_size=8_4)
       self.assertEqual(image_processor.size					,				{'''shortest_edge''': 4_2})
       self.assertEqual(image_processor.crop_size					,				{'''height''': 8_4, '''width''': 8_4})







   def      UpperCAmelCase__ (		self)							->      List[str]:
       pass







   def      UpperCAmelCase__ (		self)							->      Dict:
       # Initialize image_processing
       UpperCamelCase										=  self.image_processing_class(**self.image_processor_dict)
       # create random PIL videos
       UpperCamelCase										=  prepare_video_inputs(self.image_processor_tester					,				equal_resolution=lowerCamelCase_)
       for video in video_inputs:
           self.assertIsInstance(lowerCamelCase_					,				lowerCamelCase_)
           self.assertIsInstance(video[0]					,				Image.Image)

       # Test not batched input
       UpperCamelCase										=  image_processing(video_inputs[0]					,				return_tensors='''pt''').pixel_values
       self.assertEqual(
           encoded_videos.shape					,				(
               1,
               self.image_processor_tester.num_frames,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['''height'''],
               self.image_processor_tester.crop_size['''width'''],
           )					,				)

       # Test batched
       UpperCamelCase										=  image_processing(lowerCamelCase_					,				return_tensors='''pt''').pixel_values
       self.assertEqual(
           encoded_videos.shape					,				(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_frames,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['''height'''],
               self.image_processor_tester.crop_size['''width'''],
           )					,				)







   def      UpperCAmelCase__ (		self)							->      Dict:
       # Initialize image_processing
       UpperCamelCase										=  self.image_processing_class(**self.image_processor_dict)
       # create random numpy tensors
       UpperCamelCase										=  prepare_video_inputs(self.image_processor_tester					,				equal_resolution=lowerCamelCase_					,				numpify=lowerCamelCase_)
       for video in video_inputs:
           self.assertIsInstance(lowerCamelCase_					,				lowerCamelCase_)
           self.assertIsInstance(video[0]					,				np.ndarray)

       # Test not batched input
       UpperCamelCase										=  image_processing(video_inputs[0]					,				return_tensors='''pt''').pixel_values
       self.assertEqual(
           encoded_videos.shape					,				(
               1,
               self.image_processor_tester.num_frames,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['''height'''],
               self.image_processor_tester.crop_size['''width'''],
           )					,				)

       # Test batched
       UpperCamelCase										=  image_processing(lowerCamelCase_					,				return_tensors='''pt''').pixel_values
       self.assertEqual(
           encoded_videos.shape					,				(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_frames,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['''height'''],
               self.image_processor_tester.crop_size['''width'''],
           )					,				)







   def      UpperCAmelCase__ (		self)							->      Optional[int]:
       # Initialize image_processing
       UpperCamelCase										=  self.image_processing_class(**self.image_processor_dict)
       # create random PyTorch tensors
       UpperCamelCase										=  prepare_video_inputs(self.image_processor_tester					,				equal_resolution=lowerCamelCase_					,				torchify=lowerCamelCase_)
       for video in video_inputs:
           self.assertIsInstance(lowerCamelCase_					,				lowerCamelCase_)
           self.assertIsInstance(video[0]					,				torch.Tensor)

       # Test not batched input
       UpperCamelCase										=  image_processing(video_inputs[0]					,				return_tensors='''pt''').pixel_values
       self.assertEqual(
           encoded_videos.shape					,				(
               1,
               self.image_processor_tester.num_frames,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['''height'''],
               self.image_processor_tester.crop_size['''width'''],
           )					,				)

       # Test batched
       UpperCamelCase										=  image_processing(lowerCamelCase_					,				return_tensors='''pt''').pixel_values
       self.assertEqual(
           encoded_videos.shape					,				(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_frames,
               self.image_processor_tester.num_channels,
               self.image_processor_tester.crop_size['''height'''],
               self.image_processor_tester.crop_size['''width'''],
           )					,				)