


import random
import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
     import torch

if is_vision_available():
     import PIL

     from transformers import FlavaImageProcessor
     from transformers.image_utils import PILImageResampling
     from transformers.models.flava.image_processing_flava import (
         FLAVA_CODEBOOK_MEAN,
         FLAVA_CODEBOOK_STD,
         FLAVA_IMAGE_MEAN,
         FLAVA_IMAGE_STD,
     )
else:
     snake_case       :					Optional[Any]        =  None







class 	_snake_case						(   unittest.TestCase							):




 def __init__(	self      ,						_lowerCamelCase      ,						_lowerCamelCase=7      ,						_lowerCamelCase=3      ,						_lowerCamelCase=30      ,						_lowerCamelCase=400      ,						_lowerCamelCase=True      ,						_lowerCamelCase=None      ,						_lowerCamelCase=True      ,						_lowerCamelCase=None      ,						_lowerCamelCase=None      ,						_lowerCamelCase=True      ,						_lowerCamelCase=1 / 255      ,						_lowerCamelCase=True      ,						_lowerCamelCase=FLAVA_IMAGE_MEAN      ,						_lowerCamelCase=FLAVA_IMAGE_STD      ,						_lowerCamelCase=14      ,						_lowerCamelCase=75      ,						_lowerCamelCase=None      ,						_lowerCamelCase=16      ,						_lowerCamelCase=0.3      ,						_lowerCamelCase=None      ,						_lowerCamelCase=True      ,						_lowerCamelCase=None      ,						_lowerCamelCase=None      ,						_lowerCamelCase=True      ,						_lowerCamelCase=None      ,						_lowerCamelCase=True      ,						_lowerCamelCase=True      ,						_lowerCamelCase=FLAVA_CODEBOOK_MEAN      ,						_lowerCamelCase=FLAVA_CODEBOOK_STD      ,						):
       a		:List[str]											= size if size is not None else {'''height''': 224, '''width''': 224}
       a		:int											= crop_size if crop_size is not None else {'''height''': 224, '''width''': 224}
       a		:int											= codebook_size if codebook_size is not None else {'''height''': 112, '''width''': 112}
       a		:Tuple											= codebook_crop_size if codebook_crop_size is not None else {'''height''': 112, '''width''': 112}

       a		:List[Any]											= parent
       a		:List[str]											= batch_size
       a		:Tuple											= num_channels
       a		:Optional[Any]											= do_resize
       a		:Union[str, Any]											= do_rescale
       a		:Dict											= rescale_factor
       a		:Dict											= min_resolution
       a		:Any											= max_resolution
       a		:Dict											= size
       a		:List[str]											= resample if resample is not None else PILImageResampling.BICUBIC
       a		:Dict											= do_normalize
       a		:Any											= image_mean
       a		:Optional[int]											= image_std
       a		:Tuple											= do_center_crop
       a		:int											= crop_size

       a		:Optional[Any]											= input_size_patches
       a		:Any											= total_mask_patches
       a		:int											= mask_group_max_patches
       a		:Any											= mask_group_min_patches
       a		:Tuple											= mask_group_min_aspect_ratio
       a		:Union[str, Any]											= mask_group_max_aspect_ratio

       a		:Optional[int]											= codebook_do_resize
       a		:Tuple											= codebook_size
       a		:List[str]											= codebook_resample if codebook_resample is not None else PILImageResampling.LANCZOS
       a		:List[str]											= codebook_do_center_crop
       a		:Dict											= codebook_crop_size
       a		:List[Any]											= codebook_do_map_pixels
       a		:Dict											= codebook_do_normalize
       a		:Tuple											= codebook_image_mean
       a		:Dict											= codebook_image_std




 def  SCREAMING_SNAKE_CASE__       (	self					):
       return {
           "image_mean": self.image_mean,
           "image_std": self.image_std,
           "do_normalize": self.do_normalize,
           "do_resize": self.do_resize,
           "size": self.size,
           "resample": self.resample,
           "do_rescale": self.do_rescale,
           "rescale_factor": self.rescale_factor,
           "do_center_crop": self.do_center_crop,
           "crop_size": self.crop_size,
           "input_size_patches": self.input_size_patches,
           "total_mask_patches": self.total_mask_patches,
           "mask_group_max_patches": self.mask_group_max_patches,
           "mask_group_min_patches": self.mask_group_min_patches,
           "mask_group_min_aspect_ratio": self.mask_group_min_aspect_ratio,
           "mask_group_max_aspect_ratio": self.mask_group_min_aspect_ratio,
           "codebook_do_resize": self.codebook_do_resize,
           "codebook_size": self.codebook_size,
           "codebook_resample": self.codebook_resample,
           "codebook_do_center_crop": self.codebook_do_center_crop,
           "codebook_crop_size": self.codebook_crop_size,
           "codebook_do_map_pixels": self.codebook_do_map_pixels,
           "codebook_do_normalize": self.codebook_do_normalize,
           "codebook_image_mean": self.codebook_image_mean,
           "codebook_image_std": self.codebook_image_std,
       }




 def  SCREAMING_SNAKE_CASE__       (	self					):
       return (self.size["height"], self.size["width"])




 def  SCREAMING_SNAKE_CASE__       (	self					):
       return (
           (self.input_size_patches, self.input_size_patches)
           if not isinstance(self.input_size_patches      ,						_lowerCamelCase					)
           else self.input_size_patches
       )




 def  SCREAMING_SNAKE_CASE__       (	self					):
       return (self.codebook_size["height"], self.codebook_size["width"])







@require_torch
@require_vision
class 	_snake_case						(   _snake_case		,					unittest.TestCase							):
 SCREAMING_SNAKE_CASE__   = FlavaImageProcessor if is_vision_available() else None
 SCREAMING_SNAKE_CASE__   = None




 def  SCREAMING_SNAKE_CASE__       (	self					):
       a		:Dict											= FlavaImageProcessingTester(self					)




 @property
 def  SCREAMING_SNAKE_CASE__       (	self					):
       return self.image_processor_tester.prepare_image_processor_dict()




 def  SCREAMING_SNAKE_CASE__       (	self					):
       a		:Dict											= self.image_processing_class(**self.image_processor_dict					)
       self.assertTrue(hasattr(_lowerCamelCase      ,						'''image_mean'''					)					)
       self.assertTrue(hasattr(_lowerCamelCase      ,						'''image_std'''					)					)
       self.assertTrue(hasattr(_lowerCamelCase      ,						'''do_normalize'''					)					)
       self.assertTrue(hasattr(_lowerCamelCase      ,						'''do_resize'''					)					)
       self.assertTrue(hasattr(_lowerCamelCase      ,						'''resample'''					)					)
       self.assertTrue(hasattr(_lowerCamelCase      ,						'''crop_size'''					)					)
       self.assertTrue(hasattr(_lowerCamelCase      ,						'''do_center_crop'''					)					)
       self.assertTrue(hasattr(_lowerCamelCase      ,						'''do_rescale'''					)					)
       self.assertTrue(hasattr(_lowerCamelCase      ,						'''rescale_factor'''					)					)
       self.assertTrue(hasattr(_lowerCamelCase      ,						'''masking_generator'''					)					)
       self.assertTrue(hasattr(_lowerCamelCase      ,						'''codebook_do_resize'''					)					)
       self.assertTrue(hasattr(_lowerCamelCase      ,						'''codebook_size'''					)					)
       self.assertTrue(hasattr(_lowerCamelCase      ,						'''codebook_resample'''					)					)
       self.assertTrue(hasattr(_lowerCamelCase      ,						'''codebook_do_center_crop'''					)					)
       self.assertTrue(hasattr(_lowerCamelCase      ,						'''codebook_crop_size'''					)					)
       self.assertTrue(hasattr(_lowerCamelCase      ,						'''codebook_do_map_pixels'''					)					)
       self.assertTrue(hasattr(_lowerCamelCase      ,						'''codebook_do_normalize'''					)					)
       self.assertTrue(hasattr(_lowerCamelCase      ,						'''codebook_image_mean'''					)					)
       self.assertTrue(hasattr(_lowerCamelCase      ,						'''codebook_image_std'''					)					)




 def  SCREAMING_SNAKE_CASE__       (	self					):
       a		:Union[str, Any]											= self.image_processing_class.from_dict(self.image_processor_dict					)
       self.assertEqual(image_processor.size      ,						{'''height''': 224, '''width''': 224}					)
       self.assertEqual(image_processor.crop_size      ,						{'''height''': 224, '''width''': 224}					)
       self.assertEqual(image_processor.codebook_size      ,						{'''height''': 112, '''width''': 112}					)
       self.assertEqual(image_processor.codebook_crop_size      ,						{'''height''': 112, '''width''': 112}					)

       a		:Union[str, Any]											= self.image_processing_class.from_dict(
           self.image_processor_dict      ,						size=42      ,						crop_size=84      ,						codebook_size=33      ,						codebook_crop_size=66					)
       self.assertEqual(image_processor.size      ,						{'''height''': 42, '''width''': 42}					)
       self.assertEqual(image_processor.crop_size      ,						{'''height''': 84, '''width''': 84}					)
       self.assertEqual(image_processor.codebook_size      ,						{'''height''': 33, '''width''': 33}					)
       self.assertEqual(image_processor.codebook_crop_size      ,						{'''height''': 66, '''width''': 66}					)




 def  SCREAMING_SNAKE_CASE__       (	self					):
       pass




 def  SCREAMING_SNAKE_CASE__       (	self					):
       # Initialize image_processing
       a		:Optional[int]											= self.image_processing_class(**self.image_processor_dict					)
       # create random PIL images
       a		:Optional[Any]											= prepare_image_inputs(self.image_processor_tester      ,						equal_resolution=_lowerCamelCase					)
       for image in image_inputs:
             self.assertIsInstance(_lowerCamelCase      ,						PIL.Image.Image					)

       # Test not batched input
       a		:Optional[int]											= image_processing(image_inputs[0]      ,						return_tensors='''pt'''					)

       # Test no bool masked pos
       self.assertFalse('''bool_masked_pos''' in encoded_images					)

       a							,       a		:int											= self.image_processor_tester.get_expected_image_size()

       self.assertEqual(
           encoded_images.pixel_values.shape      ,						(1, self.image_processor_tester.num_channels, expected_height, expected_width)      ,						)

       # Test batched
       a		:Dict											= image_processing(_lowerCamelCase      ,						return_tensors='''pt'''					)
       a							,       a		:List[str]											= self.image_processor_tester.get_expected_image_size()

       # Test no bool masked pos
       self.assertFalse('''bool_masked_pos''' in encoded_images					)

       self.assertEqual(
           encoded_images.pixel_values.shape      ,						(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_channels,
               expected_height,
               expected_width,
           )      ,						)




 def  SCREAMING_SNAKE_CASE__       (	self      ,						_lowerCamelCase      ,						_lowerCamelCase					):
       # Initialize image_processing
       a		:str											= self.image_processing_class(**self.image_processor_dict					)
       # create random tensors
       a		:Union[str, Any]											= prepare_image_inputs(self.image_processor_tester      ,						equal_resolution=_lowerCamelCase      ,						**_lowerCamelCase					)
       for image in image_inputs:
             self.assertIsInstance(_lowerCamelCase      ,						_lowerCamelCase					)

       # Test not batched input
       a		:Dict											= image_processing(image_inputs[0]      ,						return_tensors='''pt'''					)

       a							,       a		:str											= self.image_processor_tester.get_expected_image_size()
       self.assertEqual(
           encoded_images.pixel_values.shape      ,						(1, self.image_processor_tester.num_channels, expected_height, expected_width)      ,						)

       a		:Dict											= image_processing(_lowerCamelCase      ,						return_image_mask=_lowerCamelCase      ,						return_tensors='''pt'''					)

       a							,       a		:str											= self.image_processor_tester.get_expected_image_size()
       self.assertEqual(
           encoded_images.pixel_values.shape      ,						(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_channels,
               expected_height,
               expected_width,
           )      ,						)

       a							,       a		:int											= self.image_processor_tester.get_expected_mask_size()
       self.assertEqual(
           encoded_images.bool_masked_pos.shape      ,						(
               self.image_processor_tester.batch_size,
               expected_height,
               expected_width,
           )      ,						)

       # Test batched
       a		:Optional[int]											= image_processing(_lowerCamelCase      ,						return_tensors='''pt'''					).pixel_values

       a							,       a		:Dict											= self.image_processor_tester.get_expected_image_size()
       self.assertEqual(
           encoded_images.shape      ,						(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_channels,
               expected_height,
               expected_width,
           )      ,						)

       # Test masking
       a		:Any											= image_processing(_lowerCamelCase      ,						return_image_mask=_lowerCamelCase      ,						return_tensors='''pt'''					)

       a							,       a		:Optional[Any]											= self.image_processor_tester.get_expected_image_size()
       self.assertEqual(
           encoded_images.pixel_values.shape      ,						(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_channels,
               expected_height,
               expected_width,
           )      ,						)

       a							,       a		:str											= self.image_processor_tester.get_expected_mask_size()
       self.assertEqual(
           encoded_images.bool_masked_pos.shape      ,						(
               self.image_processor_tester.batch_size,
               expected_height,
               expected_width,
           )      ,						)




 def  SCREAMING_SNAKE_CASE__       (	self					):
       self._test_call_framework(np.ndarray      ,						prepare_kwargs={'''numpify''': True}					)




 def  SCREAMING_SNAKE_CASE__       (	self					):
       self._test_call_framework(torch.Tensor      ,						prepare_kwargs={'''torchify''': True}					)




 def  SCREAMING_SNAKE_CASE__       (	self					):
       # Initialize image_processing
       random.seed(1234					)
       a		:Tuple											= self.image_processing_class(**self.image_processor_dict					)
       a		:Optional[Any]											= prepare_image_inputs(self.image_processor_tester      ,						equal_resolution=_lowerCamelCase      ,						torchify=_lowerCamelCase					)

       # Test not batched input
       a		:Dict											= image_processing(image_inputs[0]      ,						return_image_mask=_lowerCamelCase      ,						return_tensors='''pt'''					)
       self.assertEqual(encoded_images.bool_masked_pos.sum().item()      ,						75					)




 def  SCREAMING_SNAKE_CASE__       (	self					):
       # Initialize image_processing
       a		:Any											= self.image_processing_class(**self.image_processor_dict					)
       # create random PIL images
       a		:Union[str, Any]											= prepare_image_inputs(self.image_processor_tester      ,						equal_resolution=_lowerCamelCase					)
       for image in image_inputs:
             self.assertIsInstance(_lowerCamelCase      ,						PIL.Image.Image					)

       # Test not batched input
       a		:Dict											= image_processing(image_inputs[0]      ,						return_codebook_pixels=_lowerCamelCase      ,						return_tensors='''pt'''					)
       a							,       a		:Any											= self.image_processor_tester.get_expected_codebook_image_size()
       self.assertEqual(
           encoded_images.codebook_pixel_values.shape      ,						(1, self.image_processor_tester.num_channels, expected_height, expected_width)      ,						)

       # Test batched
       a		:Dict											= image_processing(_lowerCamelCase      ,						return_codebook_pixels=_lowerCamelCase      ,						return_tensors='''pt'''					)
       a							,       a		:Union[str, Any]											= self.image_processor_tester.get_expected_codebook_image_size()
       self.assertEqual(
           encoded_images.codebook_pixel_values.shape      ,						(
               self.image_processor_tester.batch_size,
               self.image_processor_tester.num_channels,
               expected_height,
               expected_width,
           )      ,						)









from typing import Optional, Union

from ...file_utils import TensorType
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy







class 	_snake_case						(   _snake_case							):
 SCREAMING_SNAKE_CASE__   = 'MarkupLMFeatureExtractor'
 SCREAMING_SNAKE_CASE__   = ('MarkupLMTokenizer', 'MarkupLMTokenizerFast')
 SCREAMING_SNAKE_CASE__   = True




 def __call__(	self      ,						_lowerCamelCase=None      ,						_lowerCamelCase=None      ,						_lowerCamelCase=None      ,						_lowerCamelCase=None      ,						_lowerCamelCase=None      ,						_lowerCamelCase = True      ,						_lowerCamelCase = False      ,						_lowerCamelCase = None      ,						_lowerCamelCase = None      ,						_lowerCamelCase = 0      ,						_lowerCamelCase = None      ,						_lowerCamelCase = None      ,						_lowerCamelCase = None      ,						_lowerCamelCase = False      ,						_lowerCamelCase = False      ,						_lowerCamelCase = False      ,						_lowerCamelCase = False      ,						_lowerCamelCase = True      ,						_lowerCamelCase = None      ,						**_lowerCamelCase      ,						):
       # first, create nodes and xpaths
       if self.parse_html:
             if html_strings is None:
                   raise ValueError('''Make sure to pass HTML strings in case `parse_html` is set to `True`'''					)

             if nodes is not None or xpaths is not None or node_labels is not None:
                   raise ValueError(
                       '''Please don\'t pass nodes, xpaths nor node labels in case `parse_html` is set to `True`'''					)

             a		:int											= self.feature_extractor(_lowerCamelCase					)
             a		:Tuple											= features['''nodes''']
             a		:Dict											= features['''xpaths''']
       else:
             if html_strings is not None:
                   raise ValueError('''You have passed HTML strings but `parse_html` is set to `False`.'''					)
             if nodes is None or xpaths is None:
                   raise ValueError('''Make sure to pass nodes and xpaths in case `parse_html` is set to `False`'''					)

        # # second, apply the tokenizer
       if questions is not None and self.parse_html:
             if isinstance(_lowerCamelCase      ,						_lowerCamelCase					):
                   a		:Tuple											= [questions]  # add batch dimension (as the feature extractor always adds a batch dimension)

       a		:Optional[int]											= self.tokenizer(
           text=questions if questions is not None else nodes      ,						text_pair=nodes if questions is not None else None      ,						xpaths=_lowerCamelCase      ,						node_labels=_lowerCamelCase      ,						add_special_tokens=_lowerCamelCase      ,						padding=_lowerCamelCase      ,						truncation=_lowerCamelCase      ,						max_length=_lowerCamelCase      ,						stride=_lowerCamelCase      ,						pad_to_multiple_of=_lowerCamelCase      ,						return_token_type_ids=_lowerCamelCase      ,						return_attention_mask=_lowerCamelCase      ,						return_overflowing_tokens=_lowerCamelCase      ,						return_special_tokens_mask=_lowerCamelCase      ,						return_offsets_mapping=_lowerCamelCase      ,						return_length=_lowerCamelCase      ,						verbose=_lowerCamelCase      ,						return_tensors=_lowerCamelCase      ,						**_lowerCamelCase      ,						)

       return encoded_inputs




 def  SCREAMING_SNAKE_CASE__       (	self      ,						*_lowerCamelCase      ,						**_lowerCamelCase					):
       return self.tokenizer.batch_decode(*_lowerCamelCase      ,						**_lowerCamelCase					)




 def  SCREAMING_SNAKE_CASE__       (	self      ,						*_lowerCamelCase      ,						**_lowerCamelCase					):
       return self.tokenizer.decode(*_lowerCamelCase      ,						**_lowerCamelCase					)




 @property
 def  SCREAMING_SNAKE_CASE__       (	self					):
       a		:Any											= self.tokenizer.model_input_names
       return tokenizer_input_names





