

import random
import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
 import torch

if is_vision_available():
 import PIL

 from transformers import FlavaImageProcessor
 from transformers.image_utils import PILImageResampling
 from transformers.models.flava.image_processing_flava import (
     FLAVA_CODEBOOK_MEAN,
     FLAVA_CODEBOOK_STD,
     FLAVA_IMAGE_MEAN,
     FLAVA_IMAGE_STD,
 )
else:
 _UpperCamelCase     =    _UpperCamelCase     =    _UpperCamelCase     =    _UpperCamelCase     =    None



class 					lowercase		( unittest.TestCase      ):




 def __init__(self , __a , __a=7 , __a=3 , __a=30 , __a=400 , __a=True , __a=None , __a=True , __a=None , __a=None , __a=True , __a=1 / 255 , __a=True , __a=FLAVA_IMAGE_MEAN , __a=FLAVA_IMAGE_STD , __a=14 , __a=75 , __a=None , __a=16 , __a=0.3 , __a=None , __a=True , __a=None , __a=None , __a=True , __a=None , __a=True , __a=True , __a=FLAVA_CODEBOOK_MEAN , __a=FLAVA_CODEBOOK_STD , )				->      Optional[int]:






  """simple docstring"""





  UpperCAmelCase__           =       size if size is not None else {'height': 224, 'width': 224}
  UpperCAmelCase__           =       crop_size if crop_size is not None else {'height': 224, 'width': 224}
  UpperCAmelCase__           =       codebook_size if codebook_size is not None else {'height': 112, 'width': 112}
  UpperCAmelCase__           =       codebook_crop_size if codebook_crop_size is not None else {'height': 112, 'width': 112}

  UpperCAmelCase__           =       parent
  UpperCAmelCase__           =       batch_size
  UpperCAmelCase__           =       num_channels
  UpperCAmelCase__           =       do_resize
  UpperCAmelCase__           =       do_rescale
  UpperCAmelCase__           =       rescale_factor
  UpperCAmelCase__           =       min_resolution
  UpperCAmelCase__           =       max_resolution
  UpperCAmelCase__           =       size
  UpperCAmelCase__           =       resample if resample is not None else PILImageResampling.BICUBIC
  UpperCAmelCase__           =       do_normalize
  UpperCAmelCase__           =       image_mean
  UpperCAmelCase__           =       image_std
  UpperCAmelCase__           =       do_center_crop
  UpperCAmelCase__           =       crop_size

  UpperCAmelCase__           =       input_size_patches
  UpperCAmelCase__           =       total_mask_patches
  UpperCAmelCase__           =       mask_group_max_patches
  UpperCAmelCase__           =       mask_group_min_patches
  UpperCAmelCase__           =       mask_group_min_aspect_ratio
  UpperCAmelCase__           =       mask_group_max_aspect_ratio

  UpperCAmelCase__           =       codebook_do_resize
  UpperCAmelCase__           =       codebook_size
  UpperCAmelCase__           =       codebook_resample if codebook_resample is not None else PILImageResampling.LANCZOS
  UpperCAmelCase__           =       codebook_do_center_crop
  UpperCAmelCase__           =       codebook_crop_size
  UpperCAmelCase__           =       codebook_do_map_pixels
  UpperCAmelCase__           =       codebook_do_normalize
  UpperCAmelCase__           =       codebook_image_mean
  UpperCAmelCase__           =       codebook_image_std




 def 						UpperCamelCase__							(self				)				->      Any:






  """simple docstring"""





  return {
      "image_mean": self.image_mean,
      "image_std": self.image_std,
      "do_normalize": self.do_normalize,
      "do_resize": self.do_resize,
      "size": self.size,
      "resample": self.resample,
      "do_rescale": self.do_rescale,
      "rescale_factor": self.rescale_factor,
      "do_center_crop": self.do_center_crop,
      "crop_size": self.crop_size,
      "input_size_patches": self.input_size_patches,
      "total_mask_patches": self.total_mask_patches,
      "mask_group_max_patches": self.mask_group_max_patches,
      "mask_group_min_patches": self.mask_group_min_patches,
      "mask_group_min_aspect_ratio": self.mask_group_min_aspect_ratio,
      "mask_group_max_aspect_ratio": self.mask_group_min_aspect_ratio,
      "codebook_do_resize": self.codebook_do_resize,
      "codebook_size": self.codebook_size,
      "codebook_resample": self.codebook_resample,
      "codebook_do_center_crop": self.codebook_do_center_crop,
      "codebook_crop_size": self.codebook_crop_size,
      "codebook_do_map_pixels": self.codebook_do_map_pixels,
      "codebook_do_normalize": self.codebook_do_normalize,
      "codebook_image_mean": self.codebook_image_mean,
      "codebook_image_std": self.codebook_image_std,
  }




 def 						UpperCamelCase__							(self				)				->      int:






  """simple docstring"""





  return (self.size["height"], self.size["width"])




 def 						UpperCamelCase__							(self				)				->      Optional[Any]:






  """simple docstring"""





  return (
      (self.input_size_patches, self.input_size_patches)
      if not isinstance(self.input_size_patches , __a				)
      else self.input_size_patches
  )




 def 						UpperCamelCase__							(self				)				->      Optional[Any]:






  """simple docstring"""





  return (self.codebook_size["height"], self.codebook_size["width"])







@require_torch
@require_vision
class 					lowercase		( _UpperCamelCase				,       unittest.TestCase      ):
 __SCREAMING_SNAKE_CASE      							=       FlavaImageProcessor if is_vision_available() else None
 __SCREAMING_SNAKE_CASE      							=       None




 def 						UpperCamelCase__							(self				)				->      Tuple:






  """simple docstring"""





  UpperCAmelCase__           =       FlavaImageProcessingTester(self				)




 @property
 def 						UpperCamelCase__							(self				)				->      Dict:






  """simple docstring"""





  return self.image_processor_tester.prepare_image_processor_dict()




 def 						UpperCamelCase__							(self				)				->      int:






  """simple docstring"""





  UpperCAmelCase__           =       self.image_processing_class(**self.image_processor_dict				)
  self.assertTrue(hasattr(__a , 'image_mean'				)				)
  self.assertTrue(hasattr(__a , 'image_std'				)				)
  self.assertTrue(hasattr(__a , 'do_normalize'				)				)
  self.assertTrue(hasattr(__a , 'do_resize'				)				)
  self.assertTrue(hasattr(__a , 'resample'				)				)
  self.assertTrue(hasattr(__a , 'crop_size'				)				)
  self.assertTrue(hasattr(__a , 'do_center_crop'				)				)
  self.assertTrue(hasattr(__a , 'do_rescale'				)				)
  self.assertTrue(hasattr(__a , 'rescale_factor'				)				)
  self.assertTrue(hasattr(__a , 'masking_generator'				)				)
  self.assertTrue(hasattr(__a , 'codebook_do_resize'				)				)
  self.assertTrue(hasattr(__a , 'codebook_size'				)				)
  self.assertTrue(hasattr(__a , 'codebook_resample'				)				)
  self.assertTrue(hasattr(__a , 'codebook_do_center_crop'				)				)
  self.assertTrue(hasattr(__a , 'codebook_crop_size'				)				)
  self.assertTrue(hasattr(__a , 'codebook_do_map_pixels'				)				)
  self.assertTrue(hasattr(__a , 'codebook_do_normalize'				)				)
  self.assertTrue(hasattr(__a , 'codebook_image_mean'				)				)
  self.assertTrue(hasattr(__a , 'codebook_image_std'				)				)




 def 						UpperCamelCase__							(self				)				->      Tuple:






  """simple docstring"""





  UpperCAmelCase__           =       self.image_processing_class.from_dict(self.image_processor_dict				)
  self.assertEqual(image_processor.size , {'height': 224, 'width': 224}				)
  self.assertEqual(image_processor.crop_size , {'height': 224, 'width': 224}				)
  self.assertEqual(image_processor.codebook_size , {'height': 112, 'width': 112}				)
  self.assertEqual(image_processor.codebook_crop_size , {'height': 112, 'width': 112}				)

  UpperCAmelCase__           =       self.image_processing_class.from_dict(
      self.image_processor_dict , size=42 , crop_size=84 , codebook_size=33 , codebook_crop_size=66				)
  self.assertEqual(image_processor.size , {'height': 42, 'width': 42}				)
  self.assertEqual(image_processor.crop_size , {'height': 84, 'width': 84}				)
  self.assertEqual(image_processor.codebook_size , {'height': 33, 'width': 33}				)
  self.assertEqual(image_processor.codebook_crop_size , {'height': 66, 'width': 66}				)




 def 						UpperCamelCase__							(self				)				->      Any:






  """simple docstring"""





  pass




 def 						UpperCamelCase__							(self				)				->      List[Any]:






  """simple docstring"""





  UpperCAmelCase__           =       self.image_processing_class(**self.image_processor_dict				)
  # create random PIL images
  UpperCAmelCase__           =       prepare_image_inputs(self.image_processor_tester , equal_resolution=__a				)
  for image in image_inputs:
   self.assertIsInstance(__a , PIL.Image.Image				)

  # Test not batched input
  UpperCAmelCase__           =       image_processing(image_inputs[0] , return_tensors='pt'				)

  # Test no bool masked pos
  self.assertFalse('bool_masked_pos' in encoded_images				)

  UpperCAmelCase__       ,					UpperCAmelCase__           =       self.image_processor_tester.get_expected_image_size()

  self.assertEqual(
      encoded_images.pixel_values.shape , (1, self.image_processor_tester.num_channels, expected_height, expected_width) , )

  # Test batched
  UpperCAmelCase__           =       image_processing(__a , return_tensors='pt'				)
  UpperCAmelCase__       ,					UpperCAmelCase__           =       self.image_processor_tester.get_expected_image_size()

  # Test no bool masked pos
  self.assertFalse('bool_masked_pos' in encoded_images				)

  self.assertEqual(
      encoded_images.pixel_values.shape , (
          self.image_processor_tester.batch_size,
          self.image_processor_tester.num_channels,
          expected_height,
          expected_width,
      ) , )




 def 						UpperCamelCase__							(self , __a , __a				)				->      Any:






  """simple docstring"""





  UpperCAmelCase__           =       self.image_processing_class(**self.image_processor_dict				)
  # create random tensors
  UpperCAmelCase__           =       prepare_image_inputs(self.image_processor_tester , equal_resolution=__a , **__a				)
  for image in image_inputs:
   self.assertIsInstance(__a , __a				)

  # Test not batched input
  UpperCAmelCase__           =       image_processing(image_inputs[0] , return_tensors='pt'				)

  UpperCAmelCase__       ,					UpperCAmelCase__           =       self.image_processor_tester.get_expected_image_size()
  self.assertEqual(
      encoded_images.pixel_values.shape , (1, self.image_processor_tester.num_channels, expected_height, expected_width) , )

  UpperCAmelCase__           =       image_processing(__a , return_image_mask=__a , return_tensors='pt'				)

  UpperCAmelCase__       ,					UpperCAmelCase__           =       self.image_processor_tester.get_expected_image_size()
  self.assertEqual(
      encoded_images.pixel_values.shape , (
          self.image_processor_tester.batch_size,
          self.image_processor_tester.num_channels,
          expected_height,
          expected_width,
      ) , )

  UpperCAmelCase__       ,					UpperCAmelCase__           =       self.image_processor_tester.get_expected_mask_size()
  self.assertEqual(
      encoded_images.bool_masked_pos.shape , (
          self.image_processor_tester.batch_size,
          expected_height,
          expected_width,
      ) , )

  # Test batched
  UpperCAmelCase__           =       image_processing(__a , return_tensors='pt'				).pixel_values

  UpperCAmelCase__       ,					UpperCAmelCase__           =       self.image_processor_tester.get_expected_image_size()
  self.assertEqual(
      encoded_images.shape , (
          self.image_processor_tester.batch_size,
          self.image_processor_tester.num_channels,
          expected_height,
          expected_width,
      ) , )

  # Test masking
  UpperCAmelCase__           =       image_processing(__a , return_image_mask=__a , return_tensors='pt'				)

  UpperCAmelCase__       ,					UpperCAmelCase__           =       self.image_processor_tester.get_expected_image_size()
  self.assertEqual(
      encoded_images.pixel_values.shape , (
          self.image_processor_tester.batch_size,
          self.image_processor_tester.num_channels,
          expected_height,
          expected_width,
      ) , )

  UpperCAmelCase__       ,					UpperCAmelCase__           =       self.image_processor_tester.get_expected_mask_size()
  self.assertEqual(
      encoded_images.bool_masked_pos.shape , (
          self.image_processor_tester.batch_size,
          expected_height,
          expected_width,
      ) , )




 def 						UpperCamelCase__							(self				)				->      Dict:






  """simple docstring"""





  self._test_call_framework(np.ndarray , prepare_kwargs={'numpify': True}				)




 def 						UpperCamelCase__							(self				)				->      int:






  """simple docstring"""





  self._test_call_framework(torch.Tensor , prepare_kwargs={'torchify': True}				)




 def 						UpperCamelCase__							(self				)				->      Dict:






  """simple docstring"""





  random.seed(1234				)
  UpperCAmelCase__           =       self.image_processing_class(**self.image_processor_dict				)
  UpperCAmelCase__           =       prepare_image_inputs(self.image_processor_tester , equal_resolution=__a , torchify=__a				)

  # Test not batched input
  UpperCAmelCase__           =       image_processing(image_inputs[0] , return_image_mask=__a , return_tensors='pt'				)
  self.assertEqual(encoded_images.bool_masked_pos.sum().item() , 75				)




 def 						UpperCamelCase__							(self				)				->      Optional[int]:






  """simple docstring"""





  UpperCAmelCase__           =       self.image_processing_class(**self.image_processor_dict				)
  # create random PIL images
  UpperCAmelCase__           =       prepare_image_inputs(self.image_processor_tester , equal_resolution=__a				)
  for image in image_inputs:
   self.assertIsInstance(__a , PIL.Image.Image				)

  # Test not batched input
  UpperCAmelCase__           =       image_processing(image_inputs[0] , return_codebook_pixels=__a , return_tensors='pt'				)
  UpperCAmelCase__       ,					UpperCAmelCase__           =       self.image_processor_tester.get_expected_codebook_image_size()
  self.assertEqual(
      encoded_images.codebook_pixel_values.shape , (1, self.image_processor_tester.num_channels, expected_height, expected_width) , )

  # Test batched
  UpperCAmelCase__           =       image_processing(__a , return_codebook_pixels=__a , return_tensors='pt'				)
  UpperCAmelCase__       ,					UpperCAmelCase__           =       self.image_processor_tester.get_expected_codebook_image_size()
  self.assertEqual(
      encoded_images.codebook_pixel_values.shape , (
          self.image_processor_tester.batch_size,
          self.image_processor_tester.num_channels,
          expected_height,
          expected_width,
      ) , )






import logging
import os
import sys
from dataclasses import dataclass, field
from typing import Optional

import transformers
from transformers import (
    AutoConfig,
    AutoModelForQuestionAnswering,
    AutoTokenizer,
    DataCollatorWithPadding,
    HfArgumentParser,
    SquadDataset,
    Trainer,
    TrainingArguments,
)
from transformers import SquadDataTrainingArguments as DataTrainingArguments
from transformers.trainer_utils import is_main_process


_UpperCamelCase     =    logging.getLogger(__name__)



@dataclass
class 					lowercase		:



 '''simple docstring'''


 __SCREAMING_SNAKE_CASE      							=       field(
     metadata={"""help""": """Path to pretrained model or model identifier from huggingface.co/models"""}      )
 __SCREAMING_SNAKE_CASE      							=       field(
     default=_UpperCamelCase				,       metadata={"""help""": """Pretrained config name or path if not the same as model_name"""}      )
 __SCREAMING_SNAKE_CASE      							=       field(
     default=_UpperCamelCase				,       metadata={"""help""": """Pretrained tokenizer name or path if not the same as model_name"""}      )
 __SCREAMING_SNAKE_CASE      							=       field(default=_UpperCamelCase				,       metadata={"""help""": """Set this flag to use fast tokenization."""}      )
 # If you want to tweak more attributes on your tokenizer, you should do it in a distinct script,
 # or just modify its tokenizer_config.json.
 __SCREAMING_SNAKE_CASE      							=       field(
     default=_UpperCamelCase				,       metadata={"""help""": """Where do you want to store the pretrained models downloaded from huggingface.co"""}				,       )







def 		UpperCamelCase_(    ) ->				List[str]:
 # See all possible arguments in src/transformers/training_args.py
 # or by passing the --help flag to this script.
 # We now keep distinct sets of args, for a cleaner separation of concerns.

 UpperCAmelCase__           =       HfArgumentParser((ModelArguments, DataTrainingArguments, TrainingArguments)     )

 if len(sys.argv     ) == 2 and sys.argv[1].endswith('.json'     ):
  # If we pass only one argument to the script and it's the path to a json file,
  # let's parse it to get our arguments.
  UpperCAmelCase__       ,					UpperCAmelCase__       ,					UpperCAmelCase__           =       parser.parse_json_file(json_file=os.path.abspath(sys.argv[1]     )     )
 else:
  UpperCAmelCase__       ,					UpperCAmelCase__       ,					UpperCAmelCase__           =       parser.parse_args_into_dataclasses()

 if (
     os.path.exists(training_args.output_dir     )
     and os.listdir(training_args.output_dir     )
     and training_args.do_train
     and not training_args.overwrite_output_dir
 ):
  raise ValueError(
      f"Output directory ({training_args.output_dir}) already exists and is not empty. Use"
      ' --overwrite_output_dir to overcome.'     )

 # Setup logging
 logging.basicConfig(
     format='%(asctime)s - %(levelname)s - %(name)s - %(message)s'						,					datefmt='%m/%d/%Y %H:%M:%S'						,					level=logging.INFO if training_args.local_rank in [-1, 0] else logging.WARN						,					)
 logger.warning(
     'Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s'						,					training_args.local_rank						,					training_args.device						,					training_args.n_gpu						,					bool(training_args.local_rank != -1     )						,					training_args.fpaa						,					)
 # Set the verbosity to info of the Transformers logger (on main process only):
 if is_main_process(training_args.local_rank     ):
  transformers.utils.logging.set_verbosity_info()
  transformers.utils.logging.enable_default_handler()
  transformers.utils.logging.enable_explicit_format()
 logger.info('Training/evaluation parameters %s'						,					snake_case__     )

 # Prepare Question-Answering task
 # Load pretrained model and tokenizer
 #
 # Distributed training:
 # The .from_pretrained methods guarantee that only one local process can concurrently
 # download model & vocab.

 UpperCAmelCase__           =       AutoConfig.from_pretrained(
     model_args.config_name if model_args.config_name else model_args.model_name_or_path						,					cache_dir=model_args.cache_dir						,					)
 UpperCAmelCase__           =       AutoTokenizer.from_pretrained(
     model_args.tokenizer_name if model_args.tokenizer_name else model_args.model_name_or_path						,					cache_dir=model_args.cache_dir						,					use_fast=snake_case__						,					)
 UpperCAmelCase__           =       AutoModelForQuestionAnswering.from_pretrained(
     model_args.model_name_or_path						,					from_tf=bool('.ckpt' in model_args.model_name_or_path     )						,					config=snake_case__						,					cache_dir=model_args.cache_dir						,					)

 # Get datasets
 UpperCAmelCase__           =       hasattr(model.config						,					'lang2id'     )
 UpperCAmelCase__           =       (
     SquadDataset(
         snake_case__						,					tokenizer=snake_case__						,					is_language_sensitive=snake_case__						,					cache_dir=model_args.cache_dir     )
     if training_args.do_train
     else None
 )
 UpperCAmelCase__           =       (
     SquadDataset(
         snake_case__						,					tokenizer=snake_case__						,					mode='dev'						,					is_language_sensitive=snake_case__						,					cache_dir=model_args.cache_dir						,					)
     if training_args.do_eval
     else None
 )

 # Data collator
 UpperCAmelCase__           =       DataCollatorWithPadding(snake_case__						,					pad_to_multiple_of=8     ) if training_args.fpaa else None

 # Initialize our Trainer
 UpperCAmelCase__           =       Trainer(
     model=snake_case__						,					args=snake_case__						,					train_dataset=snake_case__						,					eval_dataset=snake_case__						,					data_collator=snake_case__						,					)

 # Training
 if training_args.do_train:
  trainer.train(
      model_path=model_args.model_name_or_path if os.path.isdir(model_args.model_name_or_path     ) else None     )
  trainer.save_model()
  # For convenience, we also re-save the tokenizer to the same directory,
  # so that you can share your model easily on huggingface.co/models =)
  if trainer.is_world_master():
   tokenizer.save_pretrained(training_args.output_dir     )
def 		UpperCamelCase_(    snake_case__:						Optional[int]     ) ->				Union[str, Any]:
 # For xla_spawn (TPUs)
 main()


if __name__ == "__main__":
 main()



