




'''simple docstring'''





import json
import os
import random
import shutil
import tempfile
import unittest

import numpy as np
import pytest

from transformers import BertTokenizer, BertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_vision
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available


if is_vision_available():
   from PIL import Image

   from transformers import FlavaImageProcessor, FlavaProcessor
   from transformers.models.flava.image_processing_flava import (
       FLAVA_CODEBOOK_MEAN,
       FLAVA_CODEBOOK_STD,
       FLAVA_IMAGE_MEAN,
       FLAVA_IMAGE_STD,
   )

@require_vision
class      _a       (		unittest.TestCase							):






     def        _lowercase			(				self					)							->    Optional[int]:
          _snake_case							=   tempfile.mkdtemp()

          # fmt: off
          _snake_case							=   ["[UNK]", "[CLS]", "[SEP]", "[PAD]", "[MASK]", "want", "##want", "##ed", "wa", "un", "runn", "##ing", ",", "low", "lowest"]
          # fmt: on
          _snake_case							=   os.path.join(self.tmpdirname      ,VOCAB_FILES_NAMES["vocab_file"]					)

          with open(self.vocab_file      ,"w"      ,encoding="utf-8"					) as fp:
               fp.write("".join([x + "\n" for x in vocab_tokens]					)					)

          _snake_case							=   {
              "image_mean": FLAVA_IMAGE_MEAN,
              "image_std": FLAVA_IMAGE_STD,
              "do_normalize": True,
              "do_resize": True,
              "size": 224,
              "do_center_crop": True,
              "crop_size": 224,
              "input_size_patches": 14,
              "total_mask_patches": 75,
              "mask_group_max_patches": None,
              "mask_group_min_patches": 16,
              "mask_group_min_aspect_ratio": 0.3,
              "mask_group_max_aspect_ratio": None,
              "codebook_do_resize": True,
              "codebook_size": 112,
              "codebook_do_center_crop": True,
              "codebook_crop_size": 112,
              "codebook_do_map_pixels": True,
              "codebook_do_normalize": True,
              "codebook_image_mean": FLAVA_CODEBOOK_MEAN,
              "codebook_image_std": FLAVA_CODEBOOK_STD,
          }

          _snake_case							=   os.path.join(self.tmpdirname      ,_SCREAMING_SNAKE_CASE					)
          with open(self.image_processor_file      ,"w"      ,encoding="utf-8"					) as fp:
               json.dump(_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE					)






     def        _lowercase			(				self      ,**_SCREAMING_SNAKE_CASE					)							->    Optional[int]:
          return BertTokenizer.from_pretrained(self.tmpdirname      ,**_SCREAMING_SNAKE_CASE					)






     def        _lowercase			(				self      ,**_SCREAMING_SNAKE_CASE					)							->    Optional[Any]:
          return BertTokenizerFast.from_pretrained(self.tmpdirname      ,**_SCREAMING_SNAKE_CASE					)






     def        _lowercase			(				self      ,**_SCREAMING_SNAKE_CASE					)							->    Optional[int]:
          return FlavaImageProcessor.from_pretrained(self.tmpdirname      ,**_SCREAMING_SNAKE_CASE					)






     def        _lowercase			(				self					)							->    Optional[int]:
          shutil.rmtree(self.tmpdirname					)






     def        _lowercase			(				self					)							->    str:
          _snake_case							=   [np.random.randint(255      ,size=(3, 30, 400)      ,dtype=np.uinta					)]

          _snake_case							=   [Image.fromarray(np.moveaxis(_SCREAMING_SNAKE_CASE      ,0      ,-1					)					) for x in image_inputs]

          return image_inputs






     def        _lowercase			(				self					)							->    Union[str, Any]:
          _snake_case							=   self.get_tokenizer()
          _snake_case							=   self.get_rust_tokenizer()
          _snake_case							=   self.get_image_processor()

          _snake_case							=   FlavaProcessor(tokenizer=_SCREAMING_SNAKE_CASE      ,image_processor=_SCREAMING_SNAKE_CASE					)
          processor_slow.save_pretrained(self.tmpdirname					)
          _snake_case							=   FlavaProcessor.from_pretrained(self.tmpdirname      ,use_fast=_SCREAMING_SNAKE_CASE					)

          _snake_case							=   FlavaProcessor(tokenizer=_SCREAMING_SNAKE_CASE      ,image_processor=_SCREAMING_SNAKE_CASE					)
          processor_fast.save_pretrained(self.tmpdirname					)
          _snake_case							=   FlavaProcessor.from_pretrained(self.tmpdirname					)

          self.assertEqual(processor_slow.tokenizer.get_vocab()      ,tokenizer_slow.get_vocab()					)
          self.assertEqual(processor_fast.tokenizer.get_vocab()      ,tokenizer_fast.get_vocab()					)
          self.assertEqual(tokenizer_slow.get_vocab()      ,tokenizer_fast.get_vocab()					)
          self.assertIsInstance(processor_slow.tokenizer      ,_SCREAMING_SNAKE_CASE					)
          self.assertIsInstance(processor_fast.tokenizer      ,_SCREAMING_SNAKE_CASE					)

          self.assertEqual(processor_slow.image_processor.to_json_string()      ,image_processor.to_json_string()					)
          self.assertEqual(processor_fast.image_processor.to_json_string()      ,image_processor.to_json_string()					)
          self.assertIsInstance(processor_slow.image_processor      ,_SCREAMING_SNAKE_CASE					)
          self.assertIsInstance(processor_fast.image_processor      ,_SCREAMING_SNAKE_CASE					)






     def        _lowercase			(				self					)							->    str:
          _snake_case							=   FlavaProcessor(tokenizer=self.get_tokenizer()      ,image_processor=self.get_image_processor()					)
          processor.save_pretrained(self.tmpdirname					)

          _snake_case							=   self.get_tokenizer(bos_token="(BOS)"      ,eos_token="(EOS)"					)
          _snake_case							=   self.get_image_processor(do_normalize=_SCREAMING_SNAKE_CASE      ,padding_value=1.0					)

          _snake_case							=   FlavaProcessor.from_pretrained(
              self.tmpdirname      ,bos_token="(BOS)"      ,eos_token="(EOS)"      ,do_normalize=_SCREAMING_SNAKE_CASE      ,padding_value=1.0					)

          self.assertEqual(processor.tokenizer.get_vocab()      ,tokenizer_add_kwargs.get_vocab()					)
          self.assertIsInstance(processor.tokenizer      ,_SCREAMING_SNAKE_CASE					)

          self.assertEqual(processor.image_processor.to_json_string()      ,image_processor_add_kwargs.to_json_string()					)
          self.assertIsInstance(processor.image_processor      ,_SCREAMING_SNAKE_CASE					)






     def        _lowercase			(				self					)							->    Any:
          _snake_case							=   self.get_image_processor()
          _snake_case							=   self.get_tokenizer()

          _snake_case							=   FlavaProcessor(tokenizer=_SCREAMING_SNAKE_CASE      ,image_processor=_SCREAMING_SNAKE_CASE					)

          _snake_case							=   self.prepare_image_inputs()

          _snake_case							=   image_processor(_SCREAMING_SNAKE_CASE      ,return_tensors="np"					)
          _snake_case							=   processor(images=_SCREAMING_SNAKE_CASE      ,return_tensors="np"					)

          for key in input_feat_extract.keys():
               self.assertAlmostEqual(input_feat_extract[key].sum()      ,input_processor[key].sum()      ,delta=1e-2					)

          # With rest of the args
          random.seed(1_234					)
          _snake_case							=   image_processor(
              _SCREAMING_SNAKE_CASE      ,return_image_mask=_SCREAMING_SNAKE_CASE      ,return_codebook_pixels=_SCREAMING_SNAKE_CASE      ,return_tensors="np"					)
          random.seed(1_234					)
          _snake_case							=   processor(
              images=_SCREAMING_SNAKE_CASE      ,return_image_mask=_SCREAMING_SNAKE_CASE      ,return_codebook_pixels=_SCREAMING_SNAKE_CASE      ,return_tensors="np"					)

          for key in input_feat_extract.keys():
               self.assertAlmostEqual(input_feat_extract[key].sum()      ,input_processor[key].sum()      ,delta=1e-2					)






     def        _lowercase			(				self					)							->    str:
          _snake_case							=   self.get_image_processor()
          _snake_case							=   self.get_tokenizer()

          _snake_case							=   FlavaProcessor(tokenizer=_SCREAMING_SNAKE_CASE      ,image_processor=_SCREAMING_SNAKE_CASE					)

          _snake_case							=   "lower newer"

          _snake_case							=   processor(text=_SCREAMING_SNAKE_CASE					)

          _snake_case							=   tokenizer(_SCREAMING_SNAKE_CASE					)

          for key in encoded_tok.keys():
               self.assertListEqual(encoded_tok[key]      ,encoded_processor[key]					)






     def        _lowercase			(				self					)							->    Tuple:
          _snake_case							=   self.get_image_processor()
          _snake_case							=   self.get_tokenizer()

          _snake_case							=   FlavaProcessor(tokenizer=_SCREAMING_SNAKE_CASE      ,image_processor=_SCREAMING_SNAKE_CASE					)

          _snake_case							=   "lower newer"
          _snake_case							=   self.prepare_image_inputs()

          _snake_case							=   processor(text=_SCREAMING_SNAKE_CASE      ,images=_SCREAMING_SNAKE_CASE					)

          self.assertListEqual(list(inputs.keys()					)      ,["input_ids", "token_type_ids", "attention_mask", "pixel_values"]					)

          # add extra args
          _snake_case							=   processor(text=_SCREAMING_SNAKE_CASE      ,images=_SCREAMING_SNAKE_CASE      ,return_codebook_pixels=_SCREAMING_SNAKE_CASE      ,return_image_mask=_SCREAMING_SNAKE_CASE					)

          self.assertListEqual(
              list(inputs.keys()					)      ,[
                  "input_ids",
                  "token_type_ids",
                  "attention_mask",
                  "pixel_values",
                  "codebook_pixel_values",
                  "bool_masked_pos",
              ]      ,)

          # test if it raises when no input is passed
          with pytest.raises(_SCREAMING_SNAKE_CASE					):
               processor()






     def        _lowercase			(				self					)							->    Optional[Any]:
          _snake_case							=   self.get_image_processor()
          _snake_case							=   self.get_tokenizer()

          _snake_case							=   FlavaProcessor(tokenizer=_SCREAMING_SNAKE_CASE      ,image_processor=_SCREAMING_SNAKE_CASE					)

          _snake_case							=   [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

          _snake_case							=   processor.batch_decode(_SCREAMING_SNAKE_CASE					)
          _snake_case							=   tokenizer.batch_decode(_SCREAMING_SNAKE_CASE					)

          self.assertListEqual(_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE					)






     def        _lowercase			(				self					)							->    Tuple:
          _snake_case							=   self.get_image_processor()
          _snake_case							=   self.get_tokenizer()

          _snake_case							=   FlavaProcessor(tokenizer=_SCREAMING_SNAKE_CASE      ,image_processor=_SCREAMING_SNAKE_CASE					)

          _snake_case							=   "lower newer"
          _snake_case							=   self.prepare_image_inputs()

          _snake_case							=   processor(text=_SCREAMING_SNAKE_CASE      ,images=_SCREAMING_SNAKE_CASE					)

          self.assertListEqual(list(inputs.keys()					)      ,processor.model_input_names					)








'''simple docstring'''





import unittest

import numpy as np

from transformers import ElectraConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
   from transformers.models.electra.modeling_flax_electra import (
       FlaxElectraForCausalLM,
       FlaxElectraForMaskedLM,
       FlaxElectraForMultipleChoice,
       FlaxElectraForPreTraining,
       FlaxElectraForQuestionAnswering,
       FlaxElectraForSequenceClassification,
       FlaxElectraForTokenClassification,
       FlaxElectraModel,
   )

class      _a       (		unittest.TestCase							):






     def __init__(				self      ,_SCREAMING_SNAKE_CASE      ,_SCREAMING_SNAKE_CASE=13      ,_SCREAMING_SNAKE_CASE=7      ,_SCREAMING_SNAKE_CASE=True      ,_SCREAMING_SNAKE_CASE=True      ,_SCREAMING_SNAKE_CASE=True      ,_SCREAMING_SNAKE_CASE=True      ,_SCREAMING_SNAKE_CASE=99      ,_SCREAMING_SNAKE_CASE=24      ,_SCREAMING_SNAKE_CASE=32      ,_SCREAMING_SNAKE_CASE=5      ,_SCREAMING_SNAKE_CASE=4      ,_SCREAMING_SNAKE_CASE=37      ,_SCREAMING_SNAKE_CASE="gelu"      ,_SCREAMING_SNAKE_CASE=0.1      ,_SCREAMING_SNAKE_CASE=0.1      ,_SCREAMING_SNAKE_CASE=512      ,_SCREAMING_SNAKE_CASE=16      ,_SCREAMING_SNAKE_CASE=2      ,_SCREAMING_SNAKE_CASE=0.0_2      ,_SCREAMING_SNAKE_CASE=4      ,)							->    List[str]:
          _snake_case							=   parent
          _snake_case							=   batch_size
          _snake_case							=   seq_length
          _snake_case							=   is_training
          _snake_case							=   use_attention_mask
          _snake_case							=   use_token_type_ids
          _snake_case							=   use_labels
          _snake_case							=   vocab_size
          _snake_case							=   hidden_size
          _snake_case							=   embedding_size
          _snake_case							=   num_hidden_layers
          _snake_case							=   num_attention_heads
          _snake_case							=   intermediate_size
          _snake_case							=   hidden_act
          _snake_case							=   hidden_dropout_prob
          _snake_case							=   attention_probs_dropout_prob
          _snake_case							=   max_position_embeddings
          _snake_case							=   type_vocab_size
          _snake_case							=   type_sequence_label_size
          _snake_case							=   initializer_range
          _snake_case							=   num_choices






     def        _lowercase			(				self					)							->    int:
          _snake_case							=   ids_tensor([self.batch_size, self.seq_length]      ,self.vocab_size					)

          _snake_case							=   None
          if self.use_attention_mask:
               _snake_case							=   random_attention_mask([self.batch_size, self.seq_length]					)

          _snake_case							=   None
          if self.use_token_type_ids:
               _snake_case							=   ids_tensor([self.batch_size, self.seq_length]      ,self.type_vocab_size					)

          _snake_case							=   ElectraConfig(
              vocab_size=self.vocab_size      ,hidden_size=self.hidden_size      ,embedding_size=self.embedding_size      ,num_hidden_layers=self.num_hidden_layers      ,num_attention_heads=self.num_attention_heads      ,intermediate_size=self.intermediate_size      ,hidden_act=self.hidden_act      ,hidden_dropout_prob=self.hidden_dropout_prob      ,attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,max_position_embeddings=self.max_position_embeddings      ,type_vocab_size=self.type_vocab_size      ,initializer_range=self.initializer_range      ,)

          return config, input_ids, token_type_ids, attention_mask






     def        _lowercase			(				self					)							->    List[str]:
          _snake_case							=   self.prepare_config_and_inputs()
          _snake_case							,      _snake_case							,      _snake_case							,      _snake_case							=   config_and_inputs
          _snake_case							=   {"input_ids": input_ids, "token_type_ids": token_type_ids, "attention_mask": attention_mask}
          return config, inputs_dict







@require_flax
class      _a       (		__lowerCAmelCase      ,      unittest.TestCase							):
     SCREAMING_SNAKE_CASE_							:					List[str]					       =		True

     SCREAMING_SNAKE_CASE_							:					Any					       =		(
         (
             FlaxElectraModel,
             FlaxElectraForCausalLM,
             FlaxElectraForMaskedLM,
             FlaxElectraForPreTraining,
             FlaxElectraForTokenClassification,
             FlaxElectraForQuestionAnswering,
             FlaxElectraForMultipleChoice,
             FlaxElectraForSequenceClassification,
         )
         if is_flax_available()
         else ()
     )






     def        _lowercase			(				self					)							->    List[Any]:
          _snake_case							=   FlaxElectraModelTester(self					)






     @slow
     def        _lowercase			(				self					)							->    List[Any]:
          for model_class_name in self.all_model_classes:
               if model_class_name == FlaxElectraForMaskedLM:
                    _snake_case							=   model_class_name.from_pretrained("google/electra-small-generator"					)
               else:
                    _snake_case							=   model_class_name.from_pretrained("google/electra-small-discriminator"					)
               _snake_case							=   model(np.ones((1, 1)					)					)
               self.assertIsNotNone(_SCREAMING_SNAKE_CASE					)


