






import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer





class __lowerCamelCase				(   tf.keras.layers.Layer):





	"""simple docstring"""






	def __init__(							self ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = "longest" ,					UpperCAmelCase = True ,					UpperCAmelCase = 512 ,					UpperCAmelCase = None ,					UpperCAmelCase = True ,					UpperCAmelCase = True ,					UpperCAmelCase = True ,					**UpperCAmelCase ,					):
							"""simple docstring"""







							super().__init__()
							if use_fast_bert_tokenizer:
													_UpperCAmelCase             =    FastBertTokenizer(
													    lowerCAmelCase_ ,					token_out_type=tf.intaa ,					lower_case_nfd_strip_accents=lowerCAmelCase_ ,					**lowerCAmelCase_					)
							else:
													_UpperCAmelCase             =    tf.lookup.StaticVocabularyTable(
													    tf.lookup.KeyValueTensorInitializer(
													        keys=lowerCAmelCase_ ,					key_dtype=tf.string ,					values=tf.range(tf.size(lowerCAmelCase_ ,					out_type=tf.intaa					) ,					dtype=tf.intaa					) ,					value_dtype=tf.intaa ,					) ,					num_oov_buckets=1 ,					)
													_UpperCAmelCase             =    BertTokenizerLayer(
													    lowerCAmelCase_ ,					token_out_type=tf.intaa ,					lower_case=lowerCAmelCase_ ,					**lowerCAmelCase_					)

							_UpperCAmelCase             =    vocab_list
							_UpperCAmelCase             =    do_lower_case
							_UpperCAmelCase             =    cls_token_id or vocab_list.index('[CLS]'					)
							_UpperCAmelCase             =    sep_token_id or vocab_list.index('[SEP]'					)
							_UpperCAmelCase             =    pad_token_id or vocab_list.index('[PAD]'					)
							_UpperCAmelCase             =    ShrinkLongestTrimmer(max_length - 3 ,					axis=1					)  # Allow room for special tokens
							_UpperCAmelCase             =    max_length
							_UpperCAmelCase             =    padding
							_UpperCAmelCase             =    truncation
							_UpperCAmelCase             =    pad_to_multiple_of
							_UpperCAmelCase             =    return_token_type_ids
							_UpperCAmelCase             =    return_attention_mask

	@classmethod
	def  UpperCamelCase					(							cls ,					UpperCAmelCase ,					**UpperCAmelCase					):  # noqa: F821
							"""simple docstring"""







							_UpperCAmelCase             =    kwargs.pop('do_lower_case' ,					lowerCAmelCase_					)
							_UpperCAmelCase             =    tokenizer.do_lower_case if do_lower_case is None else do_lower_case
							_UpperCAmelCase             =    kwargs.pop('cls_token_id' ,					lowerCAmelCase_					)
							_UpperCAmelCase             =    tokenizer.cls_token_id if cls_token_id is None else cls_token_id
							_UpperCAmelCase             =    kwargs.pop('sep_token_id' ,					lowerCAmelCase_					)
							_UpperCAmelCase             =    tokenizer.sep_token_id if sep_token_id is None else sep_token_id
							_UpperCAmelCase             =    kwargs.pop('pad_token_id' ,					lowerCAmelCase_					)
							_UpperCAmelCase             =    tokenizer.pad_token_id if pad_token_id is None else pad_token_id

							_UpperCAmelCase             =    tokenizer.get_vocab()
							_UpperCAmelCase             =    sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()] ,					key=lambda UpperCAmelCase					: x[1]					)
							_UpperCAmelCase             =    [entry[0] for entry in vocab]
							return cls(
							    vocab_list=lowerCAmelCase_ ,					do_lower_case=lowerCAmelCase_ ,					cls_token_id=lowerCAmelCase_ ,					sep_token_id=lowerCAmelCase_ ,					pad_token_id=lowerCAmelCase_ ,					**lowerCAmelCase_ ,					)

	@classmethod
	def  UpperCamelCase					(							cls ,					UpperCAmelCase ,					*UpperCAmelCase ,					**UpperCAmelCase					):
							"""simple docstring"""







							try:
													_UpperCAmelCase             =    BertTokenizer.from_pretrained(lowerCAmelCase_ ,					*lowerCAmelCase_ ,					**lowerCAmelCase_					)
							except:  # noqa: E722
													from .tokenization_bert_fast import BertTokenizerFast

													_UpperCAmelCase             =    BertTokenizerFast.from_pretrained(lowerCAmelCase_ ,					*lowerCAmelCase_ ,					**lowerCAmelCase_					)
							return cls.from_tokenizer(lowerCAmelCase_ ,					**lowerCAmelCase_					)

	def  UpperCamelCase					(							self ,					UpperCAmelCase					):
							"""simple docstring"""







							if self.do_lower_case:
													_UpperCAmelCase             =    case_fold_utfa(lowerCAmelCase_					)
							_UpperCAmelCase             =    self.tf_tokenizer.tokenize(lowerCAmelCase_					)
							return tokens.merge_dims(1 ,					-1					)

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					UpperCAmelCase=None ,					UpperCAmelCase=None ,					UpperCAmelCase=None ,					UpperCAmelCase=None ,					UpperCAmelCase=None ,					UpperCAmelCase=None ,					UpperCAmelCase=None ,					):
							"""simple docstring"""







							if padding is None:
													_UpperCAmelCase             =    self.padding
							if padding not in ("longest", "max_length"):
													raise ValueError('Padding must be either \'longest\' or \'max_length\'!'					)
							if max_length is not None and text_pair is not None:
													# Because we have to instantiate a Trimmer to do it properly
													raise ValueError('max_length cannot be overridden at call time when truncating paired texts!'					)
							if max_length is None:
													_UpperCAmelCase             =    self.max_length
							if truncation is None:
													_UpperCAmelCase             =    self.truncation
							if pad_to_multiple_of is None:
													_UpperCAmelCase             =    self.pad_to_multiple_of
							if return_token_type_ids is None:
													_UpperCAmelCase             =    self.return_token_type_ids
							if return_attention_mask is None:
													_UpperCAmelCase             =    self.return_attention_mask
							if not isinstance(lowerCAmelCase_ ,					tf.Tensor					):
													_UpperCAmelCase             =    tf.convert_to_tensor(lowerCAmelCase_					)
							if text_pair is not None and not isinstance(lowerCAmelCase_ ,					tf.Tensor					):
													_UpperCAmelCase             =    tf.convert_to_tensor(lowerCAmelCase_					)
							if text_pair is not None:
													if text.shape.rank > 1:
																			raise ValueError('text argument should not be multidimensional when a text pair is supplied!'					)
													if text_pair.shape.rank > 1:
																			raise ValueError('text_pair should not be multidimensional!'					)
							if text.shape.rank == 2:
													_UpperCAmelCase      ,				_UpperCAmelCase             =    text[:, 0], text[:, 1]
							_UpperCAmelCase             =    self.unpaired_tokenize(lowerCAmelCase_					)
							if text_pair is None:  # Unpaired text
													if truncation:
																			_UpperCAmelCase             =    text[:, : max_length - 2]  # Allow room for special tokens
													_UpperCAmelCase      ,				_UpperCAmelCase             =    combine_segments(
													    (text,) ,					start_of_sequence_id=self.cls_token_id ,					end_of_segment_id=self.sep_token_id					)
							else:  # Paired text
													_UpperCAmelCase             =    self.unpaired_tokenize(lowerCAmelCase_					)
													if truncation:
																			_UpperCAmelCase      ,				_UpperCAmelCase             =    self.paired_trimmer.trim([text, text_pair]					)
													_UpperCAmelCase      ,				_UpperCAmelCase             =    combine_segments(
													    (text, text_pair) ,					start_of_sequence_id=self.cls_token_id ,					end_of_segment_id=self.sep_token_id					)
							if padding == "longest":
													_UpperCAmelCase             =    input_ids.bounding_shape(axis=1					)
													if pad_to_multiple_of is not None:
																			# No ceiling division in tensorflow, so we negate floordiv instead
																			_UpperCAmelCase             =    pad_to_multiple_of * (-tf.math.floordiv(-pad_length ,					lowerCAmelCase_					))
							else:
													_UpperCAmelCase             =    max_length

							_UpperCAmelCase      ,				_UpperCAmelCase             =    pad_model_inputs(lowerCAmelCase_ ,					max_seq_length=lowerCAmelCase_ ,					pad_value=self.pad_token_id					)
							_UpperCAmelCase             =    {'input_ids': input_ids}
							if return_attention_mask:
													_UpperCAmelCase             =    attention_mask
							if return_token_type_ids:
													_UpperCAmelCase      ,				_UpperCAmelCase             =    pad_model_inputs(
													    lowerCAmelCase_ ,					max_seq_length=lowerCAmelCase_ ,					pad_value=self.pad_token_id					)
													_UpperCAmelCase             =    token_type_ids
							return output

	def  UpperCamelCase					(							self					):
							"""simple docstring"""







							return {
							    "vocab_list": self.vocab_list,
							    "do_lower_case": self.do_lower_case,
							    "cls_token_id": self.cls_token_id,
							    "sep_token_id": self.sep_token_id,
							    "pad_token_id": self.pad_token_id,
							}








from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    convert_to_rgb,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    OPENAI_CLIP_MEAN,
    OPENAI_CLIP_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


_a   			=						logging.get_logger(__name__)


if is_vision_available():
		import PIL





class __lowerCamelCase				(   snake_case__):





	"""simple docstring"""






	UpperCamelCase__			=   ["pixel_values"]

	def __init__(							self ,					UpperCAmelCase = True ,					UpperCAmelCase = None ,					UpperCAmelCase = PILImageResampling.BICUBIC ,					UpperCAmelCase = True ,					UpperCAmelCase = None ,					UpperCAmelCase = True ,					UpperCAmelCase = 1 / 255 ,					UpperCAmelCase = True ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = True ,					**UpperCAmelCase ,					):
							"""simple docstring"""







							super().__init__(**UpperCAmelCase					)
							_UpperCAmelCase             =    size if size is not None else {'shortest_edge': 224}
							_UpperCAmelCase             =    get_size_dict(UpperCAmelCase ,					default_to_square=UpperCAmelCase					)
							_UpperCAmelCase             =    crop_size if crop_size is not None else {'height': 224, 'width': 224}
							_UpperCAmelCase             =    get_size_dict(UpperCAmelCase					)

							_UpperCAmelCase             =    do_resize
							_UpperCAmelCase             =    size
							_UpperCAmelCase             =    resample
							_UpperCAmelCase             =    do_center_crop
							_UpperCAmelCase             =    crop_size
							_UpperCAmelCase             =    do_rescale
							_UpperCAmelCase             =    rescale_factor
							_UpperCAmelCase             =    do_normalize
							_UpperCAmelCase             =    image_mean if image_mean is not None else OPENAI_CLIP_MEAN
							_UpperCAmelCase             =    image_std if image_std is not None else OPENAI_CLIP_STD
							_UpperCAmelCase             =    do_convert_rgb

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase = PILImageResampling.BICUBIC ,					UpperCAmelCase = None ,					**UpperCAmelCase ,					):
							"""simple docstring"""







							_UpperCAmelCase             =    get_size_dict(UpperCAmelCase ,					default_to_square=UpperCAmelCase					)
							_UpperCAmelCase             =    get_resize_output_image_size(
							    UpperCAmelCase ,					size=(size['height'], size['width']) ,					default_to_square=UpperCAmelCase					)
							return resize(UpperCAmelCase ,					size=UpperCAmelCase ,					resample=UpperCAmelCase ,					data_format=UpperCAmelCase ,					**UpperCAmelCase					)

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase = None ,					**UpperCAmelCase ,					):
							"""simple docstring"""







							_UpperCAmelCase             =    get_size_dict(UpperCAmelCase					)
							return center_crop(UpperCAmelCase ,					size=(size['height'], size['width']) ,					data_format=UpperCAmelCase ,					**UpperCAmelCase					)

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase = None ,					**UpperCAmelCase ,					):
							"""simple docstring"""







							return rescale(UpperCAmelCase ,					scale=UpperCAmelCase ,					data_format=UpperCAmelCase ,					**UpperCAmelCase					)

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase ,					UpperCAmelCase = None ,					**UpperCAmelCase ,					):
							"""simple docstring"""







							return normalize(UpperCAmelCase ,					mean=UpperCAmelCase ,					std=UpperCAmelCase ,					data_format=UpperCAmelCase ,					**UpperCAmelCase					)

	def  UpperCamelCase					(							self ,					UpperCAmelCase ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = None ,					UpperCAmelCase = ChannelDimension.FIRST ,					**UpperCAmelCase ,					):
							"""simple docstring"""







							_UpperCAmelCase             =    do_resize if do_resize is not None else self.do_resize
							_UpperCAmelCase             =    size if size is not None else self.size
							_UpperCAmelCase             =    get_size_dict(UpperCAmelCase ,					default_to_square=UpperCAmelCase					)
							_UpperCAmelCase             =    resample if resample is not None else self.resample
							_UpperCAmelCase             =    do_center_crop if do_center_crop is not None else self.do_center_crop
							_UpperCAmelCase             =    crop_size if crop_size is not None else self.crop_size
							_UpperCAmelCase             =    get_size_dict(UpperCAmelCase					)
							_UpperCAmelCase             =    do_rescale if do_rescale is not None else self.do_rescale
							_UpperCAmelCase             =    rescale_factor if rescale_factor is not None else self.rescale_factor
							_UpperCAmelCase             =    do_normalize if do_normalize is not None else self.do_normalize
							_UpperCAmelCase             =    image_mean if image_mean is not None else self.image_mean
							_UpperCAmelCase             =    image_std if image_std is not None else self.image_std
							_UpperCAmelCase             =    do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb

							_UpperCAmelCase             =    make_list_of_images(UpperCAmelCase					)

							if not valid_images(UpperCAmelCase					):
													raise ValueError(
													    'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
													    'torch.Tensor, tf.Tensor or jax.ndarray.'					)

							if do_resize and size is None:
													raise ValueError('Size must be specified if do_resize is True.'					)

							if do_center_crop and crop_size is None:
													raise ValueError('Crop size must be specified if do_center_crop is True.'					)

							if do_rescale and rescale_factor is None:
													raise ValueError('Rescale factor must be specified if do_rescale is True.'					)

							if do_normalize and (image_mean is None or image_std is None):
													raise ValueError('Image mean and std must be specified if do_normalize is True.'					)

							# PIL RGBA images are converted to RGB
							if do_convert_rgb:
													_UpperCAmelCase             =    [convert_to_rgb(UpperCAmelCase					) for image in images]

							# All transformations expect numpy arrays.
							_UpperCAmelCase             =    [to_numpy_array(UpperCAmelCase					) for image in images]

							if do_resize:
													_UpperCAmelCase             =    [self.resize(image=UpperCAmelCase ,					size=UpperCAmelCase ,					resample=UpperCAmelCase					) for image in images]

							if do_center_crop:
													_UpperCAmelCase             =    [self.center_crop(image=UpperCAmelCase ,					size=UpperCAmelCase					) for image in images]

							if do_rescale:
													_UpperCAmelCase             =    [self.rescale(image=UpperCAmelCase ,					scale=UpperCAmelCase					) for image in images]

							if do_normalize:
													_UpperCAmelCase             =    [self.normalize(image=UpperCAmelCase ,					mean=UpperCAmelCase ,					std=UpperCAmelCase					) for image in images]

							_UpperCAmelCase             =    [to_channel_dimension_format(UpperCAmelCase ,					UpperCAmelCase					) for image in images]

							_UpperCAmelCase             =    {'pixel_values': images}
							return BatchFeature(data=UpperCAmelCase ,					tensor_type=UpperCAmelCase					)
