






"""simple docstring"""



import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer




class        __lowercase(       tf.keras.layers.Layer  ):






						def __init__(					self    ,						_SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE = None    ,						_SCREAMING_SNAKE_CASE = None    ,						_SCREAMING_SNAKE_CASE = None    ,						_SCREAMING_SNAKE_CASE = "longest"    ,						_SCREAMING_SNAKE_CASE = True    ,						_SCREAMING_SNAKE_CASE = 5_12    ,						_SCREAMING_SNAKE_CASE = None    ,						_SCREAMING_SNAKE_CASE = True    ,						_SCREAMING_SNAKE_CASE = True    ,						_SCREAMING_SNAKE_CASE = True    ,						**_SCREAMING_SNAKE_CASE    ,						)			->  int:




											"""simple docstring"""





											super().__init__()
											if use_fast_bert_tokenizer:
																lowerCamelCase__				=FastBertTokenizer(
																    _SCREAMING_SNAKE_CASE    ,						token_out_type=tf.intaa    ,						lower_case_nfd_strip_accents=_SCREAMING_SNAKE_CASE    ,						**_SCREAMING_SNAKE_CASE)
											else:
																lowerCamelCase__				=tf.lookup.StaticVocabularyTable(
																    tf.lookup.KeyValueTensorInitializer(
																        keys=_SCREAMING_SNAKE_CASE    ,						key_dtype=tf.string    ,						values=tf.range(tf.size(_SCREAMING_SNAKE_CASE    ,						out_type=tf.intaa)    ,						dtype=tf.intaa)    ,						value_dtype=tf.intaa    ,						)    ,						num_oov_buckets=1    ,						)
																lowerCamelCase__				=BertTokenizerLayer(
																    _SCREAMING_SNAKE_CASE    ,						token_out_type=tf.intaa    ,						lower_case=_SCREAMING_SNAKE_CASE    ,						**_SCREAMING_SNAKE_CASE)

											lowerCamelCase__				=vocab_list
											lowerCamelCase__				=do_lower_case
											lowerCamelCase__				=cls_token_id or vocab_list.index('[CLS]')
											lowerCamelCase__				=sep_token_id or vocab_list.index('[SEP]')
											lowerCamelCase__				=pad_token_id or vocab_list.index('[PAD]')
											lowerCamelCase__				=ShrinkLongestTrimmer(max_length - 3    ,						axis=1)  # Allow room for special tokens
											lowerCamelCase__				=max_length
											lowerCamelCase__				=padding
											lowerCamelCase__				=truncation
											lowerCamelCase__				=pad_to_multiple_of
											lowerCamelCase__				=return_token_type_ids
											lowerCamelCase__				=return_attention_mask






						@classmethod
						def 		lowerCAmelCase   (					cls    ,						_SCREAMING_SNAKE_CASE    ,						**_SCREAMING_SNAKE_CASE)			->  Union[str, Any]:  # noqa: F821




											"""simple docstring"""





											lowerCamelCase__				=kwargs.pop('do_lower_case'    ,						_SCREAMING_SNAKE_CASE)
											lowerCamelCase__				=tokenizer.do_lower_case if do_lower_case is None else do_lower_case
											lowerCamelCase__				=kwargs.pop('cls_token_id'    ,						_SCREAMING_SNAKE_CASE)
											lowerCamelCase__				=tokenizer.cls_token_id if cls_token_id is None else cls_token_id
											lowerCamelCase__				=kwargs.pop('sep_token_id'    ,						_SCREAMING_SNAKE_CASE)
											lowerCamelCase__				=tokenizer.sep_token_id if sep_token_id is None else sep_token_id
											lowerCamelCase__				=kwargs.pop('pad_token_id'    ,						_SCREAMING_SNAKE_CASE)
											lowerCamelCase__				=tokenizer.pad_token_id if pad_token_id is None else pad_token_id

											lowerCamelCase__				=tokenizer.get_vocab()
											lowerCamelCase__				=sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]    ,						key=lambda _SCREAMING_SNAKE_CASE: x[1])
											lowerCamelCase__				=[entry[0] for entry in vocab]
											return cls(
											    vocab_list=_SCREAMING_SNAKE_CASE    ,						do_lower_case=_SCREAMING_SNAKE_CASE    ,						cls_token_id=_SCREAMING_SNAKE_CASE    ,						sep_token_id=_SCREAMING_SNAKE_CASE    ,						pad_token_id=_SCREAMING_SNAKE_CASE    ,						**_SCREAMING_SNAKE_CASE    ,						)






						@classmethod
						def 		lowerCAmelCase   (					cls    ,						_SCREAMING_SNAKE_CASE    ,						*_SCREAMING_SNAKE_CASE    ,						**_SCREAMING_SNAKE_CASE)			->  List[str]:




											"""simple docstring"""





											try:
																lowerCamelCase__				=BertTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    ,						*_SCREAMING_SNAKE_CASE    ,						**_SCREAMING_SNAKE_CASE)
											except:  # noqa: E722
																from .tokenization_bert_fast import BertTokenizerFast

																lowerCamelCase__				=BertTokenizerFast.from_pretrained(_SCREAMING_SNAKE_CASE    ,						*_SCREAMING_SNAKE_CASE    ,						**_SCREAMING_SNAKE_CASE)
											return cls.from_tokenizer(_SCREAMING_SNAKE_CASE    ,						**_SCREAMING_SNAKE_CASE)






						def 		lowerCAmelCase   (					self    ,						_SCREAMING_SNAKE_CASE)			->  Any:




											"""simple docstring"""





											if self.do_lower_case:
																lowerCamelCase__				=case_fold_utfa(_SCREAMING_SNAKE_CASE)
											lowerCamelCase__				=self.tf_tokenizer.tokenize(_SCREAMING_SNAKE_CASE)
											return tokens.merge_dims(1    ,						-1)






						def 		lowerCAmelCase   (					self    ,						_SCREAMING_SNAKE_CASE    ,						_SCREAMING_SNAKE_CASE=None    ,						_SCREAMING_SNAKE_CASE=None    ,						_SCREAMING_SNAKE_CASE=None    ,						_SCREAMING_SNAKE_CASE=None    ,						_SCREAMING_SNAKE_CASE=None    ,						_SCREAMING_SNAKE_CASE=None    ,						_SCREAMING_SNAKE_CASE=None    ,						)			->  List[str]:




											"""simple docstring"""





											if padding is None:
																lowerCamelCase__				=self.padding
											if padding not in ("longest", "max_length"):
																raise ValueError('Padding must be either \'longest\' or \'max_length\'!')
											if max_length is not None and text_pair is not None:
																# Because we have to instantiate a Trimmer to do it properly
																raise ValueError('max_length cannot be overridden at call time when truncating paired texts!')
											if max_length is None:
																lowerCamelCase__				=self.max_length
											if truncation is None:
																lowerCamelCase__				=self.truncation
											if pad_to_multiple_of is None:
																lowerCamelCase__				=self.pad_to_multiple_of
											if return_token_type_ids is None:
																lowerCamelCase__				=self.return_token_type_ids
											if return_attention_mask is None:
																lowerCamelCase__				=self.return_attention_mask
											if not isinstance(_SCREAMING_SNAKE_CASE    ,						tf.Tensor):
																lowerCamelCase__				=tf.convert_to_tensor(_SCREAMING_SNAKE_CASE)
											if text_pair is not None and not isinstance(_SCREAMING_SNAKE_CASE    ,						tf.Tensor):
																lowerCamelCase__				=tf.convert_to_tensor(_SCREAMING_SNAKE_CASE)
											if text_pair is not None:
																if text.shape.rank > 1:
																					raise ValueError('text argument should not be multidimensional when a text pair is supplied!')
																if text_pair.shape.rank > 1:
																					raise ValueError('text_pair should not be multidimensional!')
											if text.shape.rank == 2:
																lowerCamelCase__				,     lowerCamelCase__				=text[:, 0], text[:, 1]
											lowerCamelCase__				=self.unpaired_tokenize(_SCREAMING_SNAKE_CASE)
											if text_pair is None:  # Unpaired text
																if truncation:
																					lowerCamelCase__				=text[:, : max_length - 2]  # Allow room for special tokens
																lowerCamelCase__				,     lowerCamelCase__				=combine_segments(
																    (text,)    ,						start_of_sequence_id=self.cls_token_id    ,						end_of_segment_id=self.sep_token_id)
											else:  # Paired text
																lowerCamelCase__				=self.unpaired_tokenize(_SCREAMING_SNAKE_CASE)
																if truncation:
																					lowerCamelCase__				,     lowerCamelCase__				=self.paired_trimmer.trim([text, text_pair])
																lowerCamelCase__				,     lowerCamelCase__				=combine_segments(
																    (text, text_pair)    ,						start_of_sequence_id=self.cls_token_id    ,						end_of_segment_id=self.sep_token_id)
											if padding == "longest":
																lowerCamelCase__				=input_ids.bounding_shape(axis=1)
																if pad_to_multiple_of is not None:
																					# No ceiling division in tensorflow, so we negate floordiv instead
																					lowerCamelCase__				=pad_to_multiple_of * (-tf.math.floordiv(-pad_length    ,						_SCREAMING_SNAKE_CASE))
											else:
																lowerCamelCase__				=max_length

											lowerCamelCase__				,     lowerCamelCase__				=pad_model_inputs(_SCREAMING_SNAKE_CASE    ,						max_seq_length=_SCREAMING_SNAKE_CASE    ,						pad_value=self.pad_token_id)
											lowerCamelCase__				={'input_ids': input_ids}
											if return_attention_mask:
																lowerCamelCase__				=attention_mask
											if return_token_type_ids:
																lowerCamelCase__				,     lowerCamelCase__				=pad_model_inputs(
																    _SCREAMING_SNAKE_CASE    ,						max_seq_length=_SCREAMING_SNAKE_CASE    ,						pad_value=self.pad_token_id)
																lowerCamelCase__				=token_type_ids
											return output






						def 		lowerCAmelCase   (					self)			->  str:




											"""simple docstring"""





											return {
											    "vocab_list": self.vocab_list,
											    "do_lower_case": self.do_lower_case,
											    "cls_token_id": self.cls_token_id,
											    "sep_token_id": self.sep_token_id,
											    "pad_token_id": self.pad_token_id,
											}







"""simple docstring"""



_snake_case       :      Dict       =					[0.485, 0.456, 0.406]
_snake_case       :      int       =					[0.229, 0.224, 0.225]
_snake_case       :      Optional[int]       =					[0.5, 0.5, 0.5]
_snake_case       :      str       =					[0.5, 0.5, 0.5]
_snake_case       :      str       =					[0.4814_5466, 0.457_8275, 0.4082_1073]
_snake_case       :      str       =					[0.2686_2954, 0.2613_0258, 0.2757_7711]