

'''simple docstring'''




import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer



class        lowercase_	(		tf.keras.layers.Layer ):

  '''simple docstring'''





  def __init__(     self   , a_   , a_   , a_ = None   , a_ = None   , a_ = None   , a_ = "longest"   , a_ = True   , a_ = 5_1_2   , a_ = None   , a_ = True   , a_ = True   , a_ = True   , **a_   , )							->				List[Any]:


    """simple docstring"""
    super().__init__()
    if use_fast_bert_tokenizer:
      UpperCAmelCase													=	FastBertTokenizer(
          lowerCAmelCase_   , token_out_type=tf.intaa   , lower_case_nfd_strip_accents=lowerCAmelCase_   , **lowerCAmelCase_     )
    else:
      UpperCAmelCase													=	tf.lookup.StaticVocabularyTable(
          tf.lookup.KeyValueTensorInitializer(
              keys=lowerCAmelCase_   , key_dtype=tf.string   , values=tf.range(tf.size(lowerCAmelCase_   , out_type=tf.intaa     )   , dtype=tf.intaa     )   , value_dtype=tf.intaa   , )   , num_oov_buckets=1   , )
      UpperCAmelCase													=	BertTokenizerLayer(
          lowerCAmelCase_   , token_out_type=tf.intaa   , lower_case=lowerCAmelCase_   , **lowerCAmelCase_     )

    UpperCAmelCase													=	vocab_list
    UpperCAmelCase													=	do_lower_case
    UpperCAmelCase													=	cls_token_id or vocab_list.index('[CLS]'     )
    UpperCAmelCase													=	sep_token_id or vocab_list.index('[SEP]'     )
    UpperCAmelCase													=	pad_token_id or vocab_list.index('[PAD]'     )
    UpperCAmelCase													=	ShrinkLongestTrimmer(max_length - 3   , axis=1     )  # Allow room for special tokens
    UpperCAmelCase													=	max_length
    UpperCAmelCase													=	padding
    UpperCAmelCase													=	truncation
    UpperCAmelCase													=	pad_to_multiple_of
    UpperCAmelCase													=	return_token_type_ids
    UpperCAmelCase													=	return_attention_mask
  @classmethod
  def 						snake_case_			(     cls   , a_   , **a_     )							->				Tuple:  # noqa: F821


    """simple docstring"""
    UpperCAmelCase													=	kwargs.pop('do_lower_case'   , lowerCAmelCase_     )
    UpperCAmelCase													=	tokenizer.do_lower_case if do_lower_case is None else do_lower_case
    UpperCAmelCase													=	kwargs.pop('cls_token_id'   , lowerCAmelCase_     )
    UpperCAmelCase													=	tokenizer.cls_token_id if cls_token_id is None else cls_token_id
    UpperCAmelCase													=	kwargs.pop('sep_token_id'   , lowerCAmelCase_     )
    UpperCAmelCase													=	tokenizer.sep_token_id if sep_token_id is None else sep_token_id
    UpperCAmelCase													=	kwargs.pop('pad_token_id'   , lowerCAmelCase_     )
    UpperCAmelCase													=	tokenizer.pad_token_id if pad_token_id is None else pad_token_id

    UpperCAmelCase													=	tokenizer.get_vocab()
    UpperCAmelCase													=	sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]   , key=lambda a_     : x[1]     )
    UpperCAmelCase													=	[entry[0] for entry in vocab]
    return cls(
        vocab_list=lowerCAmelCase_   , do_lower_case=lowerCAmelCase_   , cls_token_id=lowerCAmelCase_   , sep_token_id=lowerCAmelCase_   , pad_token_id=lowerCAmelCase_   , **lowerCAmelCase_   , )
  @classmethod
  def 						snake_case_			(     cls   , a_   , *a_   , **a_     )							->				Optional[int]:


    """simple docstring"""






    try:
      UpperCAmelCase													=	BertTokenizer.from_pretrained(lowerCAmelCase_   , *lowerCAmelCase_   , **lowerCAmelCase_     )
    except:  # noqa: E722
      from .tokenization_bert_fast import BertTokenizerFast

      UpperCAmelCase													=	BertTokenizerFast.from_pretrained(lowerCAmelCase_   , *lowerCAmelCase_   , **lowerCAmelCase_     )
    return cls.from_tokenizer(lowerCAmelCase_   , **lowerCAmelCase_     )
  def 						snake_case_			(     self   , a_     )							->				Dict:


    """simple docstring"""






    if self.do_lower_case:
      UpperCAmelCase													=	case_fold_utfa(lowerCAmelCase_     )
    UpperCAmelCase													=	self.tf_tokenizer.tokenize(lowerCAmelCase_     )
    return tokens.merge_dims(1   , -1     )
  def 						snake_case_			(     self   , a_   , a_=None   , a_=None   , a_=None   , a_=None   , a_=None   , a_=None   , a_=None   , )							->				int:


    """simple docstring"""






    if padding is None:
      UpperCAmelCase													=	self.padding
    if padding not in ("longest", "max_length"):
      raise ValueError('Padding must be either \'longest\' or \'max_length\'!'     )
    if max_length is not None and text_pair is not None:
      # Because we have to instantiate a Trimmer to do it properly
      raise ValueError('max_length cannot be overridden at call time when truncating paired texts!'     )
    if max_length is None:
      UpperCAmelCase													=	self.max_length
    if truncation is None:
      UpperCAmelCase													=	self.truncation
    if pad_to_multiple_of is None:
      UpperCAmelCase													=	self.pad_to_multiple_of
    if return_token_type_ids is None:
      UpperCAmelCase													=	self.return_token_type_ids
    if return_attention_mask is None:
      UpperCAmelCase													=	self.return_attention_mask
    if not isinstance(lowerCAmelCase_   , tf.Tensor     ):
      UpperCAmelCase													=	tf.convert_to_tensor(lowerCAmelCase_     )
    if text_pair is not None and not isinstance(lowerCAmelCase_   , tf.Tensor     ):
      UpperCAmelCase													=	tf.convert_to_tensor(lowerCAmelCase_     )
    if text_pair is not None:
      if text.shape.rank > 1:
        raise ValueError('text argument should not be multidimensional when a text pair is supplied!'     )
      if text_pair.shape.rank > 1:
        raise ValueError('text_pair should not be multidimensional!'     )
    if text.shape.rank == 2:
      UpperCAmelCase							,      UpperCAmelCase													=	text[:, 0], text[:, 1]
    UpperCAmelCase													=	self.unpaired_tokenize(lowerCAmelCase_     )
    if text_pair is None:  # Unpaired text
      if truncation:
        UpperCAmelCase													=	text[:, : max_length - 2]  # Allow room for special tokens
      UpperCAmelCase							,      UpperCAmelCase													=	combine_segments(
          (text,)   , start_of_sequence_id=self.cls_token_id   , end_of_segment_id=self.sep_token_id     )
    else:  # Paired text
      UpperCAmelCase													=	self.unpaired_tokenize(lowerCAmelCase_     )
      if truncation:
        UpperCAmelCase							,      UpperCAmelCase													=	self.paired_trimmer.trim([text, text_pair]     )
      UpperCAmelCase							,      UpperCAmelCase													=	combine_segments(
          (text, text_pair)   , start_of_sequence_id=self.cls_token_id   , end_of_segment_id=self.sep_token_id     )
    if padding == "longest":
      UpperCAmelCase													=	input_ids.bounding_shape(axis=1     )
      if pad_to_multiple_of is not None:
        # No ceiling division in tensorflow, so we negate floordiv instead
        UpperCAmelCase													=	pad_to_multiple_of * (-tf.math.floordiv(-pad_length   , lowerCAmelCase_     ))
    else:
      UpperCAmelCase													=	max_length

    UpperCAmelCase							,      UpperCAmelCase													=	pad_model_inputs(lowerCAmelCase_   , max_seq_length=lowerCAmelCase_   , pad_value=self.pad_token_id     )
    UpperCAmelCase													=	{'input_ids': input_ids}
    if return_attention_mask:
      UpperCAmelCase													=	attention_mask
    if return_token_type_ids:
      UpperCAmelCase							,      UpperCAmelCase													=	pad_model_inputs(
          lowerCAmelCase_   , max_seq_length=lowerCAmelCase_   , pad_value=self.pad_token_id     )
      UpperCAmelCase													=	token_type_ids
    return output





  def 						snake_case_			(     self     )							->				int:


    """simple docstring"""
    return {
        "vocab_list": self.vocab_list,
        "do_lower_case": self.do_lower_case,
        "cls_token_id": self.cls_token_id,
        "sep_token_id": self.sep_token_id,
        "pad_token_id": self.pad_token_id,
    }





'''simple docstring'''




from typing import List, Optional, Tuple, Union

from . import logging
from .import_utils import is_torch_available, is_torch_version


if is_torch_available():
  import torch

_a				: List[Any] 							=						logging.get_logger(__name__)  # pylint: disable=invalid-name

try:
  from torch._dynamo import allow_in_graph as maybe_allow_in_graph
except (ImportError, ModuleNotFoundError):


  def       lowerCamelCase__		(						cls      :   Dict						):
    return cls


def       lowerCamelCase__		(						SCREAMING_SNAKE_CASE      :   Union[Tuple, List]       ,   SCREAMING_SNAKE_CASE      :   Optional[Union[List["torch.Generator"], "torch.Generator"]] = None       ,   SCREAMING_SNAKE_CASE      :   Optional["torch.device"] = None       ,   SCREAMING_SNAKE_CASE      :   Optional["torch.dtype"] = None       ,   SCREAMING_SNAKE_CASE      :   Optional["torch.layout"] = None       ,   ):
  UpperCAmelCase													=	device
  UpperCAmelCase													=	shape[0]

  UpperCAmelCase													=	layout or torch.strided
  UpperCAmelCase													=	device or torch.device('cpu'						)

  if generator is not None:
    UpperCAmelCase													=	generator.device.type if not isinstance(SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE						) else generator[0].device.type
    if gen_device_type != device.type and gen_device_type == "cpu":
      UpperCAmelCase													=	'cpu'
      if device != "mps":
        logger.info(
            f'''The passed generator was created on \'cpu\' even though a tensor on {device} was expected.'''
            f''' Tensors will be created on \'cpu\' and then moved to {device}. Note that one can probably'''
            f''' slighly speed up this function by passing a generator that was created on the {device} device.'''						)
    elif gen_device_type != device.type and gen_device_type == "cuda":
      raise ValueError(f'''Cannot generate a {device} tensor from a generator of type {gen_device_type}.'''						)

  if isinstance(SCREAMING_SNAKE_CASE       ,   SCREAMING_SNAKE_CASE						):
    UpperCAmelCase													=	(1,) + shape[1:]
    UpperCAmelCase													=	[
        torch.randn(SCREAMING_SNAKE_CASE       ,   generator=generator[i]       ,   device=SCREAMING_SNAKE_CASE       ,   dtype=SCREAMING_SNAKE_CASE       ,   layout=SCREAMING_SNAKE_CASE						)
        for i in range(SCREAMING_SNAKE_CASE						)
    ]
    UpperCAmelCase													=	torch.cat(SCREAMING_SNAKE_CASE       ,   dim=0						).to(SCREAMING_SNAKE_CASE						)
  else:
    UpperCAmelCase													=	torch.randn(SCREAMING_SNAKE_CASE       ,   generator=SCREAMING_SNAKE_CASE       ,   device=SCREAMING_SNAKE_CASE       ,   dtype=SCREAMING_SNAKE_CASE       ,   layout=SCREAMING_SNAKE_CASE						).to(SCREAMING_SNAKE_CASE						)

  return latents



def       lowerCamelCase__		(						SCREAMING_SNAKE_CASE      :   Tuple						):
  if is_torch_version('<'       ,   '2.0.0'						) or not hasattr(SCREAMING_SNAKE_CASE       ,   '_dynamo'						):
    return False
  return isinstance(SCREAMING_SNAKE_CASE       ,   torch._dynamo.eval_frame.OptimizedModule						)


