


import os
from typing import List, Union

import tensorflow as tf
from tensorflow_text import BertTokenizer as BertTokenizerLayer
from tensorflow_text import FastBertTokenizer, ShrinkLongestTrimmer, case_fold_utfa, combine_segments, pad_model_inputs

from .tokenization_bert import BertTokenizer



class     snake_case__  (      tf.keras.layers.Layer  ):
				def __init__(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase     ,      _UpperCAmelCase = None     ,      _UpperCAmelCase = None     ,      _UpperCAmelCase = None     ,      _UpperCAmelCase = "longest"     ,      _UpperCAmelCase = True     ,      _UpperCAmelCase = 512     ,      _UpperCAmelCase = None     ,      _UpperCAmelCase = True     ,      _UpperCAmelCase = True     ,      _UpperCAmelCase = True     ,      **_UpperCAmelCase     ,      ):




										"""simple docstring"""




										super().__init__()
										if use_fast_bert_tokenizer:
																__magic_name__				:							int				=      FastBertTokenizer(
																    _UpperCAmelCase     ,      token_out_type=tf.intaa     ,      lower_case_nfd_strip_accents=_UpperCAmelCase     ,      **_UpperCAmelCase      )
										else:
																__magic_name__				:							List[Any]				=      tf.lookup.StaticVocabularyTable(
																    tf.lookup.KeyValueTensorInitializer(
																        keys=_UpperCAmelCase     ,      key_dtype=tf.string     ,      values=tf.range(tf.size(_UpperCAmelCase     ,      out_type=tf.intaa      )     ,      dtype=tf.intaa      )     ,      value_dtype=tf.intaa     ,      )     ,      num_oov_buckets=1     ,      )
																__magic_name__				:							List[Any]				=      BertTokenizerLayer(
																    _UpperCAmelCase     ,      token_out_type=tf.intaa     ,      lower_case=_UpperCAmelCase     ,      **_UpperCAmelCase      )

										__magic_name__				:							Dict				=      vocab_list
										__magic_name__				:							int				=      do_lower_case
										__magic_name__				:							int				=      cls_token_id or vocab_list.index('[CLS]'      )
										__magic_name__				:							Optional[Any]				=      sep_token_id or vocab_list.index('[SEP]'      )
										__magic_name__				:							Tuple				=      pad_token_id or vocab_list.index('[PAD]'      )
										__magic_name__				:							Dict				=      ShrinkLongestTrimmer(max_length - 3     ,      axis=1      )  # Allow room for special tokens
										__magic_name__				:							int				=      max_length
										__magic_name__				:							int				=      padding
										__magic_name__				:							int				=      truncation
										__magic_name__				:							int				=      pad_to_multiple_of
										__magic_name__				:							Optional[Any]				=      return_token_type_ids
										__magic_name__				:							Optional[Any]				=      return_attention_mask
				@classmethod
				def  UpperCamelCase_						(  cls     ,      _UpperCAmelCase     ,      **_UpperCAmelCase      ):  # noqa: F821




										"""simple docstring"""




										__magic_name__				:							str				=      kwargs.pop('do_lower_case'     ,      _UpperCAmelCase      )
										__magic_name__				:							Tuple				=      tokenizer.do_lower_case if do_lower_case is None else do_lower_case
										__magic_name__				:							int				=      kwargs.pop('cls_token_id'     ,      _UpperCAmelCase      )
										__magic_name__				:							Any				=      tokenizer.cls_token_id if cls_token_id is None else cls_token_id
										__magic_name__				:							str				=      kwargs.pop('sep_token_id'     ,      _UpperCAmelCase      )
										__magic_name__				:							Tuple				=      tokenizer.sep_token_id if sep_token_id is None else sep_token_id
										__magic_name__				:							Any				=      kwargs.pop('pad_token_id'     ,      _UpperCAmelCase      )
										__magic_name__				:							Any				=      tokenizer.pad_token_id if pad_token_id is None else pad_token_id

										__magic_name__				:							int				=      tokenizer.get_vocab()
										__magic_name__				:							Union[str, Any]				=      sorted([(wordpiece, idx) for wordpiece, idx in vocab.items()]     ,      key=lambda _UpperCAmelCase      : x[1]      )
										__magic_name__				:							int				=      [entry[0] for entry in vocab]
										return cls(
										    vocab_list=_UpperCAmelCase     ,      do_lower_case=_UpperCAmelCase     ,      cls_token_id=_UpperCAmelCase     ,      sep_token_id=_UpperCAmelCase     ,      pad_token_id=_UpperCAmelCase     ,      **_UpperCAmelCase     ,      )
				@classmethod
				def  UpperCamelCase_						(  cls     ,      _UpperCAmelCase     ,      *_UpperCAmelCase     ,      **_UpperCAmelCase      ):




										"""simple docstring"""




										try:
																__magic_name__				:							Dict				=      BertTokenizer.from_pretrained(_UpperCAmelCase     ,      *_UpperCAmelCase     ,      **_UpperCAmelCase      )
										except:  # noqa: E722
																from .tokenization_bert_fast import BertTokenizerFast

																__magic_name__				:							List[str]				=      BertTokenizerFast.from_pretrained(_UpperCAmelCase     ,      *_UpperCAmelCase     ,      **_UpperCAmelCase      )
										return cls.from_tokenizer(_UpperCAmelCase     ,      **_UpperCAmelCase      )
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase      ):




										"""simple docstring"""




										if self.do_lower_case:
																__magic_name__				:							List[str]				=      case_fold_utfa(_UpperCAmelCase      )
										__magic_name__				:							List[Any]				=      self.tf_tokenizer.tokenize(_UpperCAmelCase      )
										return tokens.merge_dims(1     ,      -1      )
				def  UpperCamelCase_						(  self     ,      _UpperCAmelCase     ,      _UpperCAmelCase=None     ,      _UpperCAmelCase=None     ,      _UpperCAmelCase=None     ,      _UpperCAmelCase=None     ,      _UpperCAmelCase=None     ,      _UpperCAmelCase=None     ,      _UpperCAmelCase=None     ,      ):




										"""simple docstring"""




										if padding is None:
																__magic_name__				:							Optional[Any]				=      self.padding
										if padding not in ("longest", "max_length"):
																raise ValueError('Padding must be either \'longest\' or \'max_length\'!'      )
										if max_length is not None and text_pair is not None:
																# Because we have to instantiate a Trimmer to do it properly
																raise ValueError('max_length cannot be overridden at call time when truncating paired texts!'      )
										if max_length is None:
																__magic_name__				:							Union[str, Any]				=      self.max_length
										if truncation is None:
																__magic_name__				:							int				=      self.truncation
										if pad_to_multiple_of is None:
																__magic_name__				:							Optional[int]				=      self.pad_to_multiple_of
										if return_token_type_ids is None:
																__magic_name__				:							Tuple				=      self.return_token_type_ids
										if return_attention_mask is None:
																__magic_name__				:							Any				=      self.return_attention_mask
										if not isinstance(_UpperCAmelCase     ,      tf.Tensor      ):
																__magic_name__				:							Dict				=      tf.convert_to_tensor(_UpperCAmelCase      )
										if text_pair is not None and not isinstance(_UpperCAmelCase     ,      tf.Tensor      ):
																__magic_name__				:							str				=      tf.convert_to_tensor(_UpperCAmelCase      )
										if text_pair is not None:
																if text.shape.rank > 1:
																						raise ValueError('text argument should not be multidimensional when a text pair is supplied!'      )
																if text_pair.shape.rank > 1:
																						raise ValueError('text_pair should not be multidimensional!'      )
										if text.shape.rank == 2:
																__magic_name__				:							Optional[int]				=      text[:, 0], text[:, 1]
										__magic_name__				:							str				=      self.unpaired_tokenize(_UpperCAmelCase      )
										if text_pair is None:  # Unpaired text
																if truncation:
																						__magic_name__				:							Optional[Any]				=      text[:, : max_length - 2]  # Allow room for special tokens
																__magic_name__				:							int				=      combine_segments(
																    (text,)     ,      start_of_sequence_id=self.cls_token_id     ,      end_of_segment_id=self.sep_token_id      )
										else:  # Paired text
																__magic_name__				:							Tuple				=      self.unpaired_tokenize(_UpperCAmelCase      )
																if truncation:
																						__magic_name__				:							Union[str, Any]				=      self.paired_trimmer.trim([text, text_pair]      )
																__magic_name__				:							int				=      combine_segments(
																    (text, text_pair)     ,      start_of_sequence_id=self.cls_token_id     ,      end_of_segment_id=self.sep_token_id      )
										if padding == "longest":
																__magic_name__				:							str				=      input_ids.bounding_shape(axis=1      )
																if pad_to_multiple_of is not None:
																						# No ceiling division in tensorflow, so we negate floordiv instead
																						__magic_name__				:							Union[str, Any]				=      pad_to_multiple_of * (-tf.math.floordiv(-pad_length     ,      _UpperCAmelCase      ))
										else:
																__magic_name__				:							Optional[Any]				=      max_length

										__magic_name__				:							Tuple				=      pad_model_inputs(_UpperCAmelCase     ,      max_seq_length=_UpperCAmelCase     ,      pad_value=self.pad_token_id      )
										__magic_name__				:							str				=      {'input_ids': input_ids}
										if return_attention_mask:
																__magic_name__				:							List[str]				=      attention_mask
										if return_token_type_ids:
																__magic_name__				:							Optional[Any]				=      pad_model_inputs(
																    _UpperCAmelCase     ,      max_seq_length=_UpperCAmelCase     ,      pad_value=self.pad_token_id      )
																__magic_name__				:							Tuple				=      token_type_ids
										return output





				def  UpperCamelCase_						(  self      ):




										"""simple docstring"""




										return {
										    "vocab_list": self.vocab_list,
										    "do_lower_case": self.do_lower_case,
										    "cls_token_id": self.cls_token_id,
										    "sep_token_id": self.sep_token_id,
										    "pad_token_id": self.pad_token_id,
										}






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


snake_case__     							= {
    'configuration_flaubert': ['FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'FlaubertConfig', 'FlaubertOnnxConfig'],
    'tokenization_flaubert': ['FlaubertTokenizer'],
}

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			snake_case__     							= [
			    'FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
			    'FlaubertForMultipleChoice',
			    'FlaubertForQuestionAnswering',
			    'FlaubertForQuestionAnsweringSimple',
			    'FlaubertForSequenceClassification',
			    'FlaubertForTokenClassification',
			    'FlaubertModel',
			    'FlaubertWithLMHeadModel',
			    'FlaubertPreTrainedModel',
			]

try:
			if not is_tf_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			snake_case__     							= [
			    'TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST',
			    'TFFlaubertForMultipleChoice',
			    'TFFlaubertForQuestionAnsweringSimple',
			    'TFFlaubertForSequenceClassification',
			    'TFFlaubertForTokenClassification',
			    'TFFlaubertModel',
			    'TFFlaubertPreTrainedModel',
			    'TFFlaubertWithLMHeadModel',
			]


if TYPE_CHECKING:
			from .configuration_flaubert import FLAUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, FlaubertConfig, FlaubertOnnxConfig
			from .tokenization_flaubert import FlaubertTokenizer

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_flaubert import (
						    FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
						    FlaubertForMultipleChoice,
						    FlaubertForQuestionAnswering,
						    FlaubertForQuestionAnsweringSimple,
						    FlaubertForSequenceClassification,
						    FlaubertForTokenClassification,
						    FlaubertModel,
						    FlaubertPreTrainedModel,
						    FlaubertWithLMHeadModel,
						)

			try:
						if not is_tf_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_tf_flaubert import (
						    TF_FLAUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
						    TFFlaubertForMultipleChoice,
						    TFFlaubertForQuestionAnsweringSimple,
						    TFFlaubertForSequenceClassification,
						    TFFlaubertForTokenClassification,
						    TFFlaubertModel,
						    TFFlaubertPreTrainedModel,
						    TFFlaubertWithLMHeadModel,
						)

else:
			import sys

			snake_case__     							= _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)


