import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
      from tokenizers import Tokenizer
      from tokenizers.models import WordPiece



class 						snake_case_ (	unittest.TestCase ):






   def    __UpperCamelCase  (   self       :  Tuple			,		lowercase_       :  int						)      ->		int:
       lowercase__				:     Optional[Any]					      =       list(tokenizer_class.max_model_input_sizes.keys()						)
       for model_name in sa_models[:1]:
           lowercase__				:     int					      =       tokenizer_class.from_pretrained(lowercase_						)
           self.assertIsNotNone(lowercase_						)
           self.assertIsInstance(lowercase_			,		lowercase_						)
           self.assertIsInstance(lowercase_			,		lowercase_						)

           for special_tok in tokenizer.all_special_tokens:
               self.assertIsInstance(lowercase_			,		lowercase_						)
               lowercase__				:     List[str]					      =       tokenizer.convert_tokens_to_ids(lowercase_						)
               self.assertIsInstance(lowercase_			,		lowercase_						)






   def    __UpperCamelCase  (   self       :  Dict			,		lowercase_       :  BatchEncoding			,		lowercase_       :  Optional[Callable] = None						)      ->		Optional[int]:
       lowercase__				:     Optional[int]					      =       pickle.dumps(lowercase_						)
       self.assertIsNotNone(lowercase_						)

       lowercase__				:     List[str]					      =       pickle.loads(lowercase_						)

       # Ensure is_fast is correctly restored
       self.assertEqual(be_restored.is_fast			,		be_original.is_fast						)

       # Ensure encodings are potentially correctly restored
       if be_original.is_fast:
           self.assertIsNotNone(be_restored.encodings						)
       else:
           self.assertIsNone(be_restored.encodings						)

       # Ensure the keys are the same
       for original_v, restored_v in zip(be_original.values()			,		be_restored.values()						):
           if equal_op:
               self.assertTrue(equal_op(lowercase_			,		lowercase_						)						)
           else:
               self.assertEqual(lowercase_			,		lowercase_						)






   @slow
   def    __UpperCamelCase  (   self       :  Dict						)      ->		Any:
       self.check_tokenizer_from_pretrained(lowercase_						)






   def    __UpperCamelCase  (   self       :  Dict						)      ->		Union[str, Any]:
       self.assertEqual(TensorType("tf"						)			,		TensorType.TENSORFLOW						)
       self.assertEqual(TensorType("pt"						)			,		TensorType.PYTORCH						)
       self.assertEqual(TensorType("np"						)			,		TensorType.NUMPY						)






   @require_tokenizers
   def    __UpperCamelCase  (   self       :  Optional[Any]						)      ->		str:
       import numpy as np

       lowercase__				:     Union[str, Any]					      =       BertTokenizer.from_pretrained("bert-base-cased"						)
       lowercase__				:     int					      =       BertTokenizerFast.from_pretrained("bert-base-cased"						)

       # Python no tensor
       with self.subTest("BatchEncoding (Python, return_tensors=None)"						):
           self.assert_dump_and_restore(tokenizer_p("Small example to encode"						)						)

       with self.subTest("BatchEncoding (Python, return_tensors=NUMPY)"						):
           self.assert_dump_and_restore(
               tokenizer_p("Small example to encode"			,		return_tensors=TensorType.NUMPY						)			,		np.array_equal						)

       with self.subTest("BatchEncoding (Rust, return_tensors=None)"						):
           self.assert_dump_and_restore(tokenizer_r("Small example to encode"						)						)

       with self.subTest("BatchEncoding (Rust, return_tensors=NUMPY)"						):
           self.assert_dump_and_restore(
               tokenizer_r("Small example to encode"			,		return_tensors=TensorType.NUMPY						)			,		np.array_equal						)






   @require_tf
   @require_tokenizers
   def    __UpperCamelCase  (   self       :  Dict						)      ->		Optional[int]:
       import tensorflow as tf

       def tf_array_equals(lowercase_       :  List[Any]			,		lowercase_       :  List[Any]						):
           return tf.reduce_all(tf.equal(lowercase_			,		lowercase_						)						)

       lowercase__				:     int					      =       BertTokenizer.from_pretrained("bert-base-cased"						)
       lowercase__				:     Tuple					      =       BertTokenizerFast.from_pretrained("bert-base-cased"						)

       with self.subTest("BatchEncoding (Python, return_tensors=TENSORFLOW)"						):
           self.assert_dump_and_restore(
               tokenizer_p("Small example to encode"			,		return_tensors=TensorType.TENSORFLOW						)			,		lowercase_						)

       with self.subTest("BatchEncoding (Rust, return_tensors=TENSORFLOW)"						):
           self.assert_dump_and_restore(
               tokenizer_r("Small example to encode"			,		return_tensors=TensorType.TENSORFLOW						)			,		lowercase_						)






   @require_torch
   @require_tokenizers
   def    __UpperCamelCase  (   self       :  Optional[Any]						)      ->		Optional[Any]:
       import torch

       lowercase__				:     Union[str, Any]					      =       BertTokenizer.from_pretrained("bert-base-cased"						)
       lowercase__				:     int					      =       BertTokenizerFast.from_pretrained("bert-base-cased"						)

       with self.subTest("BatchEncoding (Python, return_tensors=PYTORCH)"						):
           self.assert_dump_and_restore(
               tokenizer_p("Small example to encode"			,		return_tensors=TensorType.PYTORCH						)			,		torch.equal						)

       with self.subTest("BatchEncoding (Rust, return_tensors=PYTORCH)"						):
           self.assert_dump_and_restore(
               tokenizer_r("Small example to encode"			,		return_tensors=TensorType.PYTORCH						)			,		torch.equal						)






   @require_tokenizers
   def    __UpperCamelCase  (   self       :  Union[str, Any]						)      ->		Union[str, Any]:
       lowercase__				:     List[Any]					      =       BertTokenizer.from_pretrained("bert-base-cased"						)
       lowercase__				:     int					      =       BertTokenizerFast.from_pretrained("bert-base-cased"						)

       with self.subTest("Python Tokenizer"						):
           self.assertFalse(tokenizer_p("Small example to_encode"						).is_fast						)

       with self.subTest("Rust Tokenizer"						):
           self.assertTrue(tokenizer_r("Small example to_encode"						).is_fast						)






   @require_tokenizers
   def    __UpperCamelCase  (   self       :  Dict						)      ->		Dict:
       lowercase__				:     Optional[int]					      =       BertTokenizerFast.from_pretrained("bert-base-cased"						)
       lowercase__				:     Tuple					      =       tokenizer_r(["Test", "\xad", "test"]			,		is_split_into_words=lowercase_						)

       self.assertEqual(encoded.word_to_tokens(0						)			,		TokenSpan(start=1			,		end=2						)						)
       self.assertEqual(encoded.word_to_tokens(1						)			,		lowercase_						)
       self.assertEqual(encoded.word_to_tokens(2						)			,		TokenSpan(start=2			,		end=3						)						)






   def    __UpperCamelCase  (   self       :  Union[str, Any]						)      ->		Any:
       lowercase__				:     List[Any]					      =       BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}						)
       lowercase__				:     Any					      =       batch.convert_to_tensors(tensor_type="np"						)
       self.assertEqual(tensor_batch["inputs"].shape			,		(2, 3)						)
       self.assertEqual(tensor_batch["labels"].shape			,		(2,)						)
       # test converting the converted
       with CaptureStderr() as cs:
           lowercase__				:     Any					      =       batch.convert_to_tensors(tensor_type="np"						)
       self.assertFalse(len(cs.err						)			,		msg=F'''should have no warning, but got {cs.err}'''						)

       lowercase__				:     int					      =       BatchEncoding({"inputs": [1, 2, 3], "labels": 0}						)
       lowercase__				:     List[Any]					      =       batch.convert_to_tensors(tensor_type="np"			,		prepend_batch_axis=lowercase_						)
       self.assertEqual(tensor_batch["inputs"].shape			,		(1, 3)						)
       self.assertEqual(tensor_batch["labels"].shape			,		(1,)						)






   @require_torch
   def    __UpperCamelCase  (   self       :  Optional[int]						)      ->		str:
       lowercase__				:     Union[str, Any]					      =       BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}						)
       lowercase__				:     Optional[int]					      =       batch.convert_to_tensors(tensor_type="pt"						)
       self.assertEqual(tensor_batch["inputs"].shape			,		(2, 3)						)
       self.assertEqual(tensor_batch["labels"].shape			,		(2,)						)
       # test converting the converted
       with CaptureStderr() as cs:
           lowercase__				:     List[Any]					      =       batch.convert_to_tensors(tensor_type="pt"						)
       self.assertFalse(len(cs.err						)			,		msg=F'''should have no warning, but got {cs.err}'''						)

       lowercase__				:     Any					      =       BatchEncoding({"inputs": [1, 2, 3], "labels": 0}						)
       lowercase__				:     str					      =       batch.convert_to_tensors(tensor_type="pt"			,		prepend_batch_axis=lowercase_						)
       self.assertEqual(tensor_batch["inputs"].shape			,		(1, 3)						)
       self.assertEqual(tensor_batch["labels"].shape			,		(1,)						)






   @require_tf
   def    __UpperCamelCase  (   self       :  Tuple						)      ->		Optional[int]:
       lowercase__				:     Any					      =       BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}						)
       lowercase__				:     int					      =       batch.convert_to_tensors(tensor_type="tf"						)
       self.assertEqual(tensor_batch["inputs"].shape			,		(2, 3)						)
       self.assertEqual(tensor_batch["labels"].shape			,		(2,)						)
       # test converting the converted
       with CaptureStderr() as cs:
           lowercase__				:     Optional[int]					      =       batch.convert_to_tensors(tensor_type="tf"						)
       self.assertFalse(len(cs.err						)			,		msg=F'''should have no warning, but got {cs.err}'''						)

       lowercase__				:     Optional[int]					      =       BatchEncoding({"inputs": [1, 2, 3], "labels": 0}						)
       lowercase__				:     Tuple					      =       batch.convert_to_tensors(tensor_type="tf"			,		prepend_batch_axis=lowercase_						)
       self.assertEqual(tensor_batch["inputs"].shape			,		(1, 3)						)
       self.assertEqual(tensor_batch["labels"].shape			,		(1,)						)






   @require_flax
   def    __UpperCamelCase  (   self       :  List[str]						)      ->		Tuple:
       lowercase__				:     int					      =       BatchEncoding({"inputs": [[1, 2, 3], [4, 5, 6]], "labels": [0, 1]}						)
       lowercase__				:     Optional[Any]					      =       batch.convert_to_tensors(tensor_type="jax"						)
       self.assertEqual(tensor_batch["inputs"].shape			,		(2, 3)						)
       self.assertEqual(tensor_batch["labels"].shape			,		(2,)						)
       # test converting the converted
       with CaptureStderr() as cs:
           lowercase__				:     str					      =       batch.convert_to_tensors(tensor_type="jax"						)
       self.assertFalse(len(cs.err						)			,		msg=F'''should have no warning, but got {cs.err}'''						)

       lowercase__				:     Tuple					      =       BatchEncoding({"inputs": [1, 2, 3], "labels": 0}						)
       lowercase__				:     str					      =       batch.convert_to_tensors(tensor_type="jax"			,		prepend_batch_axis=lowercase_						)
       self.assertEqual(tensor_batch["inputs"].shape			,		(1, 3)						)
       self.assertEqual(tensor_batch["labels"].shape			,		(1,)						)






   def    __UpperCamelCase  (   self       :  Optional[int]						)      ->		Optional[int]:
       lowercase__				:     int					      =       [{"input_ids": np.array([0, 1, 2]						)}, {"input_ids": np.array([0, 1, 2, 3]						)}]
       lowercase__				:     Optional[int]					      =       BertTokenizer.from_pretrained("bert-base-cased"						)

       lowercase__				:     Any					      =       tokenizer.pad(lowercase_			,		padding=lowercase_						)
       self.assertTrue(isinstance(batch["input_ids"]			,		np.ndarray						)						)
       self.assertEqual(batch["input_ids"].tolist()			,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
       lowercase__				:     Any					      =       tokenizer.pad(lowercase_			,		padding=lowercase_			,		return_tensors="np"						)
       self.assertTrue(isinstance(batch["input_ids"]			,		np.ndarray						)						)
       self.assertEqual(batch["input_ids"].tolist()			,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)






   @require_torch
   def    __UpperCamelCase  (   self       :  int						)      ->		List[Any]:
       import torch

       lowercase__				:     Union[str, Any]					      =       [{"input_ids": torch.tensor([0, 1, 2]						)}, {"input_ids": torch.tensor([0, 1, 2, 3]						)}]
       lowercase__				:     List[Any]					      =       BertTokenizer.from_pretrained("bert-base-cased"						)

       lowercase__				:     str					      =       tokenizer.pad(lowercase_			,		padding=lowercase_						)
       self.assertTrue(isinstance(batch["input_ids"]			,		torch.Tensor						)						)
       self.assertEqual(batch["input_ids"].tolist()			,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
       lowercase__				:     Optional[int]					      =       tokenizer.pad(lowercase_			,		padding=lowercase_			,		return_tensors="pt"						)
       self.assertTrue(isinstance(batch["input_ids"]			,		torch.Tensor						)						)
       self.assertEqual(batch["input_ids"].tolist()			,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)






   @require_tf
   def    __UpperCamelCase  (   self       :  List[str]						)      ->		Optional[int]:
       import tensorflow as tf

       lowercase__				:     Optional[int]					      =       [{"input_ids": tf.constant([0, 1, 2]						)}, {"input_ids": tf.constant([0, 1, 2, 3]						)}]
       lowercase__				:     Dict					      =       BertTokenizer.from_pretrained("bert-base-cased"						)

       lowercase__				:     Optional[Any]					      =       tokenizer.pad(lowercase_			,		padding=lowercase_						)
       self.assertTrue(isinstance(batch["input_ids"]			,		tf.Tensor						)						)
       self.assertEqual(batch["input_ids"].numpy().tolist()			,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)
       lowercase__				:     int					      =       tokenizer.pad(lowercase_			,		padding=lowercase_			,		return_tensors="tf"						)
       self.assertTrue(isinstance(batch["input_ids"]			,		tf.Tensor						)						)
       self.assertEqual(batch["input_ids"].numpy().tolist()			,		[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]						)






   @require_tokenizers
   def    __UpperCamelCase  (   self       :  Tuple						)      ->		List[Any]:
       lowercase__				:     int					      =       Tokenizer(WordPiece(unk_token="[UNK]"						)						)
       PreTrainedTokenizerFast(tokenizer_object=lowercase_						)







   @require_tokenizers
   def    __UpperCamelCase  (   self       :  str						)      ->		List[Any]:
       lowercase__				:     Union[str, Any]					      =       Tokenizer(WordPiece(unk_token="[UNK]"						)						)
       with tempfile.TemporaryDirectory() as tmpdirname:
           bert_tokenizer.save(os.path.join(lowercase_			,		"tokenizer.json"						)						)
           PreTrainedTokenizerFast(tokenizer_file=os.path.join(lowercase_			,		"tokenizer.json"						)						)




import warnings
from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType



class 						snake_case_ (	__A ):
   __A			:	Optional[Any]        =       ["image_processor", "tokenizer"]
   __A			:	Tuple        =       "LayoutLMv2ImageProcessor"
   __A			:	Optional[int]        =       ("LayoutLMv2Tokenizer", "LayoutLMv2TokenizerFast")






   def __init__(   self       :  Tuple			,		lowercase_       :  List[str]=None			,		lowercase_       :  Dict=None			,		**lowercase_       :  List[str]						)      ->		int:
       lowercase__				:     Optional[Any]					      =       None
       if "feature_extractor" in kwargs:
           warnings.warn(
               "The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"
               " instead."			,		lowercase_			,		)
           lowercase__				:     Any					      =       kwargs.pop("feature_extractor"						)

       lowercase__				:     List[str]					      =       image_processor if image_processor is not None else feature_extractor
       if image_processor is None:
           raise ValueError("You need to specify an `image_processor`."						)
       if tokenizer is None:
           raise ValueError("You need to specify a `tokenizer`."						)

       super().__init__(lowercase_			,		lowercase_						)






   def __call__(   self       :  Union[str, Any]			,		lowercase_       :  Optional[Any]			,		lowercase_       :  Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None			,		lowercase_       :  Optional[Union[PreTokenizedInput, List[PreTokenizedInput]]] = None			,		lowercase_       :  Union[List[List[int]], List[List[List[int]]]] = None			,		lowercase_       :  Optional[Union[List[int], List[List[int]]]] = None			,		lowercase_       :  bool = True			,		lowercase_       :  Union[bool, str, PaddingStrategy] = False			,		lowercase_       :  Union[bool, str, TruncationStrategy] = False			,		lowercase_       :  Optional[int] = None			,		lowercase_       :  int = 0			,		lowercase_       :  Optional[int] = None			,		lowercase_       :  Optional[bool] = None			,		lowercase_       :  Optional[bool] = None			,		lowercase_       :  bool = False			,		lowercase_       :  bool = False			,		lowercase_       :  bool = False			,		lowercase_       :  bool = False			,		lowercase_       :  bool = True			,		lowercase_       :  Optional[Union[str, TensorType]] = None			,		**lowercase_       :  List[str]			,		)      ->		BatchEncoding:
       # verify input
       if self.image_processor.apply_ocr and (boxes is not None):
           raise ValueError(
               "You cannot provide bounding boxes if you initialized the image processor with apply_ocr set to True."						)

       if self.image_processor.apply_ocr and (word_labels is not None):
           raise ValueError(
               "You cannot provide word labels if you initialized the image processor with apply_ocr set to True."						)

       if return_overflowing_tokens is True and return_offsets_mapping is False:
           raise ValueError("You cannot return overflowing tokens without returning the offsets mapping."						)

       # first, apply the image processor
       lowercase__				:     List[Any]					      =       self.image_processor(images=lowercase_			,		return_tensors=lowercase_						)

       # second, apply the tokenizer
       if text is not None and self.image_processor.apply_ocr and text_pair is None:
           if isinstance(lowercase_			,		lowercase_						):
               lowercase__				:     List[Any]					      =       [text]  # add batch dimension (as the image processor always adds a batch dimension)
           lowercase__				:     Union[str, Any]					      =       features["words"]

       lowercase__				:     List[str]					      =       self.tokenizer(
           text=text if text is not None else features["words"]			,		text_pair=text_pair if text_pair is not None else None			,		boxes=boxes if boxes is not None else features["boxes"]			,		word_labels=lowercase_			,		add_special_tokens=lowercase_			,		padding=lowercase_			,		truncation=lowercase_			,		max_length=lowercase_			,		stride=lowercase_			,		pad_to_multiple_of=lowercase_			,		return_token_type_ids=lowercase_			,		return_attention_mask=lowercase_			,		return_overflowing_tokens=lowercase_			,		return_special_tokens_mask=lowercase_			,		return_offsets_mapping=lowercase_			,		return_length=lowercase_			,		verbose=lowercase_			,		return_tensors=lowercase_			,		**lowercase_			,		)

       # add pixel values
       lowercase__				:     Optional[int]					      =       features.pop("pixel_values"						)
       if return_overflowing_tokens is True:
           lowercase__				:     Tuple					      =       self.get_overflowing_images(lowercase_			,		encoded_inputs["overflow_to_sample_mapping"]						)
       lowercase__				:     str					      =       images

       return encoded_inputs






   def    __UpperCamelCase  (   self       :  Tuple			,		lowercase_       :  Optional[int]			,		lowercase_       :  Union[str, Any]						)      ->		Any:
       # in case there's an overflow, ensure each `input_ids` sample is mapped to its corresponding image
       lowercase__				:     str					      =       []
       for sample_idx in overflow_to_sample_mapping:
           images_with_overflow.append(images[sample_idx]						)

       if len(lowercase_						) != len(lowercase_						):
           raise ValueError(
               "Expected length of images to be the same as the length of `overflow_to_sample_mapping`, but got"
               F''' {len(lowercase_						)} and {len(lowercase_						)}'''						)

       return images_with_overflow






   def    __UpperCamelCase  (   self       :  Any			,		*lowercase_       :  Tuple			,		**lowercase_       :  List[Any]						)      ->		Union[str, Any]:
       return self.tokenizer.batch_decode(*lowercase_			,		**lowercase_						)






   def    __UpperCamelCase  (   self       :  List[Any]			,		*lowercase_       :  Tuple			,		**lowercase_       :  Union[str, Any]						)      ->		List[Any]:
       return self.tokenizer.decode(*lowercase_			,		**lowercase_						)






   @property
   def    __UpperCamelCase  (   self       :  int						)      ->		Optional[Any]:
       return ["input_ids", "bbox", "token_type_ids", "attention_mask", "image"]






   @property
   def    __UpperCamelCase  (   self       :  Optional[int]						)      ->		Tuple:
       warnings.warn(
           "`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."			,		lowercase_			,		)
       return self.image_processor_class







   @property
   def    __UpperCamelCase  (   self       :  Dict						)      ->		List[Any]:
       warnings.warn(
           "`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."			,		lowercase_			,		)
       return self.image_processor



