'''simple docstring'''




import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
					from tokenizers import Tokenizer
					from tokenizers.models import WordPiece

class   _lowercase(     unittest.TestCase				):



			"""simple docstring"""





			def 			snake_case	(							self:    int			,a:    Dict	):
				__UpperCAmelCase         =					list(tokenizer_class.max_model_input_sizes.keys()	)
				for model_name in sa_models[:1]:
					__UpperCAmelCase         =					tokenizer_class.from_pretrained(a	)
					self.assertIsNotNone(a	)
					self.assertIsInstance(a			,a	)
					self.assertIsInstance(a			,a	)

					for special_tok in tokenizer.all_special_tokens:
						self.assertIsInstance(a			,a	)
						__UpperCAmelCase         =					tokenizer.convert_tokens_to_ids(a	)
						self.assertIsInstance(a			,a	)





			def 			snake_case	(							self:    Any			,a:    BatchEncoding			,a:    Optional[Callable] = None	):
				__UpperCAmelCase         =					pickle.dumps(a	)
				self.assertIsNotNone(a	)

				__UpperCAmelCase         =					pickle.loads(a	)

				# Ensure is_fast is correctly restored
				self.assertEqual(be_restored.is_fast			,be_original.is_fast	)

				# Ensure encodings are potentially correctly restored
				if be_original.is_fast:
					self.assertIsNotNone(be_restored.encodings	)
				else:
					self.assertIsNone(be_restored.encodings	)

				# Ensure the keys are the same
				for original_v, restored_v in zip(be_original.values()			,be_restored.values()	):
					if equal_op:
						self.assertTrue(equal_op(a			,a	)	)
					else:
						self.assertEqual(a			,a	)





			@slow
			def 			snake_case	(							self:    Dict	):
				self.check_tokenizer_from_pretrained(a	)





			def 			snake_case	(							self:    Optional[Any]	):
				self.assertEqual(TensorType('tf'	)			,TensorType.TENSORFLOW	)
				self.assertEqual(TensorType('pt'	)			,TensorType.PYTORCH	)
				self.assertEqual(TensorType('np'	)			,TensorType.NUMPY	)





			@require_tokenizers
			def 			snake_case	(							self:    Optional[Any]	):
				import numpy as np

				__UpperCAmelCase         =					BertTokenizer.from_pretrained('bert-base-cased'	)
				__UpperCAmelCase         =					BertTokenizerFast.from_pretrained('bert-base-cased'	)

				# Python no tensor
				with self.subTest('BatchEncoding (Python, return_tensors=None)'	):
					self.assert_dump_and_restore(tokenizer_p('Small example to encode'	)	)

				with self.subTest('BatchEncoding (Python, return_tensors=NUMPY)'	):
					self.assert_dump_and_restore(
					    tokenizer_p('Small example to encode'			,return_tensors=TensorType.NUMPY	)			,np.array_equal	)

				with self.subTest('BatchEncoding (Rust, return_tensors=None)'	):
					self.assert_dump_and_restore(tokenizer_r('Small example to encode'	)	)

				with self.subTest('BatchEncoding (Rust, return_tensors=NUMPY)'	):
					self.assert_dump_and_restore(
					    tokenizer_r('Small example to encode'			,return_tensors=TensorType.NUMPY	)			,np.array_equal	)





			@require_tf
			@require_tokenizers
			def 			snake_case	(							self:    Optional[Any]	):
				import tensorflow as tf

				def tf_array_equals(a:    Union[str, Any]			,a:    Any	):
					return tf.reduce_all(tf.equal(a			,a	)	)

				__UpperCAmelCase         =					BertTokenizer.from_pretrained('bert-base-cased'	)
				__UpperCAmelCase         =					BertTokenizerFast.from_pretrained('bert-base-cased'	)

				with self.subTest('BatchEncoding (Python, return_tensors=TENSORFLOW)'	):
					self.assert_dump_and_restore(
					    tokenizer_p('Small example to encode'			,return_tensors=TensorType.TENSORFLOW	)			,a	)

				with self.subTest('BatchEncoding (Rust, return_tensors=TENSORFLOW)'	):
					self.assert_dump_and_restore(
					    tokenizer_r('Small example to encode'			,return_tensors=TensorType.TENSORFLOW	)			,a	)





			@require_torch
			@require_tokenizers
			def 			snake_case	(							self:    Optional[Any]	):
				import torch

				__UpperCAmelCase         =					BertTokenizer.from_pretrained('bert-base-cased'	)
				__UpperCAmelCase         =					BertTokenizerFast.from_pretrained('bert-base-cased'	)

				with self.subTest('BatchEncoding (Python, return_tensors=PYTORCH)'	):
					self.assert_dump_and_restore(
					    tokenizer_p('Small example to encode'			,return_tensors=TensorType.PYTORCH	)			,torch.equal	)

				with self.subTest('BatchEncoding (Rust, return_tensors=PYTORCH)'	):
					self.assert_dump_and_restore(
					    tokenizer_r('Small example to encode'			,return_tensors=TensorType.PYTORCH	)			,torch.equal	)





			@require_tokenizers
			def 			snake_case	(							self:    Tuple	):
				__UpperCAmelCase         =					BertTokenizer.from_pretrained('bert-base-cased'	)
				__UpperCAmelCase         =					BertTokenizerFast.from_pretrained('bert-base-cased'	)

				with self.subTest('Python Tokenizer'	):
					self.assertFalse(tokenizer_p('Small example to_encode'	).is_fast	)

				with self.subTest('Rust Tokenizer'	):
					self.assertTrue(tokenizer_r('Small example to_encode'	).is_fast	)





			@require_tokenizers
			def 			snake_case	(							self:    Optional[int]	):
				__UpperCAmelCase         =					BertTokenizerFast.from_pretrained('bert-base-cased'	)
				__UpperCAmelCase         =					tokenizer_r(['Test', '\xad', 'test']			,is_split_into_words=a	)

				self.assertEqual(encoded.word_to_tokens(0	)			,TokenSpan(start=1			,end=2	)	)
				self.assertEqual(encoded.word_to_tokens(1	)			,a	)
				self.assertEqual(encoded.word_to_tokens(2	)			,TokenSpan(start=2			,end=3	)	)





			def 			snake_case	(							self:    Optional[Any]	):
				__UpperCAmelCase         =					BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}	)
				__UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='np'	)
				self.assertEqual(tensor_batch['inputs'].shape			,(2, 3)	)
				self.assertEqual(tensor_batch['labels'].shape			,(2,)	)
				# test converting the converted
				with CaptureStderr() as cs:
					__UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='np'	)
				self.assertFalse(len(cs.err	)			,msg=f"""should have no warning, but got {cs.err}"""	)

				__UpperCAmelCase         =					BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}	)
				__UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='np'			,prepend_batch_axis=a	)
				self.assertEqual(tensor_batch['inputs'].shape			,(1, 3)	)
				self.assertEqual(tensor_batch['labels'].shape			,(1,)	)





			@require_torch
			def 			snake_case	(							self:    Optional[Any]	):
				__UpperCAmelCase         =					BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}	)
				__UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='pt'	)
				self.assertEqual(tensor_batch['inputs'].shape			,(2, 3)	)
				self.assertEqual(tensor_batch['labels'].shape			,(2,)	)
				# test converting the converted
				with CaptureStderr() as cs:
					__UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='pt'	)
				self.assertFalse(len(cs.err	)			,msg=f"""should have no warning, but got {cs.err}"""	)

				__UpperCAmelCase         =					BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}	)
				__UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='pt'			,prepend_batch_axis=a	)
				self.assertEqual(tensor_batch['inputs'].shape			,(1, 3)	)
				self.assertEqual(tensor_batch['labels'].shape			,(1,)	)





			@require_tf
			def 			snake_case	(							self:    List[Any]	):
				__UpperCAmelCase         =					BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}	)
				__UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='tf'	)
				self.assertEqual(tensor_batch['inputs'].shape			,(2, 3)	)
				self.assertEqual(tensor_batch['labels'].shape			,(2,)	)
				# test converting the converted
				with CaptureStderr() as cs:
					__UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='tf'	)
				self.assertFalse(len(cs.err	)			,msg=f"""should have no warning, but got {cs.err}"""	)

				__UpperCAmelCase         =					BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}	)
				__UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='tf'			,prepend_batch_axis=a	)
				self.assertEqual(tensor_batch['inputs'].shape			,(1, 3)	)
				self.assertEqual(tensor_batch['labels'].shape			,(1,)	)





			@require_flax
			def 			snake_case	(							self:    Any	):
				__UpperCAmelCase         =					BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}	)
				__UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='jax'	)
				self.assertEqual(tensor_batch['inputs'].shape			,(2, 3)	)
				self.assertEqual(tensor_batch['labels'].shape			,(2,)	)
				# test converting the converted
				with CaptureStderr() as cs:
					__UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='jax'	)
				self.assertFalse(len(cs.err	)			,msg=f"""should have no warning, but got {cs.err}"""	)

				__UpperCAmelCase         =					BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}	)
				__UpperCAmelCase         =					batch.convert_to_tensors(tensor_type='jax'			,prepend_batch_axis=a	)
				self.assertEqual(tensor_batch['inputs'].shape			,(1, 3)	)
				self.assertEqual(tensor_batch['labels'].shape			,(1,)	)





			def 			snake_case	(							self:    List[Any]	):
				__UpperCAmelCase         =					[{'input_ids': np.array([0, 1, 2]	)}, {'input_ids': np.array([0, 1, 2, 3]	)}]
				__UpperCAmelCase         =					BertTokenizer.from_pretrained('bert-base-cased'	)

				__UpperCAmelCase         =					tokenizer.pad(a			,padding=a	)
				self.assertTrue(isinstance(batch['input_ids']			,np.ndarray	)	)
				self.assertEqual(batch['input_ids'].tolist()			,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)
				__UpperCAmelCase         =					tokenizer.pad(a			,padding=a			,return_tensors='np'	)
				self.assertTrue(isinstance(batch['input_ids']			,np.ndarray	)	)
				self.assertEqual(batch['input_ids'].tolist()			,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)





			@require_torch
			def 			snake_case	(							self:    Optional[Any]	):
				import torch

				__UpperCAmelCase         =					[{'input_ids': torch.tensor([0, 1, 2]	)}, {'input_ids': torch.tensor([0, 1, 2, 3]	)}]
				__UpperCAmelCase         =					BertTokenizer.from_pretrained('bert-base-cased'	)

				__UpperCAmelCase         =					tokenizer.pad(a			,padding=a	)
				self.assertTrue(isinstance(batch['input_ids']			,torch.Tensor	)	)
				self.assertEqual(batch['input_ids'].tolist()			,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)
				__UpperCAmelCase         =					tokenizer.pad(a			,padding=a			,return_tensors='pt'	)
				self.assertTrue(isinstance(batch['input_ids']			,torch.Tensor	)	)
				self.assertEqual(batch['input_ids'].tolist()			,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)





			@require_tf
			def 			snake_case	(							self:    Union[str, Any]	):
				import tensorflow as tf

				__UpperCAmelCase         =					[{'input_ids': tf.constant([0, 1, 2]	)}, {'input_ids': tf.constant([0, 1, 2, 3]	)}]
				__UpperCAmelCase         =					BertTokenizer.from_pretrained('bert-base-cased'	)

				__UpperCAmelCase         =					tokenizer.pad(a			,padding=a	)
				self.assertTrue(isinstance(batch['input_ids']			,tf.Tensor	)	)
				self.assertEqual(batch['input_ids'].numpy().tolist()			,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)
				__UpperCAmelCase         =					tokenizer.pad(a			,padding=a			,return_tensors='tf'	)
				self.assertTrue(isinstance(batch['input_ids']			,tf.Tensor	)	)
				self.assertEqual(batch['input_ids'].numpy().tolist()			,[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]	)





			@require_tokenizers
			def 			snake_case	(							self:    List[str]	):
				__UpperCAmelCase         =					Tokenizer(WordPiece(unk_token='[UNK]'	)	)
				PreTrainedTokenizerFast(tokenizer_object=a	)





			@require_tokenizers
			def 			snake_case	(							self:    str	):
				__UpperCAmelCase         =					Tokenizer(WordPiece(unk_token='[UNK]'	)	)
				with tempfile.TemporaryDirectory() as tmpdirname:
					bert_tokenizer.save(os.path.join(a			,'tokenizer.json'	)	)
					PreTrainedTokenizerFast(tokenizer_file=os.path.join(a			,'tokenizer.json'	)	)




'''simple docstring'''




import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


_UpperCamelCase				:		int       			=		logging.get_logger(__name__)

_UpperCamelCase				:		Tuple       			=		{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

_UpperCamelCase				:		int       			=		{
    'vocab_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
    },
    'tokenizer_file': {
        'allenai/longformer-base-4096': (
            'https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
        ),
    },
}

_UpperCamelCase				:		List[str]       			=		{
    'allenai/longformer-base-4096': 40_96,
    'allenai/longformer-large-4096': 40_96,
    'allenai/longformer-large-4096-finetuned-triviaqa': 40_96,
    'allenai/longformer-base-4096-extra.pos.embd.only': 40_96,
    'allenai/longformer-large-4096-extra.pos.embd.only': 40_96,
}

class   _lowercase(     _lowerCamelCase				):



   """simple docstring"""





   __lowerCamelCase    =  VOCAB_FILES_NAMES
   __lowerCamelCase    =  PRETRAINED_VOCAB_FILES_MAP
   __lowerCamelCase    =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __lowerCamelCase    =  ['''input_ids''', '''attention_mask''']
   __lowerCamelCase    =  LongformerTokenizer





   def __init__(							self:    Any			,a:    Optional[int]=None			,a:    Optional[int]=None			,a:    int=None			,a:    Any="replace"			,a:    Optional[int]="<s>"			,a:    Any="</s>"			,a:    Optional[int]="</s>"			,a:    Dict="<s>"			,a:    Dict="<unk>"			,a:    int="<pad>"			,a:    List[str]="<mask>"			,a:    Union[str, Any]=False			,a:    Any=True			,**a:    str			,):
    super().__init__(
        a			,a			,tokenizer_file=a			,errors=a			,bos_token=a			,eos_token=a			,sep_token=a			,cls_token=a			,unk_token=a			,pad_token=a			,mask_token=a			,add_prefix_space=a			,trim_offsets=a			,**a			,)

    __UpperCAmelCase         =					json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()	)
    if pre_tok_state.get('add_prefix_space'			,a	) != add_prefix_space:
     __UpperCAmelCase         =					getattr(a			,pre_tok_state.pop('type'	)	)
     __UpperCAmelCase         =					add_prefix_space
     __UpperCAmelCase         =					pre_tok_class(**a	)

    __UpperCAmelCase         =					add_prefix_space

    __UpperCAmelCase         =					'post_processor'
    __UpperCAmelCase         =					getattr(self.backend_tokenizer			,a			,a	)
    if tokenizer_component_instance:
     __UpperCAmelCase         =					json.loads(tokenizer_component_instance.__getstate__()	)

     # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
     if "sep" in state:
      __UpperCAmelCase         =					tuple(state['sep']	)
     if "cls" in state:
      __UpperCAmelCase         =					tuple(state['cls']	)

     __UpperCAmelCase         =					False

     if state.get('add_prefix_space'			,a	) != add_prefix_space:
      __UpperCAmelCase         =					add_prefix_space
      __UpperCAmelCase         =					True

     if state.get('trim_offsets'			,a	) != trim_offsets:
      __UpperCAmelCase         =					trim_offsets
      __UpperCAmelCase         =					True

     if changes_to_apply:
      __UpperCAmelCase         =					getattr(a			,state.pop('type'	)	)
      __UpperCAmelCase         =					component_class(**a	)
      setattr(self.backend_tokenizer			,a			,a	)





   @property
   def 			snake_case	(							self:    List[Any]	):
    if self._mask_token is None:
     if self.verbose:
      logger.error('Using mask_token, but it is not set yet.'	)
     return None
    return str(self._mask_token	)





   @mask_token.setter
   def 			snake_case	(							self:    int			,a:    Optional[int]	):
    __UpperCAmelCase         =					AddedToken(a			,lstrip=a			,rstrip=a	) if isinstance(a			,a	) else value
    __UpperCAmelCase         =					value





   def 			snake_case	(							self:    Optional[int]			,*a:    Dict			,**a:    Dict	):
    __UpperCAmelCase         =					kwargs.get('is_split_into_words'			,a	)
    assert self.add_prefix_space or not is_split_into_words, (
        f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
        "to use it with pretokenized inputs."
    )

    return super()._batch_encode_plus(*a			,**a	)





   def 			snake_case	(							self:    str			,*a:    Dict			,**a:    str	):
    __UpperCAmelCase         =					kwargs.get('is_split_into_words'			,a	)

    assert self.add_prefix_space or not is_split_into_words, (
        f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
        "to use it with pretokenized inputs."
    )

    return super()._encode_plus(*a			,**a	)





   def 			snake_case	(							self:    List[str]			,a:    str			,a:    Optional[str] = None	):
    __UpperCAmelCase         =					self._tokenizer.model.save(a			,name=a	)
    return tuple(a	)





   def 			snake_case	(							self:    Tuple			,a:    Dict			,a:    Tuple=None	):
    __UpperCAmelCase         =					[self.bos_token_id] + token_ids_a + [self.eos_token_id]
    if token_ids_a is None:
     return output

    return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]





   def 			snake_case	(							self:    int			,a:    List[int]			,a:    Optional[List[int]] = None	):
    __UpperCAmelCase         =					[self.sep_token_id]
    __UpperCAmelCase         =					[self.cls_token_id]

    if token_ids_a is None:
     return len(cls + token_ids_a + sep	) * [0]
    return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]



