

import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class      _a				( snake_case_    ,							unittest.TestCase							):




      """simple docstring"""

      _lowerCamelCase					: str					=					PreTrainedTokenizerFast
      _lowerCamelCase					: int					=					False
      _lowerCamelCase					: Optional[Any]					=					True
      _lowerCamelCase					: Tuple					=					'tokenizer_file'





      def 		__A  (      self       :							List[str]  ):
            A_	     =							False  # because we don't have pretrained_vocab_files_map
            super().setUp()
            A_	     =							True

            A_	     =							["robot-test/dummy-tokenizer-fast", "robot-test/dummy-tokenizer-wordlevel"]
            A_	     =							"SaulLu/dummy-tokenizer-bytelevel-bpe"

            # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
            A_	     =							[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

            A_	     =							PreTrainedTokenizerFast.from_pretrained(model_paths[0]  )
            tokenizer.save_pretrained(self.tmpdirname  )





      def 		__A  (      self       :							Optional[Any]  ):
            # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
            # model
            pass





      def 		__A  (      self       :							Optional[int]  ):
            # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
            # model
            pass





      def 		__A  (      self       :							List[Any]  ):
            # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
            # model
            pass





      def 		__A  (      self       :							str  ):
            # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
            pass





      def 		__A  (      self       :							List[str]  ):
            A_	     =							self.tmpdirname
            # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
                        try:
                              A_	     =							tempfile.mkdtemp()
                              A_	     =							self.rust_tokenizer_class.from_pretrained(UpperCAmelCase	, **UpperCAmelCase  )

                              tokenizer.save_pretrained(self.tmpdirname  )
                              super().test_training_new_tokenizer()
                        finally:
                              # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                              # is restored
                              shutil.rmtree(self.tmpdirname  )
                              A_	     =							tmpdirname_orig





      def 		__A  (      self       :							Union[str, Any]  ):
            A_	     =							self.tmpdirname
            # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                  with self.subTest(f'''{tokenizer.__class__.__name__} ({pretrained_name})'''  ):
                        try:
                              A_	     =							tempfile.mkdtemp()
                              A_	     =							self.rust_tokenizer_class.from_pretrained(UpperCAmelCase	, **UpperCAmelCase  )

                              tokenizer.save_pretrained(self.tmpdirname  )
                              super().test_training_new_tokenizer_with_special_tokens_change()
                        finally:
                              # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                              # is restored
                              shutil.rmtree(self.tmpdirname  )
                              A_	     =							tmpdirname_orig







      def 		__A  (      self       :							List[Any]  ):
            A_	     =							self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name  )

            A_	     =							("a" for _ in range(1000  ))
            A_	     =							tokenizer.train_new_from_iterator(text_iterator=UpperCAmelCase	, length=1000	, vocab_size=50  )

            A_	     =							new_tokenizer.encode("a🤗"  )
            self.assertEqual(UpperCAmelCase	, [64, 172, 253, 97, 245]  )




@require_tokenizers
class      _a				( unittest.TestCase							):




      """simple docstring"""

      def 		__A  (      self       :							Any  ):
            A_	     =							AutoTokenizer.from_pretrained("bert-base-cased"  )
            A_	     =							json.loads(tokenizer._tokenizer.to_str()  )
            A_	     =							len(UpperCAmelCase  )

            with tempfile.TemporaryDirectory() as tmp_dir:
                  # Hack to save this in the tokenizer_config.json
                  A_	     =							["tokenizer.4.0.0.json"]
                  tokenizer.save_pretrained(UpperCAmelCase  )
                  json.dump(UpperCAmelCase	, open(os.path.join(UpperCAmelCase	, "tokenizer.4.0.0.json"  )	, "w"  )  )

                  # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
                  A_	     =							AutoTokenizer.from_pretrained(UpperCAmelCase  )
                  self.assertEqual(len(UpperCAmelCase  )	, len(UpperCAmelCase  ) + 1  )
                  A_	     =							json.loads(new_tokenizer._tokenizer.to_str()  )
                  self.assertIn("huggingface"	, json_tokenizer["model"]["vocab"]  )

                  # Will need to be adjusted if we reach v42 and this test is still here.
                  # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
                  shutil.move(os.path.join(UpperCAmelCase	, "tokenizer.4.0.0.json"  )	, os.path.join(UpperCAmelCase	, "tokenizer.42.0.0.json"  )  )
                  A_	     =							["tokenizer.42.0.0.json"]
                  tokenizer.save_pretrained(UpperCAmelCase  )
                  A_	     =							AutoTokenizer.from_pretrained(UpperCAmelCase  )
                  self.assertEqual(len(UpperCAmelCase  )	, len(UpperCAmelCase  )  )
                  A_	     =							json.loads(new_tokenizer._tokenizer.to_str()  )
                  self.assertNotIn("huggingface"	, json_tokenizer["model"]["vocab"]  )







      def 		__A  (      self       :							Any  ):
            # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
            A_	     =							"hf-internal-testing/test-two-tokenizers"

            # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
            A_	     =							AutoTokenizer.from_pretrained(UpperCAmelCase  )
            self.assertEqual(len(UpperCAmelCase  )	, 28997  )
            A_	     =							json.loads(tokenizer._tokenizer.to_str()  )
            self.assertIn("huggingface"	, json_tokenizer["model"]["vocab"]  )

            # Testing an older version by monkey-patching the version in the module it's used.
            import transformers as old_transformers

            A_	     =							"3.0.0"
            A_	     =							old_transformers.models.auto.AutoTokenizer.from_pretrained(UpperCAmelCase  )
            self.assertEqual(len(UpperCAmelCase  )	, 28996  )
            A_	     =							json.loads(old_tokenizer._tokenizer.to_str()  )
            self.assertNotIn("huggingface"	, json_tokenizer["model"]["vocab"]  )




@require_tokenizers
class      _a				( unittest.TestCase							):




      """simple docstring"""

      def 		__A  (      self       :							Union[str, Any]  ):
            # See https://github.com/huggingface/transformers/pull/12550
            # and https://github.com/huggingface/tokenizers/issues/537
            A_	     =							PreTrainedTokenizerFast.from_pretrained("robot-test/dummy-tokenizer-wordlevel"  )
            A_	     =							"The Matrix is a 1999 science fiction action film."

            with concurrent.futures.ThreadPoolExecutor() as executor:
                  A_	     =							[executor.submit(self.fetch	, UpperCAmelCase	, UpperCAmelCase  ) for i in range(10  )]
                  A_	     =							[future.result() for future in futures]
                  self.assertEqual(UpperCAmelCase	, [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10  )]  )







      def 		__A  (      self       :							str	, UpperCAmelCase       :							str	, UpperCAmelCase       :							Dict  ):
            return tokenizer.encode(UpperCAmelCase	, truncation="longest_first"	, padding="longest"  )


import os
from shutil import copyfile
from typing import Optional, Tuple

from ...file_utils import is_sentencepiece_available
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if is_sentencepiece_available():
      from .tokenization_deberta_va import DebertaVaTokenizer
else:
      __a						:int            = None

__a						:List[str]            = logging.get_logger(__name__)

__a						:str            = {'vocab_file': 'spm.model', 'tokenizer_file': 'tokenizer.json'}

__a						:List[Any]            = {
    'vocab_file': {
        'microsoft/deberta-v2-xlarge': 'https://huggingface.co/microsoft/deberta-v2-xlarge/resolve/main/spm.model',
        'microsoft/deberta-v2-xxlarge': 'https://huggingface.co/microsoft/deberta-v2-xxlarge/resolve/main/spm.model',
        'microsoft/deberta-v2-xlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-v2-xlarge-mnli/resolve/main/spm.model'
        ),
        'microsoft/deberta-v2-xxlarge-mnli': (
            'https://huggingface.co/microsoft/deberta-v2-xxlarge-mnli/resolve/main/spm.model'
        ),
    }
}

__a						:Dict            = {
    'microsoft/deberta-v2-xlarge': 512,
    'microsoft/deberta-v2-xxlarge': 512,
    'microsoft/deberta-v2-xlarge-mnli': 512,
    'microsoft/deberta-v2-xxlarge-mnli': 512,
}

__a						:List[Any]            = {
    'microsoft/deberta-v2-xlarge': {'do_lower_case': False},
    'microsoft/deberta-v2-xxlarge': {'do_lower_case': False},
    'microsoft/deberta-v2-xlarge-mnli': {'do_lower_case': False},
    'microsoft/deberta-v2-xxlarge-mnli': {'do_lower_case': False},
}

class      _a				( snake_case_							):




      """simple docstring"""

      _lowerCamelCase					: Dict					=					VOCAB_FILES_NAMES
      _lowerCamelCase					: Optional[Any]					=					PRETRAINED_VOCAB_FILES_MAP
      _lowerCamelCase					: Tuple					=					PRETRAINED_INIT_CONFIGURATION
      _lowerCamelCase					: Tuple					=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _lowerCamelCase					: Optional[int]					=					DebertaVaTokenizer





      def __init__(      self       :							List[Any]	, UpperCAmelCase       :							str=None	, UpperCAmelCase       :							int=None	, UpperCAmelCase       :							List[str]=False	, UpperCAmelCase       :							Optional[Any]=False	, UpperCAmelCase       :							List[str]="[CLS]"	, UpperCAmelCase       :							str="[SEP]"	, UpperCAmelCase       :							List[str]="[UNK]"	, UpperCAmelCase       :							Tuple="[SEP]"	, UpperCAmelCase       :							Union[str, Any]="[PAD]"	, UpperCAmelCase       :							Union[str, Any]="[CLS]"	, UpperCAmelCase       :							Any="[MASK]"	, **UpperCAmelCase       :							Dict	, ):
            super().__init__(
                UpperCAmelCase	, tokenizer_file=UpperCAmelCase	, do_lower_case=UpperCAmelCase	, bos_token=UpperCAmelCase	, eos_token=UpperCAmelCase	, unk_token=UpperCAmelCase	, sep_token=UpperCAmelCase	, pad_token=UpperCAmelCase	, cls_token=UpperCAmelCase	, mask_token=UpperCAmelCase	, split_by_punct=UpperCAmelCase	, **UpperCAmelCase	, )

            A_	     =							do_lower_case
            A_	     =							split_by_punct
            A_	     =							vocab_file
            A_	     =							False if not self.vocab_file else True





      def 		__A  (      self       :							Optional[int]	, UpperCAmelCase       :							Tuple	, UpperCAmelCase       :							List[str]=None  ):

            if token_ids_a is None:
                  return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
            A_	     =							[self.cls_token_id]
            A_	     =							[self.sep_token_id]
            return cls + token_ids_a + sep + token_ids_a + sep





      def 		__A  (      self       :							List[Any]	, UpperCAmelCase       :							Any	, UpperCAmelCase       :							Optional[int]=None	, UpperCAmelCase       :							int=False  ):

            if already_has_special_tokens:
                  return super().get_special_tokens_mask(
                      token_ids_a=UpperCAmelCase	, token_ids_a=UpperCAmelCase	, already_has_special_tokens=UpperCAmelCase  )

            if token_ids_a is not None:
                  return [1] + ([0] * len(UpperCAmelCase  )) + [1] + ([0] * len(UpperCAmelCase  )) + [1]
            return [1] + ([0] * len(UpperCAmelCase  )) + [1]





      def 		__A  (      self       :							List[Any]	, UpperCAmelCase       :							Union[str, Any]	, UpperCAmelCase       :							Optional[int]=None  ):
            A_	     =							[self.sep_token_id]
            A_	     =							[self.cls_token_id]
            if token_ids_a is None:
                  return len(cls + token_ids_a + sep  ) * [0]
            return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]







      def 		__A  (      self       :							str	, UpperCAmelCase       :							str	, UpperCAmelCase       :							Optional[str] = None  ):
            if not self.can_save_slow_tokenizer:
                  raise ValueError(
                      "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
                      "tokenizer."  )

            if not os.path.isdir(UpperCAmelCase  ):
                  logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''  )
                  return
            A_	     =							os.path.join(
                UpperCAmelCase	, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]  )

            if os.path.abspath(self.vocab_file  ) != os.path.abspath(UpperCAmelCase  ):
                  copyfile(self.vocab_file	, UpperCAmelCase  )

            return (out_vocab_file,)