


'''simple docstring'''




import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class  _lowercase   (  __lowercase	,				unittest.TestCase       ):
   _SCREAMING_SNAKE_CASE     :     int									=  PreTrainedTokenizerFast
   _SCREAMING_SNAKE_CASE     :     List[str]									=  False
   _SCREAMING_SNAKE_CASE     :     str									=  True
   _SCREAMING_SNAKE_CASE     :     Optional[Any]									=  "tokenizer_file"
   def a     (	self					:			Any			)   ->       int:
        __snake_case    			=	False  # because we don't have pretrained_vocab_files_map
        super().setUp()
        __snake_case    			=	True

        __snake_case    			=	['robot-test/dummy-tokenizer-fast', 'robot-test/dummy-tokenizer-wordlevel']
        __snake_case    			=	'SaulLu/dummy-tokenizer-bytelevel-bpe'

        # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
        __snake_case    			=	[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

        __snake_case    			=	PreTrainedTokenizerFast.from_pretrained(model_paths[0]			)
        tokenizer.save_pretrained(self.tmpdirname			)
   def a     (	self					:			Dict			)   ->       Any:
        # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
        # model
        pass
   def a     (	self					:			Union[str, Any]			)   ->       int:
        # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
        # model
        pass
   def a     (	self					:			List[str]			)   ->       Tuple:
        # We disable this test for PreTrainedTokenizerFast because it is the only tokenizer that is not linked to any
        # model
        pass
   def a     (	self					:			Union[str, Any]			)   ->       Any:
        # PreTrainedTokenizerFast doesn't have tokenizer_file in its signature
        pass
   def a     (	self					:			str			)   ->       int:
        __snake_case    			=	self.tmpdirname
        # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'			):
                  try:
                       __snake_case    			=	tempfile.mkdtemp()
                       __snake_case    			=	self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_					,      **SCREAMING_SNAKE_CASE_			)

                       tokenizer.save_pretrained(self.tmpdirname			)
                       super().test_training_new_tokenizer()
                  finally:
                       # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                       # is restored
                       shutil.rmtree(self.tmpdirname			)
                       __snake_case    			=	tmpdirname_orig
   def a     (	self					:			List[str]			)   ->       Any:
        __snake_case    			=	self.tmpdirname
        # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
        for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
             with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'			):
                  try:
                       __snake_case    			=	tempfile.mkdtemp()
                       __snake_case    			=	self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_					,      **SCREAMING_SNAKE_CASE_			)

                       tokenizer.save_pretrained(self.tmpdirname			)
                       super().test_training_new_tokenizer_with_special_tokens_change()
                  finally:
                       # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
                       # is restored
                       shutil.rmtree(self.tmpdirname			)
                       __snake_case    			=	tmpdirname_orig





   def a     (	self					:			int			)   ->       Union[str, Any]:
        __snake_case    			=	self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name			)

        __snake_case    			=	('a' for _ in range(1000			))
        __snake_case    			=	tokenizer.train_new_from_iterator(text_iterator=SCREAMING_SNAKE_CASE_					,      length=1000					,      vocab_size=50			)

        __snake_case    			=	new_tokenizer.encode('a🤗'			)
        self.assertEqual(SCREAMING_SNAKE_CASE_					,      [64, 172, 253, 97, 245]			)




@require_tokenizers
class  _lowercase   (  unittest.TestCase       ):
   def a     (	self					:			Union[str, Any]			)   ->       Tuple:
        __snake_case    			=	AutoTokenizer.from_pretrained('bert-base-cased'			)
        __snake_case    			=	json.loads(tokenizer._tokenizer.to_str()			)
        __snake_case    			=	len(SCREAMING_SNAKE_CASE_			)

        with tempfile.TemporaryDirectory() as tmp_dir:
             # Hack to save this in the tokenizer_config.json
             __snake_case    			=	['tokenizer.4.0.0.json']
             tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_			)
             json.dump(SCREAMING_SNAKE_CASE_					,      open(os.path.join(SCREAMING_SNAKE_CASE_					,      'tokenizer.4.0.0.json'			)					,      'w'			)			)

             # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
             __snake_case    			=	AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_			)
             self.assertEqual(len(SCREAMING_SNAKE_CASE_			)					,      len(SCREAMING_SNAKE_CASE_			) + 1			)
             __snake_case    			=	json.loads(new_tokenizer._tokenizer.to_str()			)
             self.assertIn('huggingface'					,      json_tokenizer['model']['vocab']			)

             # Will need to be adjusted if we reach v42 and this test is still here.
             # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
             shutil.move(os.path.join(SCREAMING_SNAKE_CASE_					,      'tokenizer.4.0.0.json'			)					,      os.path.join(SCREAMING_SNAKE_CASE_					,      'tokenizer.42.0.0.json'			)			)
             __snake_case    			=	['tokenizer.42.0.0.json']
             tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_			)
             __snake_case    			=	AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_			)
             self.assertEqual(len(SCREAMING_SNAKE_CASE_			)					,      len(SCREAMING_SNAKE_CASE_			)			)
             __snake_case    			=	json.loads(new_tokenizer._tokenizer.to_str()			)
             self.assertNotIn('huggingface'					,      json_tokenizer['model']['vocab']			)





   def a     (	self					:			int			)   ->       Union[str, Any]:
        # This repo has two tokenizer files, one for v4.0.0 and above with an added token, one for versions lower.
        __snake_case    			=	'hf-internal-testing/test-two-tokenizers'

        # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
        __snake_case    			=	AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_			)
        self.assertEqual(len(SCREAMING_SNAKE_CASE_			)					,      2_8997			)
        __snake_case    			=	json.loads(tokenizer._tokenizer.to_str()			)
        self.assertIn('huggingface'					,      json_tokenizer['model']['vocab']			)

        # Testing an older version by monkey-patching the version in the module it's used.
        import transformers as old_transformers

        __snake_case    			=	'3.0.0'
        __snake_case    			=	old_transformers.models.auto.AutoTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_			)
        self.assertEqual(len(SCREAMING_SNAKE_CASE_			)					,      2_8996			)
        __snake_case    			=	json.loads(old_tokenizer._tokenizer.to_str()			)
        self.assertNotIn('huggingface'					,      json_tokenizer['model']['vocab']			)




@require_tokenizers
class  _lowercase   (  unittest.TestCase       ):
   def a     (	self					:			Union[str, Any]			)   ->       int:
        # See https://github.com/huggingface/transformers/pull/12550
        # and https://github.com/huggingface/tokenizers/issues/537
        __snake_case    			=	PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel'			)
        __snake_case    			=	'The Matrix is a 1999 science fiction action film.'

        with concurrent.futures.ThreadPoolExecutor() as executor:
             __snake_case    			=	[executor.submit(self.fetch					,      SCREAMING_SNAKE_CASE_					,      SCREAMING_SNAKE_CASE_			) for i in range(10			)]
             __snake_case    			=	[future.result() for future in futures]
             self.assertEqual(SCREAMING_SNAKE_CASE_					,      [[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10			)]			)





   def a     (	self					:			Optional[Any]					,      SCREAMING_SNAKE_CASE_					:			Dict					,      SCREAMING_SNAKE_CASE_					:			Optional[int]			)   ->       Union[str, Any]:
        return tokenizer.encode(SCREAMING_SNAKE_CASE_					,      truncation='longest_first'					,      padding='longest'			)








'''simple docstring'''




from functools import partial
from typing import Any, Callable, Dict, List, Type, TypeVar, Union, overload

import torch
import torch.nn as nn
import torch.types





def      _a		(lowercase__ :      torch.Tensor       ,      lowercase__ :      torch.Tensor       ,      lowercase__ :      bool	) -> torch.Tensor:






     """simple docstring"""
     # The first operation in a checkpoint can't be in-place, but it's
     # nice to have in-place addition during inference. Thus...
     if not inplace:
          __snake_case    			=	ma + ma
     else:
          ma += ma

     return ma





def      _a		(lowercase__ :      torch.Tensor       ,      lowercase__ :      List[int]	) -> torch.Tensor:






     """simple docstring"""
     __snake_case    			=	-1 * len(lowercase__	)
     __snake_case    			=	list(range(len(tensor.shape[:zero_index]	)	)	)
     return tensor.permute(first_inds + [zero_index + i for i in inds]	)





def      _a		(lowercase__ :      torch.Tensor       ,      lowercase__ :      int	) -> torch.Tensor:






     """simple docstring"""
     return t.reshape(t.shape[:-no_dims] + (-1,)	)





def      _a		(lowercase__ :      torch.Tensor       ,      lowercase__ :      torch.Tensor       ,      lowercase__ :      int       ,      lowercase__ :      float = 1e-4	) -> torch.Tensor:






     """simple docstring"""
     __snake_case    			=	mask.expand(*value.shape	)
     return torch.sum(mask * value       ,      dim=lowercase__	) / (eps + torch.sum(lowercase__       ,      dim=lowercase__	))





def      _a		(lowercase__ :      torch.Tensor       ,      lowercase__ :      torch.types.Number = 2.31_25       ,      lowercase__ :      torch.types.Number = 21.68_75       ,      lowercase__ :      int = 6_4	) -> torch.Tensor:






     """simple docstring"""
     __snake_case    			=	torch.linspace(lowercase__       ,      lowercase__       ,      no_bins - 1       ,      device=pts.device	)
     __snake_case    			=	torch.sqrt(torch.sum((pts.unsqueeze(-2	) - pts.unsqueeze(-3	)) ** 2       ,      dim=-1	)	)
     return torch.bucketize(lowercase__       ,      lowercase__	)





def      _a		(lowercase__ :      Callable[[list], Any]       ,      lowercase__ :      List[dict]	) -> dict:






     """simple docstring"""
     __snake_case    			=	dicts[0]
     __snake_case    			=	{}
     for k, v in first.items():
          __snake_case    			=	[d[k] for d in dicts]
          if isinstance(lowercase__       ,      lowercase__	):
               __snake_case    			=	dict_multimap(lowercase__       ,      lowercase__	)
          else:
               __snake_case    			=	fn(lowercase__	)

     return new_dict





def      _a		(lowercase__ :      torch.Tensor       ,      lowercase__ :      torch.Tensor	) -> torch.Tensor:






     """simple docstring"""
     __snake_case    			=	v_bins.view(((1,) * len(x.shape	)) + (len(lowercase__	),)	)
     __snake_case    			=	x[..., None] - reshaped_bins
     __snake_case    			=	torch.argmin(torch.abs(lowercase__	)       ,      dim=-1	)
     return nn.functional.one_hot(lowercase__       ,      num_classes=len(lowercase__	)	).float()





def      _a		(lowercase__ :      torch.Tensor       ,      lowercase__ :      torch.Tensor       ,      lowercase__ :      int = 0       ,      lowercase__ :      int = 0	) -> torch.Tensor:






     """simple docstring"""
     __snake_case    			=	[]
     for i, s in enumerate(data.shape[:no_batch_dims]	):
          __snake_case    			=	torch.arange(lowercase__	)
          __snake_case    			=	r.view(*(*((1,) * i), -1, *((1,) * (len(inds.shape	) - i - 1)))	)
          ranges.append(lowercase__	)

     __snake_case    			=	[slice(lowercase__	) for _ in range(len(data.shape	) - no_batch_dims	)]
     __snake_case    			=	inds
     ranges.extend(lowercase__	)
     # Matt note: Editing this to get around the behaviour of using a list as an array index changing
     # in recent Numpy versions
     return data[tuple(lowercase__	)]


_a      :      str       				=      TypeVar("T")





def      _a		(lowercase__ :      Callable[[T], Any]       ,      lowercase__ :      Dict[Any, Union[dict, list, tuple, T]]       ,      lowercase__ :      Type[T]	) -> Dict[Any, Union[dict, list, tuple, Any]]:






     """simple docstring"""
     __snake_case    			=	{}
     for k, v in dic.items():
          if isinstance(lowercase__       ,      lowercase__	):
               __snake_case    			=	dict_map(lowercase__       ,      lowercase__       ,      lowercase__	)
          else:
               __snake_case    			=	tree_map(lowercase__       ,      lowercase__       ,      lowercase__	)

     return new_dict





@overload
def      _a		(lowercase__ :      Callable[[T], Any]       ,      lowercase__ :      T       ,      lowercase__ :      Type[T]	) -> Any:






     """simple docstring"""
     ...





@overload
def      _a		(lowercase__ :      Callable[[T], Any]       ,      lowercase__ :      dict       ,      lowercase__ :      Type[T]	) -> dict:






     """simple docstring"""
     ...





@overload
def      _a		(lowercase__ :      Callable[[T], Any]       ,      lowercase__ :      list       ,      lowercase__ :      Type[T]	) -> list:






     """simple docstring"""
     ...





@overload
def      _a		(lowercase__ :      Callable[[T], Any]       ,      lowercase__ :      tuple       ,      lowercase__ :      Type[T]	) -> tuple:






     """simple docstring"""
     ...





def      _a		(lowercase__ :      str       ,      lowercase__ :      Any       ,      lowercase__ :      Dict	) -> Union[str, Any]:






     """simple docstring"""
     if isinstance(lowercase__       ,      lowercase__	):
          return dict_map(lowercase__       ,      lowercase__       ,      lowercase__	)
     elif isinstance(lowercase__       ,      lowercase__	):
          return [tree_map(lowercase__       ,      lowercase__       ,      lowercase__	) for x in tree]
     elif isinstance(lowercase__       ,      lowercase__	):
          return tuple(tree_map(lowercase__       ,      lowercase__       ,      lowercase__	) for x in tree	)
     elif isinstance(lowercase__       ,      lowercase__	):
          return fn(lowercase__	)
     else:
          print(type(lowercase__	)	)
          raise ValueError('Not supported'	)


_a      :      Union[str, Any]       				=      partial(tree_map, leaf_type=torch.Tensor)




