

'''simple docstring'''

import concurrent.futures
import json
import os
import shutil
import tempfile
import unittest

from transformers import AutoTokenizer, PreTrainedTokenizerFast
from transformers.testing_utils import require_tokenizers

from ..test_tokenization_common import TokenizerTesterMixin







@require_tokenizers
class lowerCamelCase      (						__lowerCamelCase							,							unittest.TestCase							):
    snake_case_           =   PreTrainedTokenizerFast
    snake_case_           =   False
    snake_case_           =   True
    snake_case_           =   'tokenizer_file'

    def 				_lowerCamelCase						(					self   )						->       str:
     snake_case							 =				False  # because we don't have pretrained_vocab_files_map
     super().setUp()
     snake_case							 =				True

     snake_case							 =				['robot-test/dummy-tokenizer-fast', 'robot-test/dummy-tokenizer-wordlevel']
     snake_case							 =				'SaulLu/dummy-tokenizer-bytelevel-bpe'

     # Inclusion of 2 tokenizers to test different types of models (Unigram and WordLevel for the moment)
     snake_case							 =				[(PreTrainedTokenizerFast, model_path, {}) for model_path in model_paths]

     snake_case							 =				PreTrainedTokenizerFast.from_pretrained(model_paths[0]   )
     tokenizer.save_pretrained(self.tmpdirname   )

    def 				_lowerCamelCase						(					self   )						->       Any:
     pass

    def 				_lowerCamelCase						(					self   )						->       str:
     pass

    def 				_lowerCamelCase						(					self   )						->       Optional[Any]:
     pass

    def 				_lowerCamelCase						(					self   )						->       Optional[Any]:
     pass

    def 				_lowerCamelCase						(					self   )						->       int:
     snake_case							 =				self.tmpdirname
     # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
       try:
        snake_case							 =				tempfile.mkdtemp()
        snake_case							 =				self.rust_tokenizer_class.from_pretrained(lowercase_,	**lowercase_   )

        tokenizer.save_pretrained(self.tmpdirname   )
        super().test_training_new_tokenizer()
       finally:
        # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
        # is restored
        shutil.rmtree(self.tmpdirname   )
        snake_case							 =				tmpdirname_orig

    def 				_lowerCamelCase						(					self   )						->       str:
     snake_case							 =				self.tmpdirname
     # Here we want to test the 2 available tokenizers that use 2 different types of models: Unigram and WordLevel.
     for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
      with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''   ):
       try:
        snake_case							 =				tempfile.mkdtemp()
        snake_case							 =				self.rust_tokenizer_class.from_pretrained(lowercase_,	**lowercase_   )

        tokenizer.save_pretrained(self.tmpdirname   )
        super().test_training_new_tokenizer_with_special_tokens_change()
       finally:
        # Even if the test fails, we must be sure that the folder is deleted and that the default tokenizer
        # is restored
        shutil.rmtree(self.tmpdirname   )
        snake_case							 =				tmpdirname_orig






    def 				_lowerCamelCase						(					self   )						->       Any:
     snake_case							 =				self.rust_tokenizer_class.from_pretrained(self.bytelevel_bpe_model_name   )

     snake_case							 =				('a' for _ in range(1000   ))
     snake_case							 =				tokenizer.train_new_from_iterator(text_iterator=lowercase_,	length=1000,	vocab_size=50   )

     snake_case							 =				new_tokenizer.encode('a🤗'   )
     self.assertEqual(lowercase_,	[64, 172, 253, 97, 245]   )







@require_tokenizers
class lowerCamelCase      (						unittest.TestCase							):

    def 				_lowerCamelCase						(					self   )						->       Optional[int]:
     snake_case							 =				AutoTokenizer.from_pretrained('bert-base-cased'   )
     snake_case							 =				json.loads(tokenizer._tokenizer.to_str()   )
     snake_case							 =				len(lowercase_   )

     with tempfile.TemporaryDirectory() as tmp_dir:
      # Hack to save this in the tokenizer_config.json
      snake_case							 =				['tokenizer.4.0.0.json']
      tokenizer.save_pretrained(lowercase_   )
      json.dump(lowercase_,	open(os.path.join(lowercase_,	'tokenizer.4.0.0.json'   ),	'w'   )   )

      # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
      snake_case							 =				AutoTokenizer.from_pretrained(lowercase_   )
      self.assertEqual(len(lowercase_   ),	len(lowercase_   ) + 1   )
      snake_case							 =				json.loads(new_tokenizer._tokenizer.to_str()   )
      self.assertIn('huggingface',	json_tokenizer['model']['vocab']   )

      # Will need to be adjusted if we reach v42 and this test is still here.
      # Should pick the old tokenizer file as the version of Transformers is < 4.0.0
      shutil.move(os.path.join(lowercase_,	'tokenizer.4.0.0.json'   ),	os.path.join(lowercase_,	'tokenizer.42.0.0.json'   )   )
      snake_case							 =				['tokenizer.42.0.0.json']
      tokenizer.save_pretrained(lowercase_   )
      snake_case							 =				AutoTokenizer.from_pretrained(lowercase_   )
      self.assertEqual(len(lowercase_   ),	len(lowercase_   )   )
      snake_case							 =				json.loads(new_tokenizer._tokenizer.to_str()   )
      self.assertNotIn('huggingface',	json_tokenizer['model']['vocab']   )






    def 				_lowerCamelCase						(					self   )						->       Tuple:
     snake_case							 =				'hf-internal-testing/test-two-tokenizers'

     # This should pick the new tokenizer file as the version of Transformers is > 4.0.0
     snake_case							 =				AutoTokenizer.from_pretrained(lowercase_   )
     self.assertEqual(len(lowercase_   ),	28997   )
     snake_case							 =				json.loads(tokenizer._tokenizer.to_str()   )
     self.assertIn('huggingface',	json_tokenizer['model']['vocab']   )

     # Testing an older version by monkey-patching the version in the module it's used.
     import transformers as old_transformers

     snake_case							 =				'3.0.0'
     snake_case							 =				old_transformers.models.auto.AutoTokenizer.from_pretrained(lowercase_   )
     self.assertEqual(len(lowercase_   ),	28996   )
     snake_case							 =				json.loads(old_tokenizer._tokenizer.to_str()   )
     self.assertNotIn('huggingface',	json_tokenizer['model']['vocab']   )







@require_tokenizers
class lowerCamelCase      (						unittest.TestCase							):

    def 				_lowerCamelCase						(					self   )						->       Any:
     snake_case							 =				PreTrainedTokenizerFast.from_pretrained('robot-test/dummy-tokenizer-wordlevel'   )
     snake_case							 =				'The Matrix is a 1999 science fiction action film.'

     with concurrent.futures.ThreadPoolExecutor() as executor:
      snake_case							 =				[executor.submit(self.fetch,	lowercase_,	lowercase_   ) for i in range(10   )]
      snake_case							 =				[future.result() for future in futures]
      self.assertEqual(lowercase_,	[[1, 10, 0, 8, 0, 18, 0, 0, 0, 2] for i in range(10   )]   )






    def 				_lowerCamelCase						(					self,	lowercase_,	lowercase_   )						->       Optional[int]:
     return tokenizer.encode(lowercase_,	truncation='longest_first',	padding='longest'   )








'''simple docstring'''

from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


lowerCAmelCase_						   =			logging.get_logger(__name__)

lowerCAmelCase_						   =			{
    "mnaylor/mega-base-wikitext": "https://huggingface.co/mnaylor/mega-base-wikitext/resolve/main/config.json",
}







class lowerCamelCase      (						__lowerCAmelCase							):
				snake_case_           =   '''mega'''

				def __init__(					self,	lowercase_=30522,	lowercase_=128,	lowercase_=4,	lowercase_=256,	lowercase_=16,	lowercase_=True,	lowercase_=64,	lowercase_=False,	lowercase_=-1,	lowercase_=None,	lowercase_=True,	lowercase_="scalenorm",	lowercase_=True,	lowercase_="silu",	lowercase_="softmax",	lowercase_=0.1,	lowercase_=0.1,	lowercase_=0.1,	lowercase_=False,	lowercase_=True,	lowercase_=256,	lowercase_=True,	lowercase_=0.1,	lowercase_=2048,	lowercase_=False,	lowercase_=2,	lowercase_=0.02,	lowercase_=0.2,	lowercase_=0.02,	lowercase_=1.0,	lowercase_=1,	lowercase_=0,	lowercase_=2,	lowercase_="rotary",	lowercase_=None,	lowercase_=True,	lowercase_=True,	**lowercase_,	)						->       List[Any]:
					super().__init__(pad_token_id=lowercase_,	bos_token_id=lowercase_,	eos_token_id=lowercase_,	**lowercase_   )

					snake_case							 =				vocab_size
					snake_case							 =				hidden_size
					snake_case							 =				num_hidden_layers
					snake_case							 =				activation
					snake_case							 =				attention_activation
					snake_case							 =				intermediate_size
					snake_case							 =				ema_projection_size
					snake_case							 =				bidirectional
					snake_case							 =				shared_representation_size
					snake_case							 =				use_chunking
					snake_case							 =				chunk_size
					snake_case							 =				truncation
					snake_case							 =				normalize_before_mega
					snake_case							 =				normalization_type
					snake_case							 =				norm_affine
					snake_case							 =				dropout_prob
					snake_case							 =				hidden_dropout_prob
					snake_case							 =				attention_probs_dropout_prob
					snake_case							 =				use_feature_dropout
					snake_case							 =				use_normalized_ffn
					snake_case							 =				nffn_hidden_size
					snake_case							 =				normalize_before_ffn
					snake_case							 =				nffn_activation_dropout_prob
					snake_case							 =				max_positions
					snake_case							 =				add_token_type_embeddings
					snake_case							 =				type_vocab_size
					snake_case							 =				initializer_range
					snake_case							 =				ema_delta_alpha_range
					snake_case							 =				ema_beta_range
					snake_case							 =				ema_gamma_omega_range
					snake_case							 =				relative_positional_bias
					snake_case							 =				use_cache
					snake_case							 =				classifier_dropout
					snake_case							 =				add_lm_hidden_dense_layer
					snake_case							 =				1  # not used but required by Hugging Face







class lowerCamelCase      (						__lowerCAmelCase							):

				@property
				def 				_lowerCamelCase						(					self   )						->       Mapping[str, Mapping[int, str]]:
					if self.task == "multiple-choice":
						snake_case							 =				{0: 'batch', 1: 'choice', 2: 'sequence'}
					else:
						snake_case							 =				{0: 'batch', 1: 'sequence'}
					return OrderedDict(
					    [
					        ('input_ids', dynamic_axis),
					        ('attention_mask', dynamic_axis),
					    ]   )





