






"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__SCREAMING_SNAKE_CASE      =      {
    'configuration_nystromformer': ['NYSTROMFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP', 'NystromformerConfig'],
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		__SCREAMING_SNAKE_CASE      =      [
		    'NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'NystromformerForMaskedLM',
		    'NystromformerForMultipleChoice',
		    'NystromformerForQuestionAnswering',
		    'NystromformerForSequenceClassification',
		    'NystromformerForTokenClassification',
		    'NystromformerLayer',
		    'NystromformerModel',
		    'NystromformerPreTrainedModel',
		]


if TYPE_CHECKING:
		from .configuration_nystromformer import NYSTROMFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, NystromformerConfig

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_nystromformer import (
				    NYSTROMFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
				    NystromformerForMaskedLM,
				    NystromformerForMultipleChoice,
				    NystromformerForQuestionAnswering,
				    NystromformerForSequenceClassification,
				    NystromformerForTokenClassification,
				    NystromformerLayer,
				    NystromformerModel,
				    NystromformerPreTrainedModel,
				)


else:
		import sys

		__SCREAMING_SNAKE_CASE      =      _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)







"""simple docstring"""


import unittest
import warnings
from dataclasses import dataclass

from transformers.convert_slow_tokenizer import SpmConverter
from transformers.testing_utils import get_tests_dir




@dataclass
class 	__UpperCamelCase    :
					lowercase_						:							str




class 	__UpperCamelCase    (    unittest.TestCase	):
					def 					UpperCAmelCase__   (       self       :  Union[str, Any]      )  ->					Optional[Any]:
											lowerCAmelCase      :List[str]				  = get_tests_dir('fixtures/test_sentencepiece.model'      )
											lowerCAmelCase      :Tuple				  = get_tests_dir('fixtures/test_sentencepiece_with_bytefallback.model'      )

											lowerCAmelCase      :List[str]				  = FakeOriginalTokenizer(vocab_file=UpperCAmelCase      )

											with warnings.catch_warnings(record=UpperCAmelCase      ) as w:
																	lowerCAmelCase      :Dict				  = SpmConverter(UpperCAmelCase      )
											self.assertEqual(len(UpperCAmelCase      )    ,   0      )

											lowerCAmelCase      :int				  = FakeOriginalTokenizer(vocab_file=UpperCAmelCase      )

											with warnings.catch_warnings(record=UpperCAmelCase      ) as w:
																	lowerCAmelCase      :List[str]				  = SpmConverter(UpperCAmelCase      )
											self.assertEqual(len(UpperCAmelCase      )    ,   1      )

											self.assertIn(
											    'The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option'
											    ' which is not implemented in the fast tokenizers.'    ,   str(w[0].message      )    ,   )