


from typing import TYPE_CHECKING

from ..utils import OptionalDependencyNotAvailable, _LazyModule, is_flax_available, is_tf_available, is_torch_available


UpperCamelCase :				Union[str, Any]						    =       {
    '''configuration_utils''': ['''GenerationConfig'''],
    '''streamers''': ['''TextIteratorStreamer''', '''TextStreamer'''],
}

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase :				Tuple						    =       [
					    '''Constraint''',
					    '''ConstraintListState''',
					    '''DisjunctiveConstraint''',
					    '''PhrasalConstraint''',
					]
					UpperCamelCase :				List[str]						    =       [
					    '''BeamHypotheses''',
					    '''BeamScorer''',
					    '''BeamSearchScorer''',
					    '''ConstrainedBeamSearchScorer''',
					]
					UpperCamelCase :				List[Any]						    =       [
					    '''EpsilonLogitsWarper''',
					    '''EtaLogitsWarper''',
					    '''ForcedBOSTokenLogitsProcessor''',
					    '''ForcedEOSTokenLogitsProcessor''',
					    '''HammingDiversityLogitsProcessor''',
					    '''InfNanRemoveLogitsProcessor''',
					    '''LogitsProcessor''',
					    '''LogitsProcessorList''',
					    '''LogitsWarper''',
					    '''MinLengthLogitsProcessor''',
					    '''MinNewTokensLengthLogitsProcessor''',
					    '''NoBadWordsLogitsProcessor''',
					    '''NoRepeatNGramLogitsProcessor''',
					    '''PrefixConstrainedLogitsProcessor''',
					    '''RepetitionPenaltyLogitsProcessor''',
					    '''SequenceBiasLogitsProcessor''',
					    '''EncoderRepetitionPenaltyLogitsProcessor''',
					    '''TemperatureLogitsWarper''',
					    '''TopKLogitsWarper''',
					    '''TopPLogitsWarper''',
					    '''TypicalLogitsWarper''',
					    '''EncoderNoRepeatNGramLogitsProcessor''',
					    '''ExponentialDecayLengthPenalty''',
					    '''LogitNormalization''',
					]
					UpperCamelCase :				Optional[int]						    =       [
					    '''MaxNewTokensCriteria''',
					    '''MaxLengthCriteria''',
					    '''MaxTimeCriteria''',
					    '''StoppingCriteria''',
					    '''StoppingCriteriaList''',
					    '''validate_stopping_criteria''',
					]
					UpperCamelCase :				Dict						    =       [
					    '''GenerationMixin''',
					    '''top_k_top_p_filtering''',
					    '''GreedySearchEncoderDecoderOutput''',
					    '''GreedySearchDecoderOnlyOutput''',
					    '''SampleEncoderDecoderOutput''',
					    '''SampleDecoderOnlyOutput''',
					    '''BeamSearchEncoderDecoderOutput''',
					    '''BeamSearchDecoderOnlyOutput''',
					    '''BeamSampleEncoderDecoderOutput''',
					    '''BeamSampleDecoderOnlyOutput''',
					    '''ContrastiveSearchEncoderDecoderOutput''',
					    '''ContrastiveSearchDecoderOnlyOutput''',
					]

try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase :				Dict						    =       [
					    '''TFForcedBOSTokenLogitsProcessor''',
					    '''TFForcedEOSTokenLogitsProcessor''',
					    '''TFLogitsProcessor''',
					    '''TFLogitsProcessorList''',
					    '''TFLogitsWarper''',
					    '''TFMinLengthLogitsProcessor''',
					    '''TFNoBadWordsLogitsProcessor''',
					    '''TFNoRepeatNGramLogitsProcessor''',
					    '''TFRepetitionPenaltyLogitsProcessor''',
					    '''TFTemperatureLogitsWarper''',
					    '''TFTopKLogitsWarper''',
					    '''TFTopPLogitsWarper''',
					    '''TFForceTokensLogitsProcessor''',
					    '''TFSuppressTokensAtBeginLogitsProcessor''',
					    '''TFSuppressTokensLogitsProcessor''',
					]
					UpperCamelCase :				List[Any]						    =       [
					    '''TFGenerationMixin''',
					    '''tf_top_k_top_p_filtering''',
					    '''TFGreedySearchDecoderOnlyOutput''',
					    '''TFGreedySearchEncoderDecoderOutput''',
					    '''TFSampleEncoderDecoderOutput''',
					    '''TFSampleDecoderOnlyOutput''',
					    '''TFBeamSearchEncoderDecoderOutput''',
					    '''TFBeamSearchDecoderOnlyOutput''',
					    '''TFBeamSampleEncoderDecoderOutput''',
					    '''TFBeamSampleDecoderOnlyOutput''',
					    '''TFContrastiveSearchEncoderDecoderOutput''',
					    '''TFContrastiveSearchDecoderOnlyOutput''',
					]

try:
					if not is_flax_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					UpperCamelCase :				Tuple						    =       [
					    '''FlaxForcedBOSTokenLogitsProcessor''',
					    '''FlaxForcedEOSTokenLogitsProcessor''',
					    '''FlaxLogitsProcessor''',
					    '''FlaxLogitsProcessorList''',
					    '''FlaxLogitsWarper''',
					    '''FlaxMinLengthLogitsProcessor''',
					    '''FlaxTemperatureLogitsWarper''',
					    '''FlaxTopKLogitsWarper''',
					    '''FlaxTopPLogitsWarper''',
					]
					UpperCamelCase :				Any						    =       [
					    '''FlaxGenerationMixin''',
					    '''FlaxGreedySearchOutput''',
					    '''FlaxSampleOutput''',
					    '''FlaxBeamSearchOutput''',
					]

if TYPE_CHECKING:
					from .configuration_utils import GenerationConfig
					from .streamers import TextIteratorStreamer, TextStreamer

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .beam_constraints import Constraint, ConstraintListState, DisjunctiveConstraint, PhrasalConstraint
										from .beam_search import BeamHypotheses, BeamScorer, BeamSearchScorer, ConstrainedBeamSearchScorer
										from .logits_process import (
										    EncoderNoRepeatNGramLogitsProcessor,
										    EncoderRepetitionPenaltyLogitsProcessor,
										    EpsilonLogitsWarper,
										    EtaLogitsWarper,
										    ExponentialDecayLengthPenalty,
										    ForcedBOSTokenLogitsProcessor,
										    ForcedEOSTokenLogitsProcessor,
										    HammingDiversityLogitsProcessor,
										    InfNanRemoveLogitsProcessor,
										    LogitNormalization,
										    LogitsProcessor,
										    LogitsProcessorList,
										    LogitsWarper,
										    MinLengthLogitsProcessor,
										    MinNewTokensLengthLogitsProcessor,
										    NoBadWordsLogitsProcessor,
										    NoRepeatNGramLogitsProcessor,
										    PrefixConstrainedLogitsProcessor,
										    RepetitionPenaltyLogitsProcessor,
										    SequenceBiasLogitsProcessor,
										    TemperatureLogitsWarper,
										    TopKLogitsWarper,
										    TopPLogitsWarper,
										    TypicalLogitsWarper,
										)
										from .stopping_criteria import (
										    MaxLengthCriteria,
										    MaxNewTokensCriteria,
										    MaxTimeCriteria,
										    StoppingCriteria,
										    StoppingCriteriaList,
										    validate_stopping_criteria,
										)
										from .utils import (
										    BeamSampleDecoderOnlyOutput,
										    BeamSampleEncoderDecoderOutput,
										    BeamSearchDecoderOnlyOutput,
										    BeamSearchEncoderDecoderOutput,
										    ContrastiveSearchDecoderOnlyOutput,
										    ContrastiveSearchEncoderDecoderOutput,
										    GenerationMixin,
										    GreedySearchDecoderOnlyOutput,
										    GreedySearchEncoderDecoderOutput,
										    SampleDecoderOnlyOutput,
										    SampleEncoderDecoderOutput,
										    top_k_top_p_filtering,
										)

					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .tf_logits_process import (
										    TFForcedBOSTokenLogitsProcessor,
										    TFForcedEOSTokenLogitsProcessor,
										    TFForceTokensLogitsProcessor,
										    TFLogitsProcessor,
										    TFLogitsProcessorList,
										    TFLogitsWarper,
										    TFMinLengthLogitsProcessor,
										    TFNoBadWordsLogitsProcessor,
										    TFNoRepeatNGramLogitsProcessor,
										    TFRepetitionPenaltyLogitsProcessor,
										    TFSuppressTokensAtBeginLogitsProcessor,
										    TFSuppressTokensLogitsProcessor,
										    TFTemperatureLogitsWarper,
										    TFTopKLogitsWarper,
										    TFTopPLogitsWarper,
										)
										from .tf_utils import (
										    TFBeamSampleDecoderOnlyOutput,
										    TFBeamSampleEncoderDecoderOutput,
										    TFBeamSearchDecoderOnlyOutput,
										    TFBeamSearchEncoderDecoderOutput,
										    TFContrastiveSearchDecoderOnlyOutput,
										    TFContrastiveSearchEncoderDecoderOutput,
										    TFGenerationMixin,
										    TFGreedySearchDecoderOnlyOutput,
										    TFGreedySearchEncoderDecoderOutput,
										    TFSampleDecoderOnlyOutput,
										    TFSampleEncoderDecoderOutput,
										    tf_top_k_top_p_filtering,
										)

					try:
										if not is_flax_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .flax_logits_process import (
										    FlaxForcedBOSTokenLogitsProcessor,
										    FlaxForcedEOSTokenLogitsProcessor,
										    FlaxLogitsProcessor,
										    FlaxLogitsProcessorList,
										    FlaxLogitsWarper,
										    FlaxMinLengthLogitsProcessor,
										    FlaxTemperatureLogitsWarper,
										    FlaxTopKLogitsWarper,
										    FlaxTopPLogitsWarper,
										)
										from .flax_utils import FlaxBeamSearchOutput, FlaxGenerationMixin, FlaxGreedySearchOutput, FlaxSampleOutput
else:
					import sys

					UpperCamelCase :				List[Any]						    =       _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





import os
import tempfile
import unittest
from typing import List

from transformers.models.esm.tokenization_esm import VOCAB_FILES_NAMES, EsmTokenizer
from transformers.testing_utils import require_tokenizers
from transformers.tokenization_utils import PreTrainedTokenizer
from transformers.tokenization_utils_base import PreTrainedTokenizerBase

@require_tokenizers
class     __snake_case							(      unittest.TestCase):
					SCREAMING_SNAKE_CASE_		:					Tuple 				= EsmTokenizer
					def   a (	self					:	List[Any]     )       ->				Dict:



											'''simple docstring'''





											super().setUp()
											__magic_name__ 		=			tempfile.mkdtemp()
											# fmt: off
											__magic_name__ 		=			["<cls>", "<pad>", "<eos>", "<unk>", "L", "A", "G", "V", "S", "E", "R", "T", "I", "D", "P", "K", "Q", "N", "F", "Y", "M", "H", "W", "C", "X", "B", "U", "Z", "O", ".", "-", "<null_1>", "<mask>"]  # noqa: E501
											# fmt: on
											__magic_name__ 		=			os.path.join(self.tmpdirname  ,			VOCAB_FILES_NAMES['vocab_file']     )
											with open(self.vocab_file  ,			'w'  ,			encoding='utf-8'     ) as vocab_writer:
																	vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]     )     )
					def   a (	self					:	Optional[int]  ,			**_lowerCAmelCase					:	int     )       ->				List[PreTrainedTokenizerBase]:



											'''simple docstring'''





											return [self.get_tokenizer(**_lowerCAmelCase     )]
					def   a (	self					:	List[str]  ,			**_lowerCAmelCase					:	int     )       ->				PreTrainedTokenizer:



											'''simple docstring'''





											return self.tokenizer_class.from_pretrained(self.tmpdirname  ,			**_lowerCAmelCase     )
					def   a (	self					:	int     )       ->				int:



											'''simple docstring'''





											__magic_name__ 		=			self.tokenizer_class(self.vocab_file     )

											__magic_name__ 		=			tokenizer.tokenize('LAGVS'     )
											self.assertListEqual(_lowerCAmelCase  ,			['L', 'A', 'G', 'V', 'S']     )
											self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCAmelCase     )  ,			[4, 5, 6, 7, 8]     )
					def   a (	self					:	str     )       ->				str:



											'''simple docstring'''





											__magic_name__ 		=			self.tokenizer_class(self.vocab_file     )

											__magic_name__ 		=			'LAGVS'
											self.assertListEqual(tokenizer.encode(_lowerCAmelCase     )  ,			[0, 4, 5, 6, 7, 8, 2]     )
					def   a (	self					:	int     )       ->				List[Any]:



											'''simple docstring'''





											__magic_name__ 		=			self.tokenizer_class(self.vocab_file     )

											__magic_name__ 		=			['LAGVS', 'WCB']
											__magic_name__ 		=			tokenizer(_lowerCAmelCase  ,			padding=_lowerCAmelCase     )['input_ids']

											self.assertListEqual(_lowerCAmelCase  ,			[[0, 4, 5, 6, 7, 8, 2], [0, 2_2, 2_3, 2_5, 2]]     )
					def   a (	self					:	Dict     )       ->				List[Any]:



											'''simple docstring'''





											__magic_name__ 		=			self.tokenizer_class(self.vocab_file     )

											__magic_name__ 		=			['LAGVS', 'WCB']
											__magic_name__ 		=			tokenizer(_lowerCAmelCase  ,			padding=_lowerCAmelCase     )['input_ids']

											self.assertListEqual(_lowerCAmelCase  ,			[[0, 4, 5, 6, 7, 8, 2], [0, 2_2, 2_3, 2_5, 2, 1, 1]]     )







					def   a (	self					:	Optional[int]     )       ->				str:



											'''simple docstring'''





											__magic_name__ 		=			self.get_tokenizers(fast=_lowerCAmelCase     )
											for tokenizer in tokenizers:
																	with self.subTest(F'''{tokenizer.__class__.__name__}'''     ):
																							__magic_name__ 		=			'<unk>'
																							__magic_name__ 		=			'<mask>'

																							__magic_name__ 		=			tokenizer.tokenize(_lowerCAmelCase     )
																							__magic_name__ 		=			tokenizer.tokenize(_lowerCAmelCase     )

																							self.assertEqual(len(_lowerCAmelCase     )  ,			1     )
																							self.assertEqual(len(_lowerCAmelCase     )  ,			1     )
																							self.assertEqual(token_a[0]  ,			_lowerCAmelCase     )
																							self.assertEqual(token_a[0]  ,			_lowerCAmelCase     )

