



"""simple docstring"""






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mpnet import MPNetTokenizer


__UpperCAmelCase      :      Union[str, Any]   					=					logging.get_logger(__name__)

__UpperCAmelCase      :      int   					=					{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

__UpperCAmelCase      :      List[Any]   					=					{
    'vocab_file': {
        'microsoft/mpnet-base': 'https://huggingface.co/microsoft/mpnet-base/resolve/main/vocab.txt',
    },
    'tokenizer_file': {
        'microsoft/mpnet-base': 'https://huggingface.co/microsoft/mpnet-base/resolve/main/tokenizer.json',
    },
}

__UpperCAmelCase      :      Union[str, Any]   					=					{
    'microsoft/mpnet-base': 5_12,
}

__UpperCAmelCase      :      List[Any]   					=					{
    'microsoft/mpnet-base': {'do_lower_case': True},
}

class 	__lowerCAmelCase							(__UpperCamelCase  ):


					'''simple docstring'''


					a__        =     VOCAB_FILES_NAMES
					a__        =     PRETRAINED_VOCAB_FILES_MAP
					a__        =     PRETRAINED_INIT_CONFIGURATION
					a__        =     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					a__        =     MPNetTokenizer
					a__        =     ['input_ids', 'attention_mask']







					def __init__(      self    ,       a=None    ,       a=None    ,       a=True    ,       a="<s>"    ,       a="</s>"    ,       a="</s>"    ,       a="<s>"    ,       a="[UNK]"    ,       a="<pad>"    ,       a="<mask>"    ,       a=True    ,       a=None    ,       **a    ,       ):




										"""simple docstring"""





										super().__init__(
										    a    ,       tokenizer_file=a    ,       do_lower_case=a    ,       bos_token=a    ,       eos_token=a    ,       sep_token=a    ,       cls_token=a    ,       unk_token=a    ,       pad_token=a    ,       mask_token=a    ,       tokenize_chinese_chars=a    ,       strip_accents=a    ,       **a    ,       )

										snake_case_						:Any								=						json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
										if (
										    pre_tok_state.get("lowercase"    ,       a   ) != do_lower_case
										    or pre_tok_state.get("strip_accents"    ,       a   ) != strip_accents
										):
															snake_case_						:Dict								=						getattr(a    ,       pre_tok_state.pop("type"   )   )
															snake_case_						:int								=						do_lower_case
															snake_case_						:List[str]								=						strip_accents
															snake_case_						:Optional[Any]								=						pre_tok_class(**a   )

										snake_case_						:List[str]								=						do_lower_case







					@property
					def 		_a (      self   ):




										"""simple docstring"""





										if self._mask_token is None:
															if self.verbose:
																				logger.error("Using mask_token, but it is not set yet."   )
															return None
										return str(self._mask_token   )







					@mask_token.setter
					def 		_a (      self    ,       a   ):




										"""simple docstring"""





										snake_case_						:List[str]								=						AddedToken(a    ,       lstrip=a    ,       rstrip=a   ) if isinstance(a    ,       a   ) else value
										snake_case_						:Optional[Any]								=						value







					def 		_a (      self    ,       a    ,       a=None   ):




										"""simple docstring"""





										snake_case_						:Dict								=						[self.bos_token_id] + token_ids_a + [self.eos_token_id]
										if token_ids_a is None:
															return output

										return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







					def 		_a (      self    ,       a    ,       a = None   ):




										"""simple docstring"""





										snake_case_						:Optional[int]								=						[self.sep_token_id]
										snake_case_						:str								=						[self.cls_token_id]

										if token_ids_a is None:
															return len(cls + token_ids_a + sep   ) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep   ) * [0]







					def 		_a (      self    ,       a    ,       a = None   ):




										"""simple docstring"""





										snake_case_						:List[str]								=						self._tokenizer.model.save(a    ,       name=a   )
										return tuple(a   )










"""simple docstring"""






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


__UpperCAmelCase      :      Optional[int]   					=					{
    'configuration_segformer': ['SEGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP', 'SegformerConfig', 'SegformerOnnxConfig']
}

try:
				if not is_vision_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				__UpperCAmelCase      :      int   					=					['SegformerFeatureExtractor']
				__UpperCAmelCase      :      Dict   					=					['SegformerImageProcessor']

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				__UpperCAmelCase      :      Optional[Any]   					=					[
				    'SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
				    'SegformerDecodeHead',
				    'SegformerForImageClassification',
				    'SegformerForSemanticSegmentation',
				    'SegformerLayer',
				    'SegformerModel',
				    'SegformerPreTrainedModel',
				]

try:
				if not is_tf_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				__UpperCAmelCase      :      Dict   					=					[
				    'TF_SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
				    'TFSegformerDecodeHead',
				    'TFSegformerForImageClassification',
				    'TFSegformerForSemanticSegmentation',
				    'TFSegformerModel',
				    'TFSegformerPreTrainedModel',
				]


if TYPE_CHECKING:
				from .configuration_segformer import SEGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, SegformerConfig, SegformerOnnxConfig

				try:
								if not is_vision_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .feature_extraction_segformer import SegformerFeatureExtractor
								from .image_processing_segformer import SegformerImageProcessor

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_segformer import (
								    SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
								    SegformerDecodeHead,
								    SegformerForImageClassification,
								    SegformerForSemanticSegmentation,
								    SegformerLayer,
								    SegformerModel,
								    SegformerPreTrainedModel,
								)
				try:
								if not is_tf_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_tf_segformer import (
								    TF_SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
								    TFSegformerDecodeHead,
								    TFSegformerForImageClassification,
								    TFSegformerForSemanticSegmentation,
								    TFSegformerModel,
								    TFSegformerPreTrainedModel,
								)

else:
				import sys

				__UpperCAmelCase      :      Optional[Any]   					=					_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

