




'''simple docstring'''






import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


A_ 			=		logging.get_logger(__name__)

A_ 			=		{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

A_ 			=		{
    "vocab_file": {
        "microsoft/layoutlm-base-uncased": (
            "https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt"
        ),
        "microsoft/layoutlm-large-uncased": (
            "https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "microsoft/layoutlm-base-uncased": (
            "https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json"
        ),
        "microsoft/layoutlm-large-uncased": (
            "https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json"
        ),
    },
}

A_ 			=		{
    "microsoft/layoutlm-base-uncased": 512,
    "microsoft/layoutlm-large-uncased": 512,
}

A_ 			=		{
    "microsoft/layoutlm-base-uncased": {"do_lower_case": True},
    "microsoft/layoutlm-large-uncased": {"do_lower_case": True},
}






class 					lowercase_			(	lowerCAmelCase_       ):
     A_       							= VOCAB_FILES_NAMES
     A_       							= PRETRAINED_VOCAB_FILES_MAP
     A_       							= PRETRAINED_INIT_CONFIGURATION
     A_       							= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     A_       							= LayoutLMTokenizer





     def __init__(    self    :				Optional[int]    ,       __lowerCamelCase    :				str=None    ,       __lowerCamelCase    :				Optional[Any]=None    ,       __lowerCamelCase    :				Union[str, Any]=True    ,       __lowerCamelCase    :				Optional[int]="[UNK]"    ,       __lowerCamelCase    :				Dict="[SEP]"    ,       __lowerCamelCase    :				List[str]="[PAD]"    ,       __lowerCamelCase    :				List[Any]="[CLS]"    ,       __lowerCamelCase    :				Any="[MASK]"    ,       __lowerCamelCase    :				int=True    ,       __lowerCamelCase    :				int=None    ,       **__lowerCamelCase    :				Dict    ,       ):
           super().__init__(
               __lowerCamelCase    ,       tokenizer_file=__lowerCamelCase    ,       do_lower_case=__lowerCamelCase    ,       unk_token=__lowerCamelCase    ,       sep_token=__lowerCamelCase    ,       pad_token=__lowerCamelCase    ,       cls_token=__lowerCamelCase    ,       mask_token=__lowerCamelCase    ,       tokenize_chinese_chars=__lowerCamelCase    ,       strip_accents=__lowerCamelCase    ,       **__lowerCamelCase    ,       )

           snake_case__					:      Optional[int]  			=						json.loads(self.backend_tokenizer.normalizer.__getstate__() )
           if (
               normalizer_state.get('lowercase'    ,       __lowerCamelCase ) != do_lower_case
               or normalizer_state.get('strip_accents'    ,       __lowerCamelCase ) != strip_accents
               or normalizer_state.get('handle_chinese_chars'    ,       __lowerCamelCase ) != tokenize_chinese_chars
           ):
                 snake_case__					:      Any  			=						getattr(__lowerCamelCase    ,       normalizer_state.pop('type' ) )
                 snake_case__					:      List[Any]  			=						do_lower_case
                 snake_case__					:      Optional[Any]  			=						strip_accents
                 snake_case__					:      Dict  			=						tokenize_chinese_chars
                 snake_case__					:      List[str]  			=						normalizer_class(**__lowerCamelCase )

           snake_case__					:      Dict  			=						do_lower_case





     def  _lowerCAmelCase			(    self    :				Optional[Any]    ,       __lowerCamelCase    :				Tuple    ,       __lowerCamelCase    :				str=None ):
           snake_case__					:      List[str]  			=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                 output += token_ids_a + [self.sep_token_id]

           return output





     def  _lowerCAmelCase			(    self    :				Tuple    ,       __lowerCamelCase    :				List[int]    ,       __lowerCamelCase    :				Optional[List[int]] = None ):
           snake_case__					:      Union[str, Any]  			=						[self.sep_token_id]
           snake_case__					:      str  			=						[self.cls_token_id]
           if token_ids_a is None:
                 return len(cls + token_ids_a + sep ) * [0]
           return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]





     def  _lowerCAmelCase			(    self    :				int    ,       __lowerCamelCase    :				str    ,       __lowerCamelCase    :				Optional[str] = None ):
           snake_case__					:      Tuple  			=						self._tokenizer.model.save(__lowerCamelCase    ,       name=__lowerCamelCase )
           return tuple(__lowerCamelCase )






'''simple docstring'''






from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_sentencepiece_available, is_torch_available


A_ 			=		{
    "configuration_ernie_m": ["ERNIE_M_PRETRAINED_CONFIG_ARCHIVE_MAP", "ErnieMConfig"],
}

try:
      if not is_sentencepiece_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      A_ 			=		["ErnieMTokenizer"]

try:
      if not is_torch_available():
            raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
      pass
else:
      A_ 			=		[
          "ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST",
          "ErnieMForMultipleChoice",
          "ErnieMForQuestionAnswering",
          "ErnieMForSequenceClassification",
          "ErnieMForTokenClassification",
          "ErnieMModel",
          "ErnieMPreTrainedModel",
          "ErnieMForInformationExtraction",
      ]


if TYPE_CHECKING:
      from .configuration_ernie_m import ERNIE_M_PRETRAINED_CONFIG_ARCHIVE_MAP, ErnieMConfig

      try:
            if not is_sentencepiece_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .tokenization_ernie_m import ErnieMTokenizer

      try:
            if not is_torch_available():
                  raise OptionalDependencyNotAvailable()
      except OptionalDependencyNotAvailable:
            pass
      else:
            from .modeling_ernie_m import (
                ERNIE_M_PRETRAINED_MODEL_ARCHIVE_LIST,
                ErnieMForInformationExtraction,
                ErnieMForMultipleChoice,
                ErnieMForQuestionAnswering,
                ErnieMForSequenceClassification,
                ErnieMForTokenClassification,
                ErnieMModel,
                ErnieMPreTrainedModel,
            )


else:
      import sys

      A_ 			=		_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)
