


"""simple docstring"""



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


lowercase_							 =       logging.get_logger(__name__)

lowercase_							 =       {'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowercase_							 =       {
    'vocab_file': {
        'microsoft/layoutlm-base-uncased': (
            'https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt'
        ),
        'microsoft/layoutlm-large-uncased': (
            'https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt'
        ),
    },
    'tokenizer_file': {
        'microsoft/layoutlm-base-uncased': (
            'https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json'
        ),
        'microsoft/layoutlm-large-uncased': (
            'https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json'
        ),
    },
}

lowercase_							 =       {
    'microsoft/layoutlm-base-uncased': 5_12,
    'microsoft/layoutlm-large-uncased': 5_12,
}

lowercase_							 =       {
    'microsoft/layoutlm-base-uncased': {'do_lower_case': True},
    'microsoft/layoutlm-large-uncased': {'do_lower_case': True},
}







class       __a	(   __snake_case       ):
				lowerCamelCase    :					Union[str, Any] 					=VOCAB_FILES_NAMES
				lowerCamelCase    :					str 					=PRETRAINED_VOCAB_FILES_MAP
				lowerCamelCase    :					Any 					=PRETRAINED_INIT_CONFIGURATION
				lowerCamelCase    :					Union[str, Any] 					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCamelCase    :					Optional[Any] 					=LayoutLMTokenizer







				def __init__(   self			,  UpperCAmelCase=None			,  UpperCAmelCase=None			,  UpperCAmelCase=True			,  UpperCAmelCase="[UNK]"			,  UpperCAmelCase="[SEP]"			,  UpperCAmelCase="[PAD]"			,  UpperCAmelCase="[CLS]"			,  UpperCAmelCase="[MASK]"			,  UpperCAmelCase=True			,  UpperCAmelCase=None			,  **UpperCAmelCase			,  ):

					'''simple docstring'''






					super().__init__(
					    UpperCAmelCase			,  tokenizer_file=UpperCAmelCase			,  do_lower_case=UpperCAmelCase			,  unk_token=UpperCAmelCase			,  sep_token=UpperCAmelCase			,  pad_token=UpperCAmelCase			,  cls_token=UpperCAmelCase			,  mask_token=UpperCAmelCase			,  tokenize_chinese_chars=UpperCAmelCase			,  strip_accents=UpperCAmelCase			,  **UpperCAmelCase			,  )

					lowerCAmelCase_			       = json.loads(self.backend_tokenizer.normalizer.__getstate__()      )
					if (
					    normalizer_state.get('''lowercase'''			,  UpperCAmelCase      ) != do_lower_case
					    or normalizer_state.get('''strip_accents'''			,  UpperCAmelCase      ) != strip_accents
					    or normalizer_state.get('''handle_chinese_chars'''			,  UpperCAmelCase      ) != tokenize_chinese_chars
					):
						lowerCAmelCase_			       = getattr(UpperCAmelCase			,  normalizer_state.pop('''type'''      )      )
						lowerCAmelCase_			       = do_lower_case
						lowerCAmelCase_			       = strip_accents
						lowerCAmelCase_			       = tokenize_chinese_chars
						lowerCAmelCase_			       = normalizer_class(**UpperCAmelCase      )

					lowerCAmelCase_			       = do_lower_case







				def     lowerCamelCase_    (   self			,  UpperCAmelCase			,  UpperCAmelCase=None      ):

					'''simple docstring'''






					lowerCAmelCase_			       = [self.cls_token_id] + token_ids_a + [self.sep_token_id]

					if token_ids_a:
						output += token_ids_a + [self.sep_token_id]

					return output







				def     lowerCamelCase_    (   self			,  UpperCAmelCase			,  UpperCAmelCase = None      ):

					'''simple docstring'''






					lowerCAmelCase_			       = [self.sep_token_id]
					lowerCAmelCase_			       = [self.cls_token_id]
					if token_ids_a is None:
						return len(cls + token_ids_a + sep      ) * [0]
					return len(cls + token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1]







				def     lowerCamelCase_    (   self			,  UpperCAmelCase			,  UpperCAmelCase = None      ):

					'''simple docstring'''






					lowerCAmelCase_			       = self._tokenizer.model.save(UpperCAmelCase			,  name=UpperCAmelCase      )
					return tuple(UpperCAmelCase      )



"""simple docstring"""



lowercase_							 =       [0.485, 0.456, 0.406]
lowercase_							 =       [0.229, 0.224, 0.225]
lowercase_							 =       [0.5, 0.5, 0.5]
lowercase_							 =       [0.5, 0.5, 0.5]
lowercase_							 =       [0.4814_5466, 0.457_8275, 0.4082_1073]
lowercase_							 =       [0.2686_2954, 0.2613_0258, 0.2757_7711]