




'''simple docstring'''
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_layoutlm import LayoutLMTokenizer


_SCREAMING_SNAKE_CASE =	logging.get_logger(__name__)

_SCREAMING_SNAKE_CASE =	{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_SCREAMING_SNAKE_CASE =	{
    "vocab_file": {
        "microsoft/layoutlm-base-uncased": (
            "https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/vocab.txt"
        ),
        "microsoft/layoutlm-large-uncased": (
            "https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "microsoft/layoutlm-base-uncased": (
            "https://huggingface.co/microsoft/layoutlm-base-uncased/resolve/main/tokenizer.json"
        ),
        "microsoft/layoutlm-large-uncased": (
            "https://huggingface.co/microsoft/layoutlm-large-uncased/resolve/main/tokenizer.json"
        ),
    },
}

_SCREAMING_SNAKE_CASE =	{
    "microsoft/layoutlm-base-uncased": 5_12,
    "microsoft/layoutlm-large-uncased": 5_12,
}

_SCREAMING_SNAKE_CASE =	{
    "microsoft/layoutlm-base-uncased": {"do_lower_case": True},
    "microsoft/layoutlm-large-uncased": {"do_lower_case": True},
}
class      __UpperCAmelCase							(    lowerCAmelCase							):



					'''simple docstring'''





					_UpperCamelCase  	=		VOCAB_FILES_NAMES
					_UpperCamelCase  	=		PRETRAINED_VOCAB_FILES_MAP
					_UpperCamelCase  	=		PRETRAINED_INIT_CONFIGURATION
					_UpperCamelCase  	=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					_UpperCamelCase  	=		LayoutLMTokenizer



					def __init__(  self    :	Tuple			,					_lowercase    :	Any=None			,					_lowercase    :	Optional[Any]=None			,					_lowercase    :	Union[str, Any]=True			,					_lowercase    :	Any="[UNK]"			,					_lowercase    :	Any="[SEP]"			,					_lowercase    :	List[Any]="[PAD]"			,					_lowercase    :	List[Any]="[CLS]"			,					_lowercase    :	List[str]="[MASK]"			,					_lowercase    :	Optional[Any]=True			,					_lowercase    :	Tuple=None			,					**_lowercase    :	Union[str, Any]			,					)						->       Union[str, Any]:
								super().__init__(
								    _lowercase			,					tokenizer_file=_lowercase			,					do_lower_case=_lowercase			,					unk_token=_lowercase			,					sep_token=_lowercase			,					pad_token=_lowercase			,					cls_token=_lowercase			,					mask_token=_lowercase			,					tokenize_chinese_chars=_lowercase			,					strip_accents=_lowercase			,					**_lowercase			,					)

								A_       		=   json.loads(self.backend_tokenizer.normalizer.__getstate__())
								if (
								    normalizer_state.get('lowercase'			,					_lowercase) != do_lower_case
								    or normalizer_state.get('strip_accents'			,					_lowercase) != strip_accents
								    or normalizer_state.get('handle_chinese_chars'			,					_lowercase) != tokenize_chinese_chars
								):
											A_       		=   getattr(_lowercase			,					normalizer_state.pop('type'))
											A_       		=   do_lower_case
											A_       		=   strip_accents
											A_       		=   tokenize_chinese_chars
											A_       		=   normalizer_class(**_lowercase)

								A_       		=   do_lower_case



					def 						__snake_case	(  self    :	str			,					_lowercase    :	str			,					_lowercase    :	Tuple=None)						->       int:
								A_       		=   [self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
											output += token_ids_a + [self.sep_token_id]

								return output



					def 						__snake_case	(  self    :	str			,					_lowercase    :	List[int]			,					_lowercase    :	Optional[List[int]] = None)						->       List[int]:
								A_       		=   [self.sep_token_id]
								A_       		=   [self.cls_token_id]
								if token_ids_a is None:
											return len(cls + token_ids_a + sep) * [0]
								return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]



					def 						__snake_case	(  self    :	List[str]			,					_lowercase    :	str			,					_lowercase    :	Optional[str] = None)						->       Tuple[str]:
								A_       		=   self._tokenizer.model.save(_lowercase			,					name=_lowercase)
								return tuple(_lowercase)










'''simple docstring'''
from typing import Dict, List, Optional, Tuple

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_SCREAMING_SNAKE_CASE =	logging.get_logger(__name__)
class      __UpperCAmelCase							(    lowerCAmelCase							):



					'''simple docstring'''





					_UpperCamelCase  	=		["""input_ids""", """attention_mask"""]



					def __init__(  self    :	str			,					_lowercase    :	List[Any]="[PAD]"			,					_lowercase    :	Any="[BOS]"			,					_lowercase    :	Any="[EOS]"			,					_lowercase    :	str="[MASK]"			,					_lowercase    :	str="[CLS]"			,					_lowercase    :	Optional[Any]="[SEP]"			,					_lowercase    :	List[str]=2_048			,					**_lowercase    :	int			,					)						->       None:
								A_       		=   AddedToken(_lowercase			,					lstrip=_lowercase			,					rstrip=_lowercase) if isinstance(_lowercase			,					_lowercase) else pad_token
								A_       		=   AddedToken(_lowercase			,					lstrip=_lowercase			,					rstrip=_lowercase) if isinstance(_lowercase			,					_lowercase) else bos_token
								A_       		=   AddedToken(_lowercase			,					lstrip=_lowercase			,					rstrip=_lowercase) if isinstance(_lowercase			,					_lowercase) else eos_token
								A_       		=   AddedToken(_lowercase			,					lstrip=_lowercase			,					rstrip=_lowercase) if isinstance(_lowercase			,					_lowercase) else mask_token
								A_       		=   AddedToken(_lowercase			,					lstrip=_lowercase			,					rstrip=_lowercase) if isinstance(_lowercase			,					_lowercase) else cls_token
								A_       		=   AddedToken(_lowercase			,					lstrip=_lowercase			,					rstrip=_lowercase) if isinstance(_lowercase			,					_lowercase) else sep_token

								super().__init__(
								    pad_token=_lowercase			,					bos_token=_lowercase			,					eos_token=_lowercase			,					mask_token=_lowercase			,					cls_token=_lowercase			,					sep_token=_lowercase			,					model_max_length=_lowercase			,					**_lowercase			,					)

								A_       		=   2**8  # utf is 8 bits

								# define special tokens dict
								A_       		=   {
								    self.pad_token: 0,
								    self.bos_token: 1,
								    self.eos_token: 2,
								    self.mask_token: 3,
								    self.cls_token: 4,
								    self.sep_token: 5,
								}
								A_       		=   len(self.special_tokens_encoder)
								A_       		=   {v: k for k, v in self.special_tokens_encoder.items()}



					def 						__snake_case	(  self    :	List[str])						->       Dict[str, int]:
								A_       		=   self.special_tokens_encoder.copy()
								vocab.update(self.added_tokens_encoder)
								for i in range(self._utf_vocab_size):
											A_       		=   chr(_lowercase)
											A_       		=   i + len(self.special_tokens_encoder)
								return vocab



					@property
					def 						__snake_case	(  self    :	int)						->       Dict:
								return self._utf_vocab_size + self._num_special_tokens



					def 						__snake_case	(  self    :	Union[str, Any]			,					_lowercase    :	List[int]			,					_lowercase    :	Optional[List[int]] = None			,					_lowercase    :	bool = False)						->       List[int]:
								if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=_lowercase			,					token_ids_a=_lowercase			,					already_has_special_tokens=_lowercase)

								# normal case: some special tokens
								if token_ids_a is None:
											return [1] + [0] * len(_lowercase) + [1]
								return [1] + ([0] * len(_lowercase)) + [1] + ([0] * len(_lowercase)) + [1]



					def 						__snake_case	(  self    :	int			,					_lowercase    :	List[int]			,					_lowercase    :	Optional[List[int]] = None)						->       List[int]:
								if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
								else:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id] + token_ids_a + [self.sep_token_id]



					def 						__snake_case	(  self    :	str			,					_lowercase    :	str)						->       List[str]:
								A_       		=   [chr(_lowercase) for i in text.encode('utf-8')]
								return tokens



					def 						__snake_case	(  self    :	Union[str, Any]			,					_lowercase    :	Optional[Any])						->       Tuple:
								if token in self.special_tokens_encoder:
											A_       		=   self.special_tokens_encoder[token]
								elif token in self.added_tokens_encoder:
											A_       		=   self.added_tokens_encoder[token]
								elif len(_lowercase) != 1:
											A_       		=   self.unk_token_id
								else:
											A_       		=   ord(_lowercase) + self._num_special_tokens
								return token_id



					def 						__snake_case	(  self    :	str			,					_lowercase    :	List[str])						->       Dict:
								if index in self.special_tokens_decoder:
											A_       		=   self.special_tokens_decoder[index]
								elif index in self.added_tokens_decoder:
											A_       		=   self.added_tokens_decoder[index]
								else:
											A_       		=   chr(index - self._num_special_tokens)
								return token



					def 						__snake_case	(  self    :	str			,					_lowercase    :	Dict)						->       int:
								A_       		=   B''
								for token in tokens:
											if token in self.special_tokens_decoder:
														A_       		=   self.special_tokens_decoder[token].encode('utf-8')
											elif token in self.added_tokens_decoder:
														A_       		=   self.special_tokens_decoder[token].encode('utf-8')
											elif token in self.special_tokens_encoder:
														A_       		=   token.encode('utf-8')
											elif token in self.added_tokens_encoder:
														A_       		=   token.encode('utf-8')
											else:
														A_       		=   bytes([ord(_lowercase)])
											bstring += tok_string
								A_       		=   bstring.decode('utf-8'			,					errors='replace')
								return string



					def 						__snake_case	(  self    :	Optional[int]			,					_lowercase    :	str			,					_lowercase    :	Optional[str] = None)						->       Tuple[str]:
								return ()




