



import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


A__						:						List[str]          =	logging.get_logger(__name__)

A__						:						Any          =	{"""vocab_file""": """vocab.txt"""}

A__						:						Optional[int]          =	{
    """vocab_file""": {
        """tau/splinter-base""": """https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt""",
        """tau/splinter-base-qass""": """https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt""",
        """tau/splinter-large""": """https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt""",
        """tau/splinter-large-qass""": """https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt""",
    }
}

A__						:						str          =	{
    """tau/splinter-base""": 512,
    """tau/splinter-base-qass""": 512,
    """tau/splinter-large""": 512,
    """tau/splinter-large-qass""": 512,
}

A__						:						Optional[Any]          =	{
    """tau/splinter-base""": {"""do_lower_case""": False},
    """tau/splinter-base-qass""": {"""do_lower_case""": False},
    """tau/splinter-large""": {"""do_lower_case""": False},
    """tau/splinter-large-qass""": {"""do_lower_case""": False},
}



class        __snake_case		(  a_		):
							_a       =				VOCAB_FILES_NAMES
							_a       =				PRETRAINED_VOCAB_FILES_MAP
							_a       =				PRETRAINED_INIT_CONFIGURATION
							_a       =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_a       =				SplinterTokenizer




							def __init__( self  : str       ,       A_  : List[str]=None       ,       A_  : Tuple=None       ,       A_  : Optional[Any]=True       ,       A_  : int="[UNK]"       ,       A_  : int="[SEP]"       ,       A_  : List[str]="[PAD]"       ,       A_  : str="[CLS]"       ,       A_  : List[Any]="[MASK]"       ,       A_  : Tuple="[QUESTION]"       ,       A_  : Tuple=True       ,       A_  : Tuple=None       ,       **A_  : Optional[Any]       ,       ):
									super().__init__(
									    _snake_case       ,       tokenizer_file=_snake_case       ,       do_lower_case=_snake_case       ,       unk_token=_snake_case       ,       sep_token=_snake_case       ,       pad_token=_snake_case       ,       cls_token=_snake_case       ,       mask_token=_snake_case       ,       tokenize_chinese_chars=_snake_case       ,       strip_accents=_snake_case       ,       additional_special_tokens=(question_token,)       ,       **_snake_case       ,       )

									lowerCAmelCase_    :  Union[str, Any]             =				json.loads(self.backend_tokenizer.normalizer.__getstate__())
									if (
									    pre_tok_state.get('''lowercase'''       ,       _snake_case) != do_lower_case
									    or pre_tok_state.get('''strip_accents'''       ,       _snake_case) != strip_accents
									):
											lowerCAmelCase_    :  int             =				getattr(_snake_case       ,       pre_tok_state.pop('''type'''))
											lowerCAmelCase_    :  List[Any]             =				do_lower_case
											lowerCAmelCase_    :  Dict             =				strip_accents
											lowerCAmelCase_    :  Tuple             =				pre_tok_class(**_snake_case)

									lowerCAmelCase_    :  Optional[int]             =				do_lower_case




							@property
							def      UpperCAmelCase__    ( self  : Dict):
									return self.convert_tokens_to_ids(self.question_token)




							def      UpperCAmelCase__    ( self  : List[Any]       ,       A_  : int       ,       A_  : str = None):
									if token_ids_a is None:
											return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

									lowerCAmelCase_    :  Optional[int]             =				[self.cls_token_id]
									lowerCAmelCase_    :  Any             =				[self.sep_token_id]
									lowerCAmelCase_    :  List[str]             =				[self.question_token_id] + [self.convert_tokens_to_ids('''.''')]
									if self.padding_side == "right":
											# Input is question-then-context
											return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
									else:
											# Input is context-then-question
											return cls + token_ids_a + sep + token_ids_a + question_suffix + sep




							def      UpperCAmelCase__    ( self  : List[str]       ,       A_  : str       ,       A_  : Optional[int] = None):
									lowerCAmelCase_    :  Optional[int]             =				[self.sep_token_id]
									lowerCAmelCase_    :  Dict             =				[self.cls_token_id]
									lowerCAmelCase_    :  Any             =				[self.question_token_id] + [self.convert_tokens_to_ids('''.''')]
									if token_ids_a is None:
											return len(cls + token_ids_a + sep) * [0]

									if self.padding_side == "right":
											# Input is question-then-context
											return len(cls + token_ids_a + question_suffix + sep) * [0] + len(token_ids_a + sep) * [1]
									else:
											# Input is context-then-question
											return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + question_suffix + sep) * [1]







							def      UpperCAmelCase__    ( self  : Optional[int]       ,       A_  : Union[str, Any]       ,       A_  : Tuple = None):
									lowerCAmelCase_    :  Optional[Any]             =				self._tokenizer.model.save(_snake_case       ,       name=_snake_case)
									return tuple(_snake_case)










import warnings

from ...utils import logging
from .image_processing_detr import DetrImageProcessor


A__						:						str          =      logging.get_logger(__name__)



class        __snake_case		(  UpperCamelCase_		):




							def __init__( self  : Union[str, Any]       ,       *A_  : int       ,       **A_  : List[str]):
									warnings.warn(
									    '''The class DetrFeatureExtractor is deprecated and will be removed in version 5 of Transformers.'''
									    ''' Please use DetrImageProcessor instead.'''       ,       A_       ,       )
									super().__init__(*A_       ,       **A_)





