



"""simple docstring"""

from __future__ import annotations

from collections import deque
from enum import Enum
from math import atana, degrees
from sys import maxsize
class 						lowerCAmelCase__	(  _lowerCamelCase   ):


	'''simple docstring'''







	_SCREAMING_SNAKE_CASE  :		Optional[Any]     	=   1
	_SCREAMING_SNAKE_CASE  :		Tuple     	=   2
	_SCREAMING_SNAKE_CASE  :		Dict     	=   3





	def __repr__(			self							:					int						)	->  int:





					"""simple docstring"""


					return f"""{self.__class__.__name__}.{self.name}"""




def    __snake_case       (  __A			:	tuple[int, int]       ,		__A			:	int       ,		__A			:	int    )    ->					float:






				'''simple docstring'''
				SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE	:       Union[str, Any] 							=    point
				return degrees(atana(y - miny       ,		x - minx    )    )
def    __snake_case       (  __A			:	tuple[int, int]       ,		__A			:	tuple[int, int]       ,		__A			:	tuple[int, int]    )    ->					Direction:






				'''simple docstring'''
				SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE	:       Any 							=    starting
				SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE	:       str 							=    via
				SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE	:       List[Any] 							=    target
				SCREAMING_SNAKE_CASE	:       str 							=    degrees(atana(ya - ya       ,		xa - xa    )    )
				via_angle %= 360
				SCREAMING_SNAKE_CASE	:       Any 							=    degrees(atana(ya - ya       ,		xa - xa    )    )
				target_angle %= 360
				# t-
				#  \ \
				#   \ v
				#    \|
				#     s
				# via_angle is always lower than target_angle, if direction is left.
				# If they are same, it means they are on a same line of convex hull.
				if target_angle > via_angle:
								return Direction.left
				elif target_angle == via_angle:
								return Direction.straight
				else:
								return Direction.right







def    __snake_case       (  __A			:	list[tuple[int, int]]    )    ->					list[tuple[int, int]]:






				'''simple docstring'''

				if len(__A    ) <= 2:
								# There is no convex hull
								raise ValueError('graham_scan: argument must contain more than 3 points.'    )
				if len(__A    ) == 3:
								return points
				# find the lowest and the most left point
				SCREAMING_SNAKE_CASE	:       Dict 							=    0
				SCREAMING_SNAKE_CASE							,	SCREAMING_SNAKE_CASE	:       Any 							=    maxsize, maxsize
				for i, point in enumerate(__A    ):
								SCREAMING_SNAKE_CASE	:       Optional[int] 							=    point[0]
								SCREAMING_SNAKE_CASE	:       str 							=    point[1]
								if y < miny:
												SCREAMING_SNAKE_CASE	:       List[Any] 							=    y
												SCREAMING_SNAKE_CASE	:       Tuple 							=    x
												SCREAMING_SNAKE_CASE	:       Any 							=    i
								if y == miny and x < minx:
												SCREAMING_SNAKE_CASE	:       Dict 							=    x
												SCREAMING_SNAKE_CASE	:       Union[str, Any] 							=    i

    # remove the lowest and the most left point from points for preparing for sort
				points.pop(__A    )

				SCREAMING_SNAKE_CASE	:       Tuple 							=    sorted(__A       ,		key=lambda __A    : angle_comparer(__A       ,		__A       ,		__A    )    )
				# This insert actually costs complexity,
				# and you should instead add (minx, miny) into stack later.
				# I'm using insert just for easy understanding.
				sorted_points.insert(0       ,		(minx, miny)    )

				SCREAMING_SNAKE_CASE	:       deque[tuple[int, int]] 							=    deque()
				stack.append(sorted_points[0]    )
				stack.append(sorted_points[1]    )
				stack.append(sorted_points[2]    )
				# In any ways, the first 3 points line are towards left.
				# Because we sort them the angle from minx, miny.
				SCREAMING_SNAKE_CASE	:       List[str] 							=    Direction.left

				for i in range(3       ,		len(__A    )    ):
								while True:
												SCREAMING_SNAKE_CASE	:       Dict 							=    stack[-2]
												SCREAMING_SNAKE_CASE	:       Any 							=    stack[-1]
												SCREAMING_SNAKE_CASE	:       List[Any] 							=    sorted_points[i]
												SCREAMING_SNAKE_CASE	:       Tuple 							=    check_direction(__A       ,		__A       ,		__A    )

												if next_direction == Direction.left:
																SCREAMING_SNAKE_CASE	:       Union[str, Any] 							=    Direction.left
																break
												if next_direction == Direction.straight:
																if current_direction == Direction.left:
																				# We keep current_direction as left.
																				# Because if the straight line keeps as straight,
																				# we want to know if this straight line is towards left.
																				break
																elif current_direction == Direction.right:
																				# If the straight line is towards right,
																				# every previous points on those straigh line is not convex hull.
																				stack.pop()
												if next_direction == Direction.right:
																stack.pop()
								stack.append(sorted_points[i]    )
				return list(__A    )










"""simple docstring"""

import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_longformer import LongformerTokenizer


A_	:						Tuple				=			logging.get_logger(__name__)

A_	:						Optional[Any]				=			{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

A_	:						Any				=			{
    'vocab_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json'
        ),
    },
    'merges_file': {
        'allenai/longformer-base-4096': 'https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt',
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt'
        ),
    },
    'tokenizer_file': {
        'allenai/longformer-base-4096': (
            'https://huggingface.co/allenai/longformer-base-4096/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096': (
            'https://huggingface.co/allenai/longformer-large-4096/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096-finetuned-triviaqa': (
            'https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-base-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
        ),
        'allenai/longformer-large-4096-extra.pos.embd.only': (
            'https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/tokenizer.json'
        ),
    },
}

A_	:						int				=			{
    'allenai/longformer-base-4096': 4096,
    'allenai/longformer-large-4096': 4096,
    'allenai/longformer-large-4096-finetuned-triviaqa': 4096,
    'allenai/longformer-base-4096-extra.pos.embd.only': 4096,
    'allenai/longformer-large-4096-extra.pos.embd.only': 4096,
}
class 						lowerCAmelCase__	(  _lowerCamelCase   ):


	'''simple docstring'''







	_SCREAMING_SNAKE_CASE  :		Optional[int]     	=   VOCAB_FILES_NAMES
	_SCREAMING_SNAKE_CASE  :		List[Any]     	=   PRETRAINED_VOCAB_FILES_MAP
	_SCREAMING_SNAKE_CASE  :		int     	=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	_SCREAMING_SNAKE_CASE  :		Union[str, Any]     	=   ['''input_ids''', '''attention_mask''']
	_SCREAMING_SNAKE_CASE  :		Optional[int]     	=   LongformerTokenizer





	def __init__(			self							:					Optional[Any]    ,					_SCREAMING_SNAKE_CASE							:					Tuple=None    ,					_SCREAMING_SNAKE_CASE							:					Optional[Any]=None    ,					_SCREAMING_SNAKE_CASE							:					Any=None    ,					_SCREAMING_SNAKE_CASE							:					Union[str, Any]="replace"    ,					_SCREAMING_SNAKE_CASE							:					Optional[Any]="<s>"    ,					_SCREAMING_SNAKE_CASE							:					Dict="</s>"    ,					_SCREAMING_SNAKE_CASE							:					int="</s>"    ,					_SCREAMING_SNAKE_CASE							:					List[Any]="<s>"    ,					_SCREAMING_SNAKE_CASE							:					int="<unk>"    ,					_SCREAMING_SNAKE_CASE							:					Any="<pad>"    ,					_SCREAMING_SNAKE_CASE							:					Optional[int]="<mask>"    ,					_SCREAMING_SNAKE_CASE							:					Optional[Any]=False    ,					_SCREAMING_SNAKE_CASE							:					Dict=True    ,					**_SCREAMING_SNAKE_CASE							:					Tuple    ,					)	->  Optional[int]:





					"""simple docstring"""


					super().__init__(
					    _SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE    ,					tokenizer_file=_SCREAMING_SNAKE_CASE    ,					errors=_SCREAMING_SNAKE_CASE    ,					bos_token=_SCREAMING_SNAKE_CASE    ,					eos_token=_SCREAMING_SNAKE_CASE    ,					sep_token=_SCREAMING_SNAKE_CASE    ,					cls_token=_SCREAMING_SNAKE_CASE    ,					unk_token=_SCREAMING_SNAKE_CASE    ,					pad_token=_SCREAMING_SNAKE_CASE    ,					mask_token=_SCREAMING_SNAKE_CASE    ,					add_prefix_space=_SCREAMING_SNAKE_CASE    ,					trim_offsets=_SCREAMING_SNAKE_CASE    ,					**_SCREAMING_SNAKE_CASE    ,					)

					SCREAMING_SNAKE_CASE	:       List[str] 							=    json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()						)
					if pre_tok_state.get('add_prefix_space'    ,					_SCREAMING_SNAKE_CASE						) != add_prefix_space:
									SCREAMING_SNAKE_CASE	:       Dict 							=    getattr(_SCREAMING_SNAKE_CASE    ,					pre_tok_state.pop('type'						)						)
									SCREAMING_SNAKE_CASE	:       Any 							=    add_prefix_space
									SCREAMING_SNAKE_CASE	:       Optional[Any] 							=    pre_tok_class(**_SCREAMING_SNAKE_CASE						)

					SCREAMING_SNAKE_CASE	:       Dict 							=    add_prefix_space

					SCREAMING_SNAKE_CASE	:       List[str] 							=    'post_processor'
					SCREAMING_SNAKE_CASE	:       str 							=    getattr(self.backend_tokenizer    ,					_SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE						)
					if tokenizer_component_instance:
									SCREAMING_SNAKE_CASE	:       Any 							=    json.loads(tokenizer_component_instance.__getstate__()						)

									# The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
									if "sep" in state:
													SCREAMING_SNAKE_CASE	:       Optional[int] 							=    tuple(state['sep']						)
									if "cls" in state:
													SCREAMING_SNAKE_CASE	:       Tuple 							=    tuple(state['cls']						)

									SCREAMING_SNAKE_CASE	:       List[str] 							=    False

									if state.get('add_prefix_space'    ,					_SCREAMING_SNAKE_CASE						) != add_prefix_space:
													SCREAMING_SNAKE_CASE	:       Dict 							=    add_prefix_space
													SCREAMING_SNAKE_CASE	:       Dict 							=    True

									if state.get('trim_offsets'    ,					_SCREAMING_SNAKE_CASE						) != trim_offsets:
													SCREAMING_SNAKE_CASE	:       str 							=    trim_offsets
													SCREAMING_SNAKE_CASE	:       int 							=    True

									if changes_to_apply:
													SCREAMING_SNAKE_CASE	:       Union[str, Any] 							=    getattr(_SCREAMING_SNAKE_CASE    ,					state.pop('type'						)						)
													SCREAMING_SNAKE_CASE	:       Union[str, Any] 							=    component_class(**_SCREAMING_SNAKE_CASE						)
													setattr(self.backend_tokenizer    ,					_SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE						)





	@property
	def  _lowerCAmelCase    (			self							:					Dict						)	->  str:





					"""simple docstring"""


					if self._mask_token is None:
									if self.verbose:
													logger.error('Using mask_token, but it is not set yet.'						)
									return None
					return str(self._mask_token						)





	@mask_token.setter
	def  _lowerCAmelCase    (			self							:					Tuple    ,					_SCREAMING_SNAKE_CASE							:					str						)	->  Union[str, Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       Dict 							=    AddedToken(_SCREAMING_SNAKE_CASE    ,					lstrip=_SCREAMING_SNAKE_CASE    ,					rstrip=_SCREAMING_SNAKE_CASE						) if isinstance(_SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE						) else value
					SCREAMING_SNAKE_CASE	:       Any 							=    value





	def  _lowerCAmelCase    (			self							:					Optional[Any]    ,					*_SCREAMING_SNAKE_CASE							:					Optional[Any]    ,					**_SCREAMING_SNAKE_CASE							:					int						)	->  BatchEncoding:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       Union[str, Any] 							=    kwargs.get('is_split_into_words'    ,					_SCREAMING_SNAKE_CASE						)
					assert self.add_prefix_space or not is_split_into_words, (
					    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
					    "to use it with pretokenized inputs."
					)

					return super()._batch_encode_plus(*_SCREAMING_SNAKE_CASE    ,					**_SCREAMING_SNAKE_CASE						)





	def  _lowerCAmelCase    (			self							:					List[Any]    ,					*_SCREAMING_SNAKE_CASE							:					Optional[Any]    ,					**_SCREAMING_SNAKE_CASE							:					str						)	->  BatchEncoding:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       List[Any] 							=    kwargs.get('is_split_into_words'    ,					_SCREAMING_SNAKE_CASE						)

					assert self.add_prefix_space or not is_split_into_words, (
					    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
					    "to use it with pretokenized inputs."
					)

					return super()._encode_plus(*_SCREAMING_SNAKE_CASE    ,					**_SCREAMING_SNAKE_CASE						)





	def  _lowerCAmelCase    (			self							:					Any    ,					_SCREAMING_SNAKE_CASE							:					str    ,					_SCREAMING_SNAKE_CASE							:					Optional[str] = None						)	->  Tuple[str]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       Optional[int] 							=    self._tokenizer.model.save(_SCREAMING_SNAKE_CASE    ,					name=_SCREAMING_SNAKE_CASE						)
					return tuple(_SCREAMING_SNAKE_CASE						)





	def  _lowerCAmelCase    (			self							:					Tuple    ,					_SCREAMING_SNAKE_CASE							:					List[str]    ,					_SCREAMING_SNAKE_CASE							:					Dict=None						)	->  Union[str, Any]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       Optional[int] 							=    [self.bos_token_id] + token_ids_a + [self.eos_token_id]
					if token_ids_a is None:
									return output

					return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]







	def  _lowerCAmelCase    (			self							:					Dict    ,					_SCREAMING_SNAKE_CASE							:					List[int]    ,					_SCREAMING_SNAKE_CASE							:					Optional[List[int]] = None						)	->  List[int]:





					"""simple docstring"""


					SCREAMING_SNAKE_CASE	:       Tuple 							=    [self.sep_token_id]
					SCREAMING_SNAKE_CASE	:       Optional[int] 							=    [self.cls_token_id]

					if token_ids_a is None:
									return len(cls + token_ids_a + sep						) * [0]
					return len(cls + token_ids_a + sep + sep + token_ids_a + sep						) * [0]





