



import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
						from transformers.pipelines.conversational import Conversation

A__						:						Union[str, Any]          =      logging.get_logger(__name__)

A__						:						Any          =      {
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

A__						:						Optional[Any]          =      {
    '''vocab_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/vocab.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/vocab.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/vocab.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/vocab.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/merges.txt''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/merges.txt''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/merges.txt''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/merges.txt''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/merges.txt''',
    },
}

A__						:						Any          =      {
    '''gpt2''': 1024,
    '''gpt2-medium''': 1024,
    '''gpt2-large''': 1024,
    '''gpt2-xl''': 1024,
    '''distilgpt2''': 1024,
}


@lru_cache()
def     UpperCamelCase(						):
		lowerCAmelCase_    :  int             =				(
		    list(range(ord('''!'''				)  ,ord('''~'''				) + 1				)				) + list(range(ord('''¡'''				)  ,ord('''¬'''				) + 1				)				) + list(range(ord('''®'''				)  ,ord('''ÿ'''				) + 1				)				)
		)
		lowerCAmelCase_    :  str             =				bs[:]
		lowerCAmelCase_    :  Union[str, Any]             =				0
		for b in range(2**8				):
				if b not in bs:
						bs.append(__UpperCamelCase				)
						cs.append(2**8 + n				)
						n += 1
		lowerCAmelCase_    :  Tuple             =				[chr(__UpperCamelCase				) for n in cs]
		return dict(zip(__UpperCamelCase  ,__UpperCamelCase				)				)


def     UpperCamelCase(						__UpperCamelCase							:	Dict				):
		lowerCAmelCase_    :  str             =				set()
		lowerCAmelCase_    :  List[str]             =				word[0]
		for char in word[1:]:
				pairs.add((prev_char, char)				)
				lowerCAmelCase_    :  str             =				char
		return pairs





class        __snake_case		(  UpperCamelCase_		):
							_a       =				VOCAB_FILES_NAMES
							_a       =				PRETRAINED_VOCAB_FILES_MAP
							_a       =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_a       =				['''input_ids''', '''attention_mask''']




							def __init__( self  : Union[str, Any]       ,       A_  : Tuple       ,       A_  : Any       ,       A_  : str="replace"       ,       A_  : int="<|endoftext|>"       ,       A_  : Any="<|endoftext|>"       ,       A_  : Union[str, Any]="<|endoftext|>"       ,       A_  : str=None       ,       A_  : Tuple=False       ,       A_  : Tuple=False       ,       **A_  : Tuple       ,       ):
									lowerCAmelCase_    :  int             =				AddedToken(A_       ,       lstrip=A_       ,       rstrip=A_) if isinstance(A_       ,       A_) else bos_token
									lowerCAmelCase_    :  Optional[Any]             =				AddedToken(A_       ,       lstrip=A_       ,       rstrip=A_) if isinstance(A_       ,       A_) else eos_token
									lowerCAmelCase_    :  Union[str, Any]             =				AddedToken(A_       ,       lstrip=A_       ,       rstrip=A_) if isinstance(A_       ,       A_) else unk_token
									lowerCAmelCase_    :  List[str]             =				AddedToken(A_       ,       lstrip=A_       ,       rstrip=A_) if isinstance(A_       ,       A_) else pad_token
									super().__init__(
									    errors=A_       ,       unk_token=A_       ,       bos_token=A_       ,       eos_token=A_       ,       pad_token=A_       ,       add_prefix_space=A_       ,       add_bos_token=A_       ,       **A_       ,       )
									lowerCAmelCase_    :  Tuple             =				add_bos_token

									with open(A_       ,       encoding='''utf-8''') as vocab_handle:
											lowerCAmelCase_    :  str             =				json.load(A_)
									lowerCAmelCase_    :  Optional[int]             =				{v: k for k, v in self.encoder.items()}
									lowerCAmelCase_    :  Optional[Any]             =				errors  # how to handle errors in decoding
									lowerCAmelCase_    :  List[str]             =				bytes_to_unicode()
									lowerCAmelCase_    :  Tuple             =				{v: k for k, v in self.byte_encoder.items()}
									with open(A_       ,       encoding='''utf-8''') as merges_handle:
											lowerCAmelCase_    :  Union[str, Any]             =				merges_handle.read().split('''\n''')[1:-1]
									lowerCAmelCase_    :  int             =				[tuple(merge.split()) for merge in bpe_merges]
									lowerCAmelCase_    :  List[Any]             =				dict(zip(A_       ,       range(len(A_))))
									lowerCAmelCase_    :  int             =				{}
									lowerCAmelCase_    :  Tuple             =				add_prefix_space

									# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
									lowerCAmelCase_    :  Optional[int]             =				re.compile(r'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+''')




							@property
							def      UpperCAmelCase__    ( self  : Union[str, Any]):
									return len(self.encoder)




							def      UpperCAmelCase__    ( self  : Any):
									return dict(self.encoder       ,       **self.added_tokens_encoder)




							def      UpperCAmelCase__    ( self  : Union[str, Any]       ,       A_  : Optional[Any]):
									if token in self.cache:
											return self.cache[token]
									lowerCAmelCase_    :  Tuple             =				tuple(A_)
									lowerCAmelCase_    :  Optional[int]             =				get_pairs(A_)

									if not pairs:
											return token

									while True:
											lowerCAmelCase_    :  Dict             =				min(A_       ,       key=lambda A_: self.bpe_ranks.get(A_       ,       float('''inf''')))
											if bigram not in self.bpe_ranks:
													break
											lowerCAmelCase_      ,						lowerCAmelCase_    :  Any             =				bigram
											lowerCAmelCase_    :  Optional[Any]             =				[]
											lowerCAmelCase_    :  List[Any]             =				0
											while i < len(A_):
													try:
															lowerCAmelCase_    :  Union[str, Any]             =				word.index(A_       ,       A_)
													except ValueError:
															new_word.extend(word[i:])
															break
													else:
															new_word.extend(word[i:j])
															lowerCAmelCase_    :  Tuple             =				j

													if word[i] == first and i < len(A_) - 1 and word[i + 1] == second:
															new_word.append(first + second)
															i += 2
													else:
															new_word.append(word[i])
															i += 1
											lowerCAmelCase_    :  int             =				tuple(A_)
											lowerCAmelCase_    :  List[Any]             =				new_word
											if len(A_) == 1:
													break
											else:
													lowerCAmelCase_    :  str             =				get_pairs(A_)
									lowerCAmelCase_    :  Union[str, Any]             =				''' '''.join(A_)
									lowerCAmelCase_    :  int             =				word
									return word




							def      UpperCAmelCase__    ( self  : str       ,       A_  : str       ,       A_  : Union[str, Any]=None):
									if self.add_bos_token:
											lowerCAmelCase_    :  List[str]             =				[self.bos_token_id]
									else:
											lowerCAmelCase_    :  Dict             =				[]

									lowerCAmelCase_    :  Union[str, Any]             =				bos_token_ids + token_ids_a

									if token_ids_a is None:
											return output

									return output + bos_token_ids + token_ids_a




							def      UpperCAmelCase__    ( self  : List[Any]       ,       A_  : List[int]       ,       A_  : Optional[List[int]] = None       ,       A_  : bool = False):
									if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=A_       ,       token_ids_a=A_       ,       already_has_special_tokens=A_)

									if not self.add_bos_token:
											return super().get_special_tokens_mask(
											    token_ids_a=A_       ,       token_ids_a=A_       ,       already_has_special_tokens=A_)

									if token_ids_a is None:
											return [1] + ([0] * len(A_))
									return [1] + ([0] * len(A_)) + [1] + ([0] * len(A_))




							def      UpperCAmelCase__    ( self  : Optional[Any]       ,       A_  : Tuple):
									lowerCAmelCase_    :  Optional[Any]             =				[]
									for token in re.findall(self.pat       ,       A_):
											lowerCAmelCase_    :  Optional[Any]             =				''''''.join(
											    self.byte_encoder[b] for b in token.encode('''utf-8'''))  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
											bpe_tokens.extend(bpe_token for bpe_token in self.bpe(A_).split(''' '''))
									return bpe_tokens




							def      UpperCAmelCase__    ( self  : Union[str, Any]       ,       A_  : List[str]):
									return self.encoder.get(A_       ,       self.encoder.get(self.unk_token))




							def      UpperCAmelCase__    ( self  : Tuple       ,       A_  : List[str]):
									return self.decoder.get(A_)




							def      UpperCAmelCase__    ( self  : Optional[Any]       ,       A_  : Union[str, Any]):
									lowerCAmelCase_    :  Dict             =				''''''.join(A_)
									lowerCAmelCase_    :  Union[str, Any]             =				bytearray([self.byte_decoder[c] for c in text]).decode('''utf-8'''       ,       errors=self.errors)
									return text




							def      UpperCAmelCase__    ( self  : List[str]       ,       A_  : str       ,       A_  : Optional[str] = None):
									if not os.path.isdir(A_):
											logger.error(F"""Vocabulary path ({save_directory}) should be a directory""")
											return
									lowerCAmelCase_    :  List[str]             =				os.path.join(
									    A_       ,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])
									lowerCAmelCase_    :  Optional[int]             =				os.path.join(
									    A_       ,       (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file'''])

									with open(A_       ,       '''w'''       ,       encoding='''utf-8''') as f:
											f.write(json.dumps(self.encoder       ,       indent=2       ,       sort_keys=A_       ,       ensure_ascii=A_) + '''\n''')

									lowerCAmelCase_    :  Union[str, Any]             =				0
									with open(A_       ,       '''w'''       ,       encoding='''utf-8''') as writer:
											writer.write('''#version: 0.2\n''')
											for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,       key=lambda A_: kv[1]):
													if index != token_index:
															logger.warning(
															    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
															    ''' Please check that the tokenizer is not corrupted!''')
															lowerCAmelCase_    :  int             =				token_index
													writer.write(''' '''.join(A_) + '''\n''')
													index += 1

									return vocab_file, merge_file




							def      UpperCAmelCase__    ( self  : Dict       ,       A_  : Optional[Any]       ,       A_  : int=False       ,       **A_  : Optional[int]):
									lowerCAmelCase_    :  List[Any]             =				kwargs.pop('''add_prefix_space'''       ,       self.add_prefix_space)
									if is_split_into_words or add_prefix_space:
											lowerCAmelCase_    :  int             =				''' ''' + text
									return (text, kwargs)







							def      UpperCAmelCase__    ( self  : List[str]       ,       A_  : "Conversation"):
									lowerCAmelCase_    :  Any             =				[]
									for is_user, text in conversation.iter_texts():
											input_ids.extend(self.encode(A_       ,       add_special_tokens=A_) + [self.eos_token_id])
									if len(A_) > self.model_max_length:
											lowerCAmelCase_    :  Optional[int]             =				input_ids[-self.model_max_length :]
									return input_ids










from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


A__						:						List[str]          =      {
    '''configuration_segformer''': ['''SEGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''SegformerConfig''', '''SegformerOnnxConfig''']
}

try:
						if not is_vision_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A__						:						int          =      ['''SegformerFeatureExtractor''']
						A__						:						Dict          =      ['''SegformerImageProcessor''']

try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A__						:						Any          =      [
						    '''SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST''',
						    '''SegformerDecodeHead''',
						    '''SegformerForImageClassification''',
						    '''SegformerForSemanticSegmentation''',
						    '''SegformerLayer''',
						    '''SegformerModel''',
						    '''SegformerPreTrainedModel''',
						]

try:
						if not is_tf_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						A__						:						List[Any]          =      [
						    '''TF_SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST''',
						    '''TFSegformerDecodeHead''',
						    '''TFSegformerForImageClassification''',
						    '''TFSegformerForSemanticSegmentation''',
						    '''TFSegformerModel''',
						    '''TFSegformerPreTrainedModel''',
						]


if TYPE_CHECKING:
						from .configuration_segformer import SEGFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, SegformerConfig, SegformerOnnxConfig

						try:
												if not is_vision_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .feature_extraction_segformer import SegformerFeatureExtractor
												from .image_processing_segformer import SegformerImageProcessor

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_segformer import (
												    SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
												    SegformerDecodeHead,
												    SegformerForImageClassification,
												    SegformerForSemanticSegmentation,
												    SegformerLayer,
												    SegformerModel,
												    SegformerPreTrainedModel,
												)
						try:
												if not is_tf_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_tf_segformer import (
												    TF_SEGFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
												    TFSegformerDecodeHead,
												    TFSegformerForImageClassification,
												    TFSegformerForSemanticSegmentation,
												    TFSegformerModel,
												    TFSegformerPreTrainedModel,
												)

else:
						import sys

						A__						:						Dict          =      _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)





