

import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


__UpperCamelCase						:   Dict							   =	logging.get_logger(__name__)

__UpperCamelCase						:   Optional[int]							   =	{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

__UpperCamelCase						:   str							   =	{
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

__UpperCamelCase						:   List[Any]							   =	{
    '''openai-gpt''': 5_1_2,
}

def 						__SCREAMING_SNAKE_CASE						(   A_				):
	lowerCAmelCase__			:		int						       =			text.strip()
	if not text:
		return []
	lowerCAmelCase__			:		int						       =			text.split()
	return tokens







class        SCREAMING_SNAKE_CASE      (	UpperCamelCase_ ):






					"""simple docstring"""



					def __init__( self    :       Any     ,lowercase_    :       Optional[int]=True     ,lowercase_    :       Any=None     ,lowercase_    :       List[Any]=True     ,lowercase_    :       Tuple=None     ,lowercase_    :       str=True     ,):
						if never_split is None:
							lowerCAmelCase__			:		List[Any]						       =			[]
						lowerCAmelCase__			:		str						       =			do_lower_case
						lowerCAmelCase__			:		int						       =			set(__SCREAMING_SNAKE_CASE	)
						lowerCAmelCase__			:		Optional[int]						       =			tokenize_chinese_chars
						lowerCAmelCase__			:		Optional[Any]						       =			strip_accents
						lowerCAmelCase__			:		Optional[int]						       =			do_split_on_punc







					def  __lowerCAmelCase				( self    :       Tuple     ,lowercase_    :       int     ,lowercase_    :       List[Any]=None	):
						lowerCAmelCase__			:		Dict						       =			self.never_split.union(set(__SCREAMING_SNAKE_CASE	)	) if never_split else self.never_split
						lowerCAmelCase__			:		Union[str, Any]						       =			self._clean_text(__SCREAMING_SNAKE_CASE	)

						# This was added on November 1st, 2018 for the multilingual and Chinese
						# models. This is also applied to the English models now, but it doesn't
						# matter since the English models were not trained on any Chinese data
						# and generally don't have any Chinese data in them (there are Chinese
						# characters in the vocabulary because Wikipedia does have some Chinese
						# words in the English Wikipedia.).
						if self.tokenize_chinese_chars:
							lowerCAmelCase__			:		int						       =			self._tokenize_chinese_chars(__SCREAMING_SNAKE_CASE	)
						# prevents treating the same character with different unicode codepoints as different characters
						lowerCAmelCase__			:		Union[str, Any]						       =			unicodedata.normalize('''NFC'''     ,__SCREAMING_SNAKE_CASE	)
						lowerCAmelCase__			:		Any						       =			whitespace_tokenize(__SCREAMING_SNAKE_CASE	)
						lowerCAmelCase__			:		str						       =			[]
						for token in orig_tokens:
							if token not in never_split:
								if self.do_lower_case:
									lowerCAmelCase__			:		Optional[int]						       =			token.lower()
									if self.strip_accents is not False:
										lowerCAmelCase__			:		Optional[Any]						       =			self._run_strip_accents(__SCREAMING_SNAKE_CASE	)
								elif self.strip_accents:
									lowerCAmelCase__			:		int						       =			self._run_strip_accents(__SCREAMING_SNAKE_CASE	)
							split_tokens.extend(self._run_split_on_punc(__SCREAMING_SNAKE_CASE     ,__SCREAMING_SNAKE_CASE	)	)

						lowerCAmelCase__			:		Union[str, Any]						       =			whitespace_tokenize(''' '''.join(__SCREAMING_SNAKE_CASE	)	)
						return output_tokens







					def  __lowerCAmelCase				( self    :       Tuple     ,lowercase_    :       str	):
						lowerCAmelCase__			:		Optional[int]						       =			unicodedata.normalize('''NFD'''     ,__SCREAMING_SNAKE_CASE	)
						lowerCAmelCase__			:		Optional[int]						       =			[]
						for char in text:
							lowerCAmelCase__			:		str						       =			unicodedata.category(__SCREAMING_SNAKE_CASE	)
							if cat == "Mn":
								continue
							output.append(__SCREAMING_SNAKE_CASE	)
						return "".join(__SCREAMING_SNAKE_CASE	)







					def  __lowerCAmelCase				( self    :       Any     ,lowercase_    :       str     ,lowercase_    :       Optional[int]=None	):
						if not self.do_split_on_punc or (never_split is not None and text in never_split):
							return [text]
						lowerCAmelCase__			:		List[Any]						       =			list(__SCREAMING_SNAKE_CASE	)
						lowerCAmelCase__			:		Dict						       =			0
						lowerCAmelCase__			:		List[Any]						       =			True
						lowerCAmelCase__			:		Optional[int]						       =			[]
						while i < len(__SCREAMING_SNAKE_CASE	):
							lowerCAmelCase__			:		Union[str, Any]						       =			chars[i]
							if _is_punctuation(__SCREAMING_SNAKE_CASE	):
								output.append([char]	)
								lowerCAmelCase__			:		Optional[int]						       =			True
							else:
								if start_new_word:
									output.append([]	)
								lowerCAmelCase__			:		Tuple						       =			False
								output[-1].append(__SCREAMING_SNAKE_CASE	)
							i += 1

						return ["".join(__SCREAMING_SNAKE_CASE	) for x in output]







					def  __lowerCAmelCase				( self    :       Optional[int]     ,lowercase_    :       Dict	):
						lowerCAmelCase__			:		List[str]						       =			[]
						for char in text:
							lowerCAmelCase__			:		Optional[int]						       =			ord(__SCREAMING_SNAKE_CASE	)
							if self._is_chinese_char(__SCREAMING_SNAKE_CASE	):
								output.append(''' '''	)
								output.append(__SCREAMING_SNAKE_CASE	)
								output.append(''' '''	)
							else:
								output.append(__SCREAMING_SNAKE_CASE	)
						return "".join(__SCREAMING_SNAKE_CASE	)







					def  __lowerCAmelCase				( self    :       Optional[Any]     ,lowercase_    :       Union[str, Any]	):
						# This defines a "chinese character" as anything in the CJK Unicode block:
						#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
						#
						# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
						# despite its name. The modern Korean Hangul alphabet is a different block,
						# as is Japanese Hiragana and Katakana. Those alphabets are used to write
						# space-separated words, so they are not treated specially and handled
						# like the all of the other languages.
						if (
						    (cp >= 0x4E_00 and cp <= 0x9F_FF)
						    or (cp >= 0x34_00 and cp <= 0x4D_BF)  #
						    or (cp >= 0x2_00_00 and cp <= 0x2_A6_DF)  #
						    or (cp >= 0x2_A7_00 and cp <= 0x2_B7_3F)  #
						    or (cp >= 0x2_B7_40 and cp <= 0x2_B8_1F)  #
						    or (cp >= 0x2_B8_20 and cp <= 0x2_CE_AF)  #
						    or (cp >= 0xF9_00 and cp <= 0xFA_FF)
						    or (cp >= 0x2_F8_00 and cp <= 0x2_FA_1F)  #
						):  #
							return True

						return False







					def  __lowerCAmelCase				( self    :       List[str]     ,lowercase_    :       Union[str, Any]	):
						lowerCAmelCase__			:		int						       =			[]
						for char in text:
							lowerCAmelCase__			:		Any						       =			ord(__SCREAMING_SNAKE_CASE	)
							if cp == 0 or cp == 0xFF_FD or _is_control(__SCREAMING_SNAKE_CASE	):
								continue
							if _is_whitespace(__SCREAMING_SNAKE_CASE	):
								output.append(''' '''	)
							else:
								output.append(__SCREAMING_SNAKE_CASE	)
						return "".join(__SCREAMING_SNAKE_CASE	)

def 						__SCREAMING_SNAKE_CASE						(   A_				):
	lowerCAmelCase__			:		Union[str, Any]						       =			set()
	lowerCAmelCase__			:		List[str]						       =			word[0]
	for char in word[1:]:
		pairs.add((prev_char, char)				)
		lowerCAmelCase__			:		Optional[Any]						       =			char
	return pairs

def 						__SCREAMING_SNAKE_CASE						(   A_				):
	lowerCAmelCase__			:		Tuple						       =			text.replace('''—'''		,  '''-'''				)
	lowerCAmelCase__			:		Union[str, Any]						       =			text.replace('''–'''		,  '''-'''				)
	lowerCAmelCase__			:		List[Any]						       =			text.replace('''―'''		,  '''-'''				)
	lowerCAmelCase__			:		List[str]						       =			text.replace('''…'''		,  '''...'''				)
	lowerCAmelCase__			:		List[str]						       =			text.replace('''´'''		,  '''\''''				)
	lowerCAmelCase__			:		int						       =			re.sub(r'''(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'''		,  r''' \1 '''		,  snake_case__				)
	lowerCAmelCase__			:		Dict						       =			re.sub(r'''\s*\n\s*'''		,  ''' \n '''		,  snake_case__				)
	lowerCAmelCase__			:		List[Any]						       =			re.sub(r'''[^\S\n]+'''		,  ''' '''		,  snake_case__				)
	return text.strip()







class        SCREAMING_SNAKE_CASE      (	UpperCamelCase_ ):






					"""simple docstring"""



					lowercase__       				= VOCAB_FILES_NAMES
					lowercase__       				= PRETRAINED_VOCAB_FILES_MAP
					lowercase__       				= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowercase__       				= ["""input_ids""", """attention_mask"""]







					def __init__( self    :       Any     ,lowercase_    :       List[str]     ,lowercase_    :       Optional[Any]     ,lowercase_    :       List[str]="<unk>"     ,**lowercase_    :       str	):
						super().__init__(unk_token=__SCREAMING_SNAKE_CASE     ,**__SCREAMING_SNAKE_CASE	)

						try:
							import ftfy
							from spacy.lang.en import English

							lowerCAmelCase__			:		Any						       =			English()
							lowerCAmelCase__			:		List[str]						       =			_nlp.tokenizer
							lowerCAmelCase__			:		int						       =			ftfy.fix_text
						except ImportError:
							logger.warning('''ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.'''	)
							lowerCAmelCase__			:		Any						       =			BasicTokenizer(do_lower_case=__SCREAMING_SNAKE_CASE	)
							lowerCAmelCase__			:		List[Any]						       =			None

						with open(__SCREAMING_SNAKE_CASE     ,encoding='''utf-8'''	) as vocab_handle:
							lowerCAmelCase__			:		Dict						       =			json.load(__SCREAMING_SNAKE_CASE	)
						lowerCAmelCase__			:		Union[str, Any]						       =			{v: k for k, v in self.encoder.items()}
						with open(__SCREAMING_SNAKE_CASE     ,encoding='''utf-8'''	) as merges_handle:
							lowerCAmelCase__			:		int						       =			merges_handle.read().split('''\n'''	)[1:-1]
						lowerCAmelCase__			:		List[Any]						       =			[tuple(merge.split()	) for merge in merges]
						lowerCAmelCase__			:		int						       =			dict(zip(__SCREAMING_SNAKE_CASE     ,range(len(__SCREAMING_SNAKE_CASE	)	)	)	)
						lowerCAmelCase__			:		Any						       =			{}







					@property
					def  __lowerCAmelCase				( self    :       List[Any]	):
						return True







					@property
					def  __lowerCAmelCase				( self    :       Tuple	):
						return len(self.encoder	)







					def  __lowerCAmelCase				( self    :       Tuple	):
						return dict(self.encoder     ,**self.added_tokens_encoder	)







					def  __lowerCAmelCase				( self    :       List[str]     ,lowercase_    :       List[Any]	):
						lowerCAmelCase__			:		List[Any]						       =			tuple(token[:-1]	) + (token[-1] + '''</w>''',)
						if token in self.cache:
							return self.cache[token]
						lowerCAmelCase__			:		int						       =			get_pairs(__SCREAMING_SNAKE_CASE	)

						if not pairs:
							return token + "</w>"

						while True:
							lowerCAmelCase__			:		Optional[int]						       =			min(__SCREAMING_SNAKE_CASE     ,key=lambda lowercase_	: self.bpe_ranks.get(__SCREAMING_SNAKE_CASE     ,float('''inf'''	)	)	)
							if bigram not in self.bpe_ranks:
								break
							lowerCAmelCase__       ,lowerCAmelCase__			:		Optional[Any]						       =			bigram
							lowerCAmelCase__			:		List[Any]						       =			[]
							lowerCAmelCase__			:		int						       =			0
							while i < len(__SCREAMING_SNAKE_CASE	):
								try:
									lowerCAmelCase__			:		Optional[int]						       =			word.index(__SCREAMING_SNAKE_CASE     ,__SCREAMING_SNAKE_CASE	)
								except ValueError:
									new_word.extend(word[i:]	)
									break
								else:
									new_word.extend(word[i:j]	)
									lowerCAmelCase__			:		List[Any]						       =			j

								if word[i] == first and i < len(__SCREAMING_SNAKE_CASE	) - 1 and word[i + 1] == second:
									new_word.append(first + second	)
									i += 2
								else:
									new_word.append(word[i]	)
									i += 1
							lowerCAmelCase__			:		str						       =			tuple(__SCREAMING_SNAKE_CASE	)
							lowerCAmelCase__			:		Dict						       =			new_word
							if len(__SCREAMING_SNAKE_CASE	) == 1:
								break
							else:
								lowerCAmelCase__			:		Tuple						       =			get_pairs(__SCREAMING_SNAKE_CASE	)
						lowerCAmelCase__			:		Any						       =			''' '''.join(__SCREAMING_SNAKE_CASE	)
						if word == "\n  </w>":
							lowerCAmelCase__			:		Tuple						       =			'''\n</w>'''
						lowerCAmelCase__			:		List[Any]						       =			word
						return word







					def  __lowerCAmelCase				( self    :       Tuple     ,lowercase_    :       int	):
						lowerCAmelCase__			:		int						       =			[]
						if self.fix_text is None:
							# Using BERT's BasicTokenizer
							lowerCAmelCase__			:		Any						       =			self.nlp.tokenize(__SCREAMING_SNAKE_CASE	)
							for token in text:
								split_tokens.extend(list(self.bpe(__SCREAMING_SNAKE_CASE	).split(''' '''	)	)	)
						else:
							# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
							lowerCAmelCase__			:		Tuple						       =			self.nlp(text_standardize(self.fix_text(__SCREAMING_SNAKE_CASE	)	)	)
							for token in text:
								split_tokens.extend(list(self.bpe(token.text.lower()	).split(''' '''	)	)	)
						return split_tokens







					def  __lowerCAmelCase				( self    :       str     ,lowercase_    :       Tuple	):
						return self.encoder.get(__SCREAMING_SNAKE_CASE     ,self.encoder.get(self.unk_token	)	)







					def  __lowerCAmelCase				( self    :       Any     ,lowercase_    :       Optional[Any]	):
						return self.decoder.get(__SCREAMING_SNAKE_CASE     ,self.unk_token	)







					def  __lowerCAmelCase				( self    :       Union[str, Any]     ,lowercase_    :       Any	):
						lowerCAmelCase__			:		Dict						       =			''''''.join(__SCREAMING_SNAKE_CASE	).replace('''</w>'''     ,''' '''	).strip()
						return out_string







					def  __lowerCAmelCase				( self    :       Any     ,lowercase_    :       int     ,lowercase_    :       List[Any] = None	):
						if not os.path.isdir(__SCREAMING_SNAKE_CASE	):
							logger.error(F'Vocabulary path ({save_directory}) should be a directory'	)
							return
						lowerCAmelCase__			:		Tuple						       =			os.path.join(
						    __SCREAMING_SNAKE_CASE     ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)
						lowerCAmelCase__			:		Any						       =			os.path.join(
						    __SCREAMING_SNAKE_CASE     ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']	)

						with open(__SCREAMING_SNAKE_CASE     ,'''w'''     ,encoding='''utf-8'''	) as f:
							f.write(json.dumps(self.encoder     ,indent=2     ,sort_keys=__SCREAMING_SNAKE_CASE     ,ensure_ascii=__SCREAMING_SNAKE_CASE	) + '''\n'''	)

						lowerCAmelCase__			:		List[Any]						       =			0
						with open(__SCREAMING_SNAKE_CASE     ,'''w'''     ,encoding='''utf-8'''	) as writer:
							writer.write('''#version: 0.2\n'''	)
							for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,key=lambda lowercase_	: kv[1]	):
								if index != token_index:
									logger.warning(
									    F'Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'
									    ''' Please check that the tokenizer is not corrupted!'''	)
									lowerCAmelCase__			:		Optional[Any]						       =			token_index
								writer.write(''' '''.join(__SCREAMING_SNAKE_CASE	) + '''\n'''	)
								index += 1

						return vocab_file, merge_file







"""simple docstring"""





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__UpperCamelCase						:   Union[str, Any]							   =	{
    '''configuration_layoutlm''': ['''LAYOUTLM_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''LayoutLMConfig''', '''LayoutLMOnnxConfig'''],
    '''tokenization_layoutlm''': ['''LayoutLMTokenizer'''],
}

try:
							if not is_tokenizers_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__UpperCamelCase						:   Union[str, Any]							   =	['''LayoutLMTokenizerFast''']

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__UpperCamelCase						:   Dict							   =	[
							    '''LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''LayoutLMForMaskedLM''',
							    '''LayoutLMForSequenceClassification''',
							    '''LayoutLMForTokenClassification''',
							    '''LayoutLMForQuestionAnswering''',
							    '''LayoutLMModel''',
							    '''LayoutLMPreTrainedModel''',
							]

try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__UpperCamelCase						:   str							   =	[
							    '''TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''TFLayoutLMForMaskedLM''',
							    '''TFLayoutLMForSequenceClassification''',
							    '''TFLayoutLMForTokenClassification''',
							    '''TFLayoutLMForQuestionAnswering''',
							    '''TFLayoutLMMainLayer''',
							    '''TFLayoutLMModel''',
							    '''TFLayoutLMPreTrainedModel''',
							]


if TYPE_CHECKING:
							from .configuration_layoutlm import LAYOUTLM_PRETRAINED_CONFIG_ARCHIVE_MAP, LayoutLMConfig, LayoutLMOnnxConfig
							from .tokenization_layoutlm import LayoutLMTokenizer

							try:
														if not is_tokenizers_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_layoutlm_fast import LayoutLMTokenizerFast

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_layoutlm import (
														    LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST,
														    LayoutLMForMaskedLM,
														    LayoutLMForQuestionAnswering,
														    LayoutLMForSequenceClassification,
														    LayoutLMForTokenClassification,
														    LayoutLMModel,
														    LayoutLMPreTrainedModel,
														)
							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_layoutlm import (
														    TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFLayoutLMForMaskedLM,
														    TFLayoutLMForQuestionAnswering,
														    TFLayoutLMForSequenceClassification,
														    TFLayoutLMForTokenClassification,
														    TFLayoutLMMainLayer,
														    TFLayoutLMModel,
														    TFLayoutLMPreTrainedModel,
														)

else:
							import sys

							__UpperCamelCase						:   int							   =	_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




