
"""simple docstring"""


import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


__A 		= logging.get_logger(__name__)

__A 		= {
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

__A 		= {
    'vocab_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/vocab.json'},
    'merges_file': {'openai-gpt': 'https://huggingface.co/openai-gpt/resolve/main/merges.txt'},
}

__A 		= {
    'openai-gpt': 512,
}



def 					_lowerCamelCase(__UpperCamelCase )       ->				Union[str, Any]:
	_lowerCAmelCase			=text.strip()
	if not text:
		return []
	_lowerCAmelCase			=text.split()
	return tokens







class 	lowerCamelCase__      (      A__       ):





			'''simple docstring'''



			def __init__(							self  ,  __UpperCAmelCase=True  ,  __UpperCAmelCase=None  ,  __UpperCAmelCase=True  ,  __UpperCAmelCase=None  ,  __UpperCAmelCase=True  ,  )				->  Optional[Any]:
				if never_split is None:
					_lowerCAmelCase			=[]
				_lowerCAmelCase			=do_lower_case
				_lowerCAmelCase			=set(_lowerCAmelCase						)
				_lowerCAmelCase			=tokenize_chinese_chars
				_lowerCAmelCase			=strip_accents
				_lowerCAmelCase			=do_split_on_punc



			def       _lowerCAmelCase	(							self  ,  __UpperCAmelCase  ,  __UpperCAmelCase=None						)				->  Dict:
				_lowerCAmelCase			=self.never_split.union(set(_lowerCAmelCase						)						) if never_split else self.never_split
				_lowerCAmelCase			=self._clean_text(_lowerCAmelCase						)

				# This was added on November 1st, 2018 for the multilingual and Chinese
				# models. This is also applied to the English models now, but it doesn't
				# matter since the English models were not trained on any Chinese data
				# and generally don't have any Chinese data in them (there are Chinese
				# characters in the vocabulary because Wikipedia does have some Chinese
				# words in the English Wikipedia.).
				if self.tokenize_chinese_chars:
					_lowerCAmelCase			=self._tokenize_chinese_chars(_lowerCAmelCase						)
				# prevents treating the same character with different unicode codepoints as different characters
				_lowerCAmelCase			=unicodedata.normalize("""NFC"""  ,  _lowerCAmelCase						)
				_lowerCAmelCase			=whitespace_tokenize(_lowerCAmelCase						)
				_lowerCAmelCase			=[]
				for token in orig_tokens:
					if token not in never_split:
						if self.do_lower_case:
							_lowerCAmelCase			=token.lower()
							if self.strip_accents is not False:
								_lowerCAmelCase			=self._run_strip_accents(_lowerCAmelCase						)
						elif self.strip_accents:
							_lowerCAmelCase			=self._run_strip_accents(_lowerCAmelCase						)
					split_tokens.extend(self._run_split_on_punc(_lowerCAmelCase  ,  _lowerCAmelCase						)						)

				_lowerCAmelCase			=whitespace_tokenize(""" """.join(_lowerCAmelCase						)						)
				return output_tokens



			def       _lowerCAmelCase	(							self  ,  __UpperCAmelCase						)				->  Optional[Any]:
				_lowerCAmelCase			=unicodedata.normalize("""NFD"""  ,  _lowerCAmelCase						)
				_lowerCAmelCase			=[]
				for char in text:
					_lowerCAmelCase			=unicodedata.category(_lowerCAmelCase						)
					if cat == "Mn":
						continue
					output.append(_lowerCAmelCase						)
				return "".join(_lowerCAmelCase						)



			def       _lowerCAmelCase	(							self  ,  __UpperCAmelCase  ,  __UpperCAmelCase=None						)				->  Union[str, Any]:
				if not self.do_split_on_punc or (never_split is not None and text in never_split):
					return [text]
				_lowerCAmelCase			=list(_lowerCAmelCase						)
				_lowerCAmelCase			=0
				_lowerCAmelCase			=True
				_lowerCAmelCase			=[]
				while i < len(_lowerCAmelCase						):
					_lowerCAmelCase			=chars[i]
					if _is_punctuation(_lowerCAmelCase						):
						output.append([char]						)
						_lowerCAmelCase			=True
					else:
						if start_new_word:
							output.append([]						)
						_lowerCAmelCase			=False
						output[-1].append(_lowerCAmelCase						)
					i += 1

				return ["".join(_lowerCAmelCase						) for x in output]



			def       _lowerCAmelCase	(							self  ,  __UpperCAmelCase						)				->  int:
				_lowerCAmelCase			=[]
				for char in text:
					_lowerCAmelCase			=ord(_lowerCAmelCase						)
					if self._is_chinese_char(_lowerCAmelCase						):
						output.append(""" """						)
						output.append(_lowerCAmelCase						)
						output.append(""" """						)
					else:
						output.append(_lowerCAmelCase						)
				return "".join(_lowerCAmelCase						)



			def       _lowerCAmelCase	(							self  ,  __UpperCAmelCase						)				->  str:
				# This defines a "chinese character" as anything in the CJK Unicode block:
				#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
				#
				# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
				# despite its name. The modern Korean Hangul alphabet is a different block,
				# as is Japanese Hiragana and Katakana. Those alphabets are used to write
				# space-separated words, so they are not treated specially and handled
				# like the all of the other languages.
				if (
				    (cp >= 0x4_e_0_0 and cp <= 0x9_f_f_f)
				    or (cp >= 0x3_4_0_0 and cp <= 0x4_d_b_f)  #
				    or (cp >= 0x2_0_0_0_0 and cp <= 0x2_a_6_d_f)  #
				    or (cp >= 0x2_a_7_0_0 and cp <= 0x2_b_7_3_f)  #
				    or (cp >= 0x2_b_7_4_0 and cp <= 0x2_b_8_1_f)  #
				    or (cp >= 0x2_b_8_2_0 and cp <= 0x2_c_e_a_f)  #
				    or (cp >= 0xf_9_0_0 and cp <= 0xf_a_f_f)
				    or (cp >= 0x2_f_8_0_0 and cp <= 0x2_f_a_1_f)  #
				):  #
					return True

				return False



			def       _lowerCAmelCase	(							self  ,  __UpperCAmelCase						)				->  int:
				_lowerCAmelCase			=[]
				for char in text:
					_lowerCAmelCase			=ord(_lowerCAmelCase						)
					if cp == 0 or cp == 0xf_f_f_d or _is_control(_lowerCAmelCase						):
						continue
					if _is_whitespace(_lowerCAmelCase						):
						output.append(""" """						)
					else:
						output.append(_lowerCAmelCase						)
				return "".join(_lowerCAmelCase						)







def 					_lowerCamelCase(__UpperCamelCase )       ->				str:
	_lowerCAmelCase			=set()
	_lowerCAmelCase			=word[0]
	for char in word[1:]:
		pairs.add((prev_char, char) )
		_lowerCAmelCase			=char
	return pairs



def 					_lowerCamelCase(__UpperCamelCase )       ->				List[str]:
	_lowerCAmelCase			=text.replace("""—"""					,				"""-""" )
	_lowerCAmelCase			=text.replace("""–"""					,				"""-""" )
	_lowerCAmelCase			=text.replace("""―"""					,				"""-""" )
	_lowerCAmelCase			=text.replace("""…"""					,				"""...""" )
	_lowerCAmelCase			=text.replace("""´"""					,				"""'""" )
	_lowerCAmelCase			=re.sub(R"""(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"""					,				R""" \1 """					,				__UpperCamelCase )
	_lowerCAmelCase			=re.sub(R"""\s*\n\s*"""					,				""" \n """					,				__UpperCamelCase )
	_lowerCAmelCase			=re.sub(R"""[^\S\n]+"""					,				""" """					,				__UpperCamelCase )
	return text.strip()







class 	lowerCamelCase__      (      A__       ):





			'''simple docstring'''
			lowerCamelCase						      =	VOCAB_FILES_NAMES
			lowerCamelCase						      =	PRETRAINED_VOCAB_FILES_MAP
			lowerCamelCase						      =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			lowerCamelCase						      =	['''input_ids''', '''attention_mask''']



			def __init__(							self  ,  __UpperCAmelCase  ,  __UpperCAmelCase  ,  __UpperCAmelCase="<unk>"  ,  **__UpperCAmelCase						)				->  int:
				super().__init__(unk_token=_lowerCAmelCase  ,  **_lowerCAmelCase						)

				try:
					import ftfy
					from spacy.lang.en import English

					_lowerCAmelCase			=English()
					_lowerCAmelCase			=_nlp.tokenizer
					_lowerCAmelCase			=ftfy.fix_text
				except ImportError:
					logger.warning("""ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."""						)
					_lowerCAmelCase			=BasicTokenizer(do_lower_case=_lowerCAmelCase						)
					_lowerCAmelCase			=None

				with open(_lowerCAmelCase  ,  encoding="""utf-8"""						) as vocab_handle:
					_lowerCAmelCase			=json.load(_lowerCAmelCase						)
				_lowerCAmelCase			={v: k for k, v in self.encoder.items()}
				with open(_lowerCAmelCase  ,  encoding="""utf-8"""						) as merges_handle:
					_lowerCAmelCase			=merges_handle.read().split("""\n"""						)[1:-1]
				_lowerCAmelCase			=[tuple(merge.split()						) for merge in merges]
				_lowerCAmelCase			=dict(zip(_lowerCAmelCase  ,  range(len(_lowerCAmelCase						)						)						)						)
				_lowerCAmelCase			={}



			@property
			def       _lowerCAmelCase	(							self						)				->  List[str]:
				return True



			@property
			def       _lowerCAmelCase	(							self						)				->  List[str]:
				return len(self.encoder						)



			def       _lowerCAmelCase	(							self						)				->  List[str]:
				return dict(self.encoder  ,  **self.added_tokens_encoder						)



			def       _lowerCAmelCase	(							self  ,  __UpperCAmelCase						)				->  int:
				_lowerCAmelCase			=tuple(token[:-1]						) + (token[-1] + """</w>""",)
				if token in self.cache:
					return self.cache[token]
				_lowerCAmelCase			=get_pairs(_lowerCAmelCase						)

				if not pairs:
					return token + "</w>"

				while True:
					_lowerCAmelCase			=min(_lowerCAmelCase  ,  key=lambda __UpperCAmelCase						: self.bpe_ranks.get(_lowerCAmelCase  ,  float("""inf"""						)						)						)
					if bigram not in self.bpe_ranks:
						break
					_lowerCAmelCase		,							_lowerCAmelCase			=bigram
					_lowerCAmelCase			=[]
					_lowerCAmelCase			=0
					while i < len(_lowerCAmelCase						):
						try:
							_lowerCAmelCase			=word.index(_lowerCAmelCase  ,  _lowerCAmelCase						)
						except ValueError:
							new_word.extend(word[i:]						)
							break
						else:
							new_word.extend(word[i:j]						)
							_lowerCAmelCase			=j

						if word[i] == first and i < len(_lowerCAmelCase						) - 1 and word[i + 1] == second:
							new_word.append(first + second						)
							i += 2
						else:
							new_word.append(word[i]						)
							i += 1
					_lowerCAmelCase			=tuple(_lowerCAmelCase						)
					_lowerCAmelCase			=new_word
					if len(_lowerCAmelCase						) == 1:
						break
					else:
						_lowerCAmelCase			=get_pairs(_lowerCAmelCase						)
				_lowerCAmelCase			=""" """.join(_lowerCAmelCase						)
				if word == "\n  </w>":
					_lowerCAmelCase			="""\n</w>"""
				_lowerCAmelCase			=word
				return word



			def       _lowerCAmelCase	(							self  ,  __UpperCAmelCase						)				->  Union[str, Any]:
				_lowerCAmelCase			=[]
				if self.fix_text is None:
					# Using BERT's BasicTokenizer
					_lowerCAmelCase			=self.nlp.tokenize(_lowerCAmelCase						)
					for token in text:
						split_tokens.extend(list(self.bpe(_lowerCAmelCase						).split(""" """						)						)						)
				else:
					# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
					_lowerCAmelCase			=self.nlp(text_standardize(self.fix_text(_lowerCAmelCase						)						)						)
					for token in text:
						split_tokens.extend(list(self.bpe(token.text.lower()						).split(""" """						)						)						)
				return split_tokens



			def       _lowerCAmelCase	(							self  ,  __UpperCAmelCase						)				->  Union[str, Any]:
				return self.encoder.get(_lowerCAmelCase  ,  self.encoder.get(self.unk_token						)						)



			def       _lowerCAmelCase	(							self  ,  __UpperCAmelCase						)				->  Any:
				return self.decoder.get(_lowerCAmelCase  ,  self.unk_token						)



			def       _lowerCAmelCase	(							self  ,  __UpperCAmelCase						)				->  Optional[Any]:
				_lowerCAmelCase			="""""".join(_lowerCAmelCase						).replace("""</w>"""  ,  """ """						).strip()
				return out_string



			def       _lowerCAmelCase	(							self  ,  __UpperCAmelCase  ,  __UpperCAmelCase = None						)				->  Optional[int]:
				if not os.path.isdir(_lowerCAmelCase						):
					logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
					return
				_lowerCAmelCase			=os.path.join(
				    _lowerCAmelCase  ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)
				_lowerCAmelCase			=os.path.join(
				    _lowerCAmelCase  ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]						)

				with open(_lowerCAmelCase  ,  """w"""  ,  encoding="""utf-8"""						) as f:
					f.write(json.dumps(self.encoder  ,  indent=2  ,  sort_keys=_lowerCAmelCase  ,  ensure_ascii=_lowerCAmelCase						) + """\n"""						)

				_lowerCAmelCase			=0
				with open(_lowerCAmelCase  ,  """w"""  ,  encoding="""utf-8"""						) as writer:
					writer.write("""#version: 0.2\n"""						)
					for bpe_tokens, token_index in sorted(self.bpe_ranks.items()  ,  key=lambda __UpperCAmelCase						: kv[1]						):
						if index != token_index:
							logger.warning(
							    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
							    """ Please check that the tokenizer is not corrupted!"""						)
							_lowerCAmelCase			=token_index
						writer.write(""" """.join(_lowerCAmelCase						) + """\n"""						)
						index += 1

				return vocab_file, merge_file

"""simple docstring"""


from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__A 		= logging.get_logger(__name__)

__A 		= {
    'nghuyong/ernie-1.0-base-zh': 'https://huggingface.co/nghuyong/ernie-1.0-base-zh/resolve/main/config.json',
    'nghuyong/ernie-2.0-base-en': 'https://huggingface.co/nghuyong/ernie-2.0-base-en/resolve/main/config.json',
    'nghuyong/ernie-2.0-large-en': 'https://huggingface.co/nghuyong/ernie-2.0-large-en/resolve/main/config.json',
    'nghuyong/ernie-3.0-base-zh': 'https://huggingface.co/nghuyong/ernie-3.0-base-zh/resolve/main/config.json',
    'nghuyong/ernie-3.0-medium-zh': 'https://huggingface.co/nghuyong/ernie-3.0-medium-zh/resolve/main/config.json',
    'nghuyong/ernie-3.0-mini-zh': 'https://huggingface.co/nghuyong/ernie-3.0-mini-zh/resolve/main/config.json',
    'nghuyong/ernie-3.0-micro-zh': 'https://huggingface.co/nghuyong/ernie-3.0-micro-zh/resolve/main/config.json',
    'nghuyong/ernie-3.0-nano-zh': 'https://huggingface.co/nghuyong/ernie-3.0-nano-zh/resolve/main/config.json',
    'nghuyong/ernie-gram-zh': 'https://huggingface.co/nghuyong/ernie-gram-zh/resolve/main/config.json',
    'nghuyong/ernie-health-zh': 'https://huggingface.co/nghuyong/ernie-health-zh/resolve/main/config.json',
}







class 	lowerCamelCase__      (      __magic_name__       ):





			'''simple docstring'''
			lowerCamelCase						      =	'''ernie'''



			def __init__(							self  ,  __UpperCAmelCase=3_05_22  ,  __UpperCAmelCase=7_68  ,  __UpperCAmelCase=12  ,  __UpperCAmelCase=12  ,  __UpperCAmelCase=30_72  ,  __UpperCAmelCase="gelu"  ,  __UpperCAmelCase=0.1  ,  __UpperCAmelCase=0.1  ,  __UpperCAmelCase=5_12  ,  __UpperCAmelCase=2  ,  __UpperCAmelCase=3  ,  __UpperCAmelCase=False  ,  __UpperCAmelCase=0.0_2  ,  __UpperCAmelCase=1e-12  ,  __UpperCAmelCase=0  ,  __UpperCAmelCase="absolute"  ,  __UpperCAmelCase=True  ,  __UpperCAmelCase=None  ,  **__UpperCAmelCase  ,  )				->  str:
				super().__init__(pad_token_id=__UpperCAmelCase  ,  **__UpperCAmelCase						)

				_lowerCAmelCase			=vocab_size
				_lowerCAmelCase			=hidden_size
				_lowerCAmelCase			=num_hidden_layers
				_lowerCAmelCase			=num_attention_heads
				_lowerCAmelCase			=hidden_act
				_lowerCAmelCase			=intermediate_size
				_lowerCAmelCase			=hidden_dropout_prob
				_lowerCAmelCase			=attention_probs_dropout_prob
				_lowerCAmelCase			=max_position_embeddings
				_lowerCAmelCase			=type_vocab_size
				_lowerCAmelCase			=task_type_vocab_size
				_lowerCAmelCase			=use_task_id
				_lowerCAmelCase			=initializer_range
				_lowerCAmelCase			=layer_norm_eps
				_lowerCAmelCase			=position_embedding_type
				_lowerCAmelCase			=use_cache
				_lowerCAmelCase			=classifier_dropout







class 	lowerCamelCase__      (      __magic_name__       ):





			'''simple docstring'''



			@property
			def       _lowerCAmelCase	(							self						)				->  Mapping[str, Mapping[int, str]]:
				if self.task == "multiple-choice":
					_lowerCAmelCase			={0: """batch""", 1: """choice""", 2: """sequence"""}
				else:
					_lowerCAmelCase			={0: """batch""", 1: """sequence"""}
				return OrderedDict(
				    [
				        ("""input_ids""", dynamic_axis),
				        ("""attention_mask""", dynamic_axis),
				        ("""token_type_ids""", dynamic_axis),
				        ("""task_type_ids""", dynamic_axis),
				    ]						)



