
'''simple docstring'''





import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import cached_download, hf_hub_url
from PIL import Image

from transformers import DeformableDetrConfig, DeformableDetrForObjectDetection, DeformableDetrImageProcessor
from transformers.utils import logging


logging.set_verbosity_info()
UpperCAmelCase__				:  Tuple       					=				logging.get_logger(__name__)



def       _lowercase					(      __SCREAMING_SNAKE_CASE )	->     str:
			if "backbone.0.body" in orig_key:
						UpperCamelCase__    :		Optional[int]     =				orig_key.replace('backbone.0.body'				,    'backbone.conv_encoder.model' )
			if "transformer" in orig_key:
						UpperCamelCase__    :		Any     =				orig_key.replace('transformer.'				,    '' )
			if "norm1" in orig_key:
						if "encoder" in orig_key:
									UpperCamelCase__    :		List[str]     =				orig_key.replace('norm1'				,    'self_attn_layer_norm' )
						else:
									UpperCamelCase__    :		Optional[int]     =				orig_key.replace('norm1'				,    'encoder_attn_layer_norm' )
			if "norm2" in orig_key:
						if "encoder" in orig_key:
									UpperCamelCase__    :		Optional[Any]     =				orig_key.replace('norm2'				,    'final_layer_norm' )
						else:
									UpperCamelCase__    :		str     =				orig_key.replace('norm2'				,    'self_attn_layer_norm' )
			if "norm3" in orig_key:
						UpperCamelCase__    :		Any     =				orig_key.replace('norm3'				,    'final_layer_norm' )
			if "linear1" in orig_key:
						UpperCamelCase__    :		Tuple     =				orig_key.replace('linear1'				,    'fc1' )
			if "linear2" in orig_key:
						UpperCamelCase__    :		List[str]     =				orig_key.replace('linear2'				,    'fc2' )
			if "query_embed" in orig_key:
						UpperCamelCase__    :		int     =				orig_key.replace('query_embed'				,    'query_position_embeddings' )
			if "cross_attn" in orig_key:
						UpperCamelCase__    :		Any     =				orig_key.replace('cross_attn'				,    'encoder_attn' )

			return orig_key



def       _lowercase					(      __SCREAMING_SNAKE_CASE )	->     Dict:
			# transformer decoder self-attention layers
			for i in range(6 ):
						# read in weights + bias of input projection layer of self-attention
						UpperCamelCase__    :		str     =				state_dict.pop(F"""decoder.layers.{i}.self_attn.in_proj_weight""" )
						UpperCamelCase__    :		Tuple     =				state_dict.pop(F"""decoder.layers.{i}.self_attn.in_proj_bias""" )
						# next, add query, keys and values (in that order) to the state dict
						UpperCamelCase__    :		Optional[int]     =				in_proj_weight[:256, :]
						UpperCamelCase__    :		List[str]     =				in_proj_bias[:256]
						UpperCamelCase__    :		str     =				in_proj_weight[256:512, :]
						UpperCamelCase__    :		Optional[int]     =				in_proj_bias[256:512]
						UpperCamelCase__    :		Tuple     =				in_proj_weight[-256:, :]
						UpperCamelCase__    :		List[str]     =				in_proj_bias[-256:]



def       _lowercase					(      )	->     Dict:
			UpperCamelCase__    :		Union[str, Any]     =				'http://images.cocodataset.org/val2017/000000039769.jpg'
			UpperCamelCase__    :		str     =				Image.open(requests.get(_UpperCAmelCase				,    stream=_UpperCAmelCase ).raw )

			return im



@torch.no_grad()
def       _lowercase					(      __SCREAMING_SNAKE_CASE				,    __SCREAMING_SNAKE_CASE				,    __SCREAMING_SNAKE_CASE				,    __SCREAMING_SNAKE_CASE				,    __SCREAMING_SNAKE_CASE				,    __SCREAMING_SNAKE_CASE				,    __SCREAMING_SNAKE_CASE				,    )	->     int:
			UpperCamelCase__    :		List[str]     =				DeformableDetrConfig()
			# set config attributes
			if single_scale:
						UpperCamelCase__    :		Dict     =				1
			UpperCamelCase__    :		Any     =				dilation
			UpperCamelCase__    :		Optional[Any]     =				with_box_refine
			UpperCamelCase__    :		Optional[int]     =				two_stage
			# set labels
			UpperCamelCase__    :		Union[str, Any]     =				91
			UpperCamelCase__    :		int     =				'huggingface/label-files'
			UpperCamelCase__    :		List[str]     =				'coco-detection-id2label.json'
			UpperCamelCase__    :		Tuple     =				json.load(open(cached_download(hf_hub_url(_UpperCAmelCase				,    _UpperCAmelCase				,    repo_type='dataset' ) )				,    'r' ) )
			UpperCamelCase__    :		List[str]     =				{int(_UpperCAmelCase ): v for k, v in idalabel.items()}
			UpperCamelCase__    :		Any     =				idalabel
			UpperCamelCase__    :		str     =				{v: k for k, v in idalabel.items()}

			# load image processor
			UpperCamelCase__    :		Optional[Any]     =				DeformableDetrImageProcessor(format='coco_detection' )

			# prepare image
			UpperCamelCase__    :		Union[str, Any]     =				prepare_img()
			UpperCamelCase__    :		Dict     =				image_processor(images=_UpperCAmelCase				,    return_tensors='pt' )
			UpperCamelCase__    :		List[str]     =				encoding['pixel_values']

			logger.info('Converting model...' )

			# load original state dict
			UpperCamelCase__    :		List[Any]     =				torch.load(_UpperCAmelCase				,    map_location='cpu' )['model']
			# rename keys
			for key in state_dict.copy().keys():
						UpperCamelCase__    :		Optional[int]     =				state_dict.pop(_UpperCAmelCase )
						UpperCamelCase__    :		Optional[int]     =				val
			# query, key and value matrices need special treatment
			read_in_q_k_v(_UpperCAmelCase )
			# important: we need to prepend a prefix to each of the base model keys as the head models use different attributes for them
			UpperCamelCase__    :		List[str]     =				'model.'
			for key in state_dict.copy().keys():
						if not key.startswith('class_embed' ) and not key.startswith('bbox_embed' ):
									UpperCamelCase__    :		List[Any]     =				state_dict.pop(_UpperCAmelCase )
									UpperCamelCase__    :		Union[str, Any]     =				val
    # finally, create HuggingFace model and load state dict
			UpperCamelCase__    :		Optional[Any]     =				DeformableDetrForObjectDetection(_UpperCAmelCase )
			model.load_state_dict(_UpperCAmelCase )
			model.eval()

			UpperCamelCase__    :		str     =				'cuda' if torch.cuda.is_available() else 'cpu'
			model.to(_UpperCAmelCase )
			# verify our conversion
			UpperCamelCase__    :		List[str]     =				model(pixel_values.to(_UpperCAmelCase ) )

			UpperCamelCase__    :		Optional[int]     =				torch.tensor(
			    [[-9.6645, -4.3449, -5.8705], [-9.7035, -3.8504, -5.0724], [-10.5634, -5.3379, -7.5116]] )
			UpperCamelCase__    :		Union[str, Any]     =				torch.tensor([[0.8693, 0.2289, 0.2492], [0.3150, 0.5489, 0.5845], [0.5563, 0.7580, 0.8518]] )

			if single_scale:
						UpperCamelCase__    :		List[Any]     =				torch.tensor(
						    [[-9.9051, -4.2541, -6.4852], [-9.6947, -4.0854, -6.8033], [-10.0665, -5.8470, -7.7003]] )
						UpperCamelCase__    :		Optional[Any]     =				torch.tensor([[0.7292, 0.4991, 0.5532], [0.7959, 0.2426, 0.4236], [0.7582, 0.3518, 0.4451]] )

			if single_scale and dilation:
						UpperCamelCase__    :		List[Any]     =				torch.tensor(
						    [[-8.9652, -4.1074, -5.6635], [-9.0596, -4.9447, -6.6075], [-10.1178, -4.5275, -6.2671]] )
						UpperCamelCase__    :		Tuple     =				torch.tensor([[0.7665, 0.4130, 0.4769], [0.8364, 0.1841, 0.3391], [0.6261, 0.3895, 0.7978]] )

			if with_box_refine:
						UpperCamelCase__    :		str     =				torch.tensor(
						    [[-8.8895, -5.4187, -6.8153], [-8.4706, -6.1668, -7.6184], [-9.0042, -5.5359, -6.9141]] )
						UpperCamelCase__    :		List[str]     =				torch.tensor([[0.7828, 0.2208, 0.4323], [0.0892, 0.5996, 0.1319], [0.5524, 0.6389, 0.8914]] )

			if with_box_refine and two_stage:
						UpperCamelCase__    :		List[Any]     =				torch.tensor(
						    [[-6.7108, -4.3213, -6.3777], [-8.9014, -6.1799, -6.7240], [-6.9315, -4.4735, -6.2298]] )
						UpperCamelCase__    :		int     =				torch.tensor([[0.2583, 0.5499, 0.4683], [0.7652, 0.9068, 0.4882], [0.5490, 0.2763, 0.0564]] )

			print('Logits:'				,    outputs.logits[0, :3, :3] )

			assert torch.allclose(outputs.logits[0, :3, :3]				,    expected_logits.to(_UpperCAmelCase )				,    atol=1E-4 )
			assert torch.allclose(outputs.pred_boxes[0, :3, :3]				,    expected_boxes.to(_UpperCAmelCase )				,    atol=1E-4 )

			print('Everything ok!' )

			# Save model and image processor
			logger.info(F"""Saving PyTorch model and image processor to {pytorch_dump_folder_path}...""" )
			Path(_UpperCAmelCase ).mkdir(exist_ok=_UpperCAmelCase )
			model.save_pretrained(_UpperCAmelCase )
			image_processor.save_pretrained(_UpperCAmelCase )

			# Push to hub
			if push_to_hub:
						UpperCamelCase__    :		Optional[Any]     =				'deformable-detr'
						model_name += "-single-scale" if single_scale else ""
						model_name += "-dc5" if dilation else ""
						model_name += "-with-box-refine" if with_box_refine else ""
						model_name += "-two-stage" if two_stage else ""
						print('Pushing model to hub...' )
						model.push_to_hub(repo_path_or_name=_UpperCAmelCase				,    organization='nielsr'				,    commit_message='Add model' )


if __name__ == "__main__":
							UpperCAmelCase__				:  str       					=				argparse.ArgumentParser()

							parser.add_argument(
							    '''--checkpoint_path''',
							    type=str,
							    default='''/home/niels/checkpoints/deformable_detr/r50_deformable_detr-checkpoint.pth''',
							    help='''Path to Pytorch checkpoint (.pth file) you\'d like to convert.''',
							)
							parser.add_argument('''--single_scale''', action='''store_true''', help='''Whether to set config.num_features_levels = 1.''')
							parser.add_argument('''--dilation''', action='''store_true''', help='''Whether to set config.dilation=True.''')
							parser.add_argument('''--with_box_refine''', action='''store_true''', help='''Whether to set config.with_box_refine=True.''')
							parser.add_argument('''--two_stage''', action='''store_true''', help='''Whether to set config.two_stage=True.''')
							parser.add_argument(
							    '''--pytorch_dump_folder_path''',
							    default=None,
							    type=str,
							    required=True,
							    help='''Path to the folder to output PyTorch model.''',
							)
							parser.add_argument(
							    '''--push_to_hub''', action='''store_true''', help='''Whether or not to push the converted model to the 🤗 hub.'''
							)
							UpperCAmelCase__				:  str       					=				parser.parse_args()
							convert_deformable_detr_checkpoint(
							    args.checkpoint_path,
							    args.single_scale,
							    args.dilation,
							    args.with_box_refine,
							    args.two_stage,
							    args.pytorch_dump_folder_path,
							    args.push_to_hub,
							)





import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


UpperCAmelCase__				:  Optional[Any]       					=				logging.get_logger(__name__)

UpperCAmelCase__				:  List[str]       					=				{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

UpperCAmelCase__				:  Optional[int]       					=				{
    '''vocab_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/vocab.json'''},
    '''merges_file''': {'''openai-gpt''': '''https://huggingface.co/openai-gpt/resolve/main/merges.txt'''},
}

UpperCAmelCase__				:  Any       					=				{
    '''openai-gpt''': 512,
}



def       _lowercase					(      __SCREAMING_SNAKE_CASE )	->     Optional[int]:
			UpperCamelCase__    :		List[Any]     =				text.strip()
			if not text:
						return []
			UpperCamelCase__    :		List[Any]     =				text.split()
			return tokens



class 			UpperCamelCase_			(						__UpperCamelCase   ):




							'''simple docstring'''

							def __init__(				self     ,						UpperCamelCase=True     ,						UpperCamelCase=None     ,						UpperCamelCase=True     ,						UpperCamelCase=None     ,						UpperCamelCase=True     ,						) ->	Tuple:
										if never_split is None:
													UpperCamelCase__    :		str     =				[]
										UpperCamelCase__    :		Optional[Any]     =				do_lower_case
										UpperCamelCase__    :		Optional[int]     =				set(UpperCamelCase)
										UpperCamelCase__    :		List[Any]     =				tokenize_chinese_chars
										UpperCamelCase__    :		Optional[int]     =				strip_accents
										UpperCamelCase__    :		Any     =				do_split_on_punc
							def        lowerCAmelCase__ (				self     ,						UpperCamelCase     ,						UpperCamelCase=None) ->	List[str]:
										UpperCamelCase__    :		Tuple     =				self.never_split.union(set(UpperCamelCase)) if never_split else self.never_split
										UpperCamelCase__    :		Optional[int]     =				self._clean_text(UpperCamelCase)

										# This was added on November 1st, 2018 for the multilingual and Chinese
										# models. This is also applied to the English models now, but it doesn't
										# matter since the English models were not trained on any Chinese data
										# and generally don't have any Chinese data in them (there are Chinese
										# characters in the vocabulary because Wikipedia does have some Chinese
										# words in the English Wikipedia.).
										if self.tokenize_chinese_chars:
													UpperCamelCase__    :		List[str]     =				self._tokenize_chinese_chars(UpperCamelCase)
										# prevents treating the same character with different unicode codepoints as different characters
										UpperCamelCase__    :		Dict     =				unicodedata.normalize('NFC'     ,						UpperCamelCase)
										UpperCamelCase__    :		Optional[int]     =				whitespace_tokenize(UpperCamelCase)
										UpperCamelCase__    :		List[Any]     =				[]
										for token in orig_tokens:
													if token not in never_split:
																if self.do_lower_case:
																			UpperCamelCase__    :		str     =				token.lower()
																			if self.strip_accents is not False:
																						UpperCamelCase__    :		str     =				self._run_strip_accents(UpperCamelCase)
																elif self.strip_accents:
																			UpperCamelCase__    :		Union[str, Any]     =				self._run_strip_accents(UpperCamelCase)
													split_tokens.extend(self._run_split_on_punc(UpperCamelCase     ,						UpperCamelCase))

										UpperCamelCase__    :		Dict     =				whitespace_tokenize(' '.join(UpperCamelCase))
										return output_tokens
							def        lowerCAmelCase__ (				self     ,						UpperCamelCase) ->	List[str]:
										UpperCamelCase__    :		Union[str, Any]     =				unicodedata.normalize('NFD'     ,						UpperCamelCase)
										UpperCamelCase__    :		Dict     =				[]
										for char in text:
													UpperCamelCase__    :		Union[str, Any]     =				unicodedata.category(UpperCamelCase)
													if cat == "Mn":
																continue
													output.append(UpperCamelCase)
										return "".join(UpperCamelCase)
							def        lowerCAmelCase__ (				self     ,						UpperCamelCase     ,						UpperCamelCase=None) ->	Dict:
										if not self.do_split_on_punc or (never_split is not None and text in never_split):
													return [text]
										UpperCamelCase__    :		Dict     =				list(UpperCamelCase)
										UpperCamelCase__    :		Optional[int]     =				0
										UpperCamelCase__    :		Dict     =				True
										UpperCamelCase__    :		Optional[int]     =				[]
										while i < len(UpperCamelCase):
													UpperCamelCase__    :		List[str]     =				chars[i]
													if _is_punctuation(UpperCamelCase):
																output.append([char])
																UpperCamelCase__    :		Tuple     =				True
													else:
																if start_new_word:
																			output.append([])
																UpperCamelCase__    :		Union[str, Any]     =				False
																output[-1].append(UpperCamelCase)
													i += 1

										return ["".join(UpperCamelCase) for x in output]
							def        lowerCAmelCase__ (				self     ,						UpperCamelCase) ->	Tuple:
										UpperCamelCase__    :		List[str]     =				[]
										for char in text:
													UpperCamelCase__    :		List[str]     =				ord(UpperCamelCase)
													if self._is_chinese_char(UpperCamelCase):
																output.append(' ')
																output.append(UpperCamelCase)
																output.append(' ')
													else:
																output.append(UpperCamelCase)
										return "".join(UpperCamelCase)
							def        lowerCAmelCase__ (				self     ,						UpperCamelCase) ->	Union[str, Any]:
										# This defines a "chinese character" as anything in the CJK Unicode block:
										#   https://en.wikipedia.org/wiki/CJK_Unified_Ideographs_(Unicode_block)
										#
										# Note that the CJK Unicode block is NOT all Japanese and Korean characters,
										# despite its name. The modern Korean Hangul alphabet is a different block,
										# as is Japanese Hiragana and Katakana. Those alphabets are used to write
										# space-separated words, so they are not treated specially and handled
										# like the all of the other languages.
										if (
										    (cp >= 0X4e00 and cp <= 0X9fff)
										    or (cp >= 0X3400 and cp <= 0X4dbf)  #
										    or (cp >= 0X2_0000 and cp <= 0X2_a6df)  #
										    or (cp >= 0X2_a700 and cp <= 0X2_b73f)  #
										    or (cp >= 0X2_b740 and cp <= 0X2_b81f)  #
										    or (cp >= 0X2_b820 and cp <= 0X2_ceaf)  #
										    or (cp >= 0Xf900 and cp <= 0Xfaff)
										    or (cp >= 0X2_f800 and cp <= 0X2_fa1f)  #
										):  #
													return True

										return False






							def        lowerCAmelCase__ (				self     ,						UpperCamelCase) ->	List[str]:
										UpperCamelCase__    :		List[Any]     =				[]
										for char in text:
													UpperCamelCase__    :		Dict     =				ord(UpperCamelCase)
													if cp == 0 or cp == 0Xfffd or _is_control(UpperCamelCase):
																continue
													if _is_whitespace(UpperCamelCase):
																output.append(' ')
													else:
																output.append(UpperCamelCase)
										return "".join(UpperCamelCase)



def       _lowercase					(      __SCREAMING_SNAKE_CASE )	->     Union[str, Any]:
			UpperCamelCase__    :		Optional[int]     =				set()
			UpperCamelCase__    :		str     =				word[0]
			for char in word[1:]:
						pairs.add((prev_char, char) )
						UpperCamelCase__    :		Any     =				char
			return pairs



def       _lowercase					(      __SCREAMING_SNAKE_CASE )	->     Optional[Any]:
			UpperCamelCase__    :		Tuple     =				text.replace('—'				,    '-' )
			UpperCamelCase__    :		Any     =				text.replace('–'				,    '-' )
			UpperCamelCase__    :		Any     =				text.replace('―'				,    '-' )
			UpperCamelCase__    :		Tuple     =				text.replace('…'				,    '...' )
			UpperCamelCase__    :		Dict     =				text.replace('´'				,    '\'' )
			UpperCamelCase__    :		int     =				re.sub(R'(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)'				,    R' \1 '				,    __SCREAMING_SNAKE_CASE )
			UpperCamelCase__    :		int     =				re.sub(R'\s*\n\s*'				,    ' \n '				,    __SCREAMING_SNAKE_CASE )
			UpperCamelCase__    :		str     =				re.sub(R'[^\S\n]+'				,    ' '				,    __SCREAMING_SNAKE_CASE )
			return text.strip()



class 			UpperCamelCase_			(						__UpperCamelCase   ):




							'''simple docstring'''

							UpperCamelCase_					   =							VOCAB_FILES_NAMES
							UpperCamelCase_					   =							PRETRAINED_VOCAB_FILES_MAP
							UpperCamelCase_					   =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCamelCase_					   =							["""input_ids""", """attention_mask"""]
							def __init__(				self     ,						UpperCamelCase     ,						UpperCamelCase     ,						UpperCamelCase="<unk>"     ,						**UpperCamelCase) ->	Union[str, Any]:
										super().__init__(unk_token=UpperCamelCase     ,						**UpperCamelCase)

										try:
													import ftfy
													from spacy.lang.en import English

													UpperCamelCase__    :		Optional[int]     =				English()
													UpperCamelCase__    :		Dict     =				_nlp.tokenizer
													UpperCamelCase__    :		List[Any]     =				ftfy.fix_text
										except ImportError:
													logger.warning('ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.')
													UpperCamelCase__    :		Dict     =				BasicTokenizer(do_lower_case=UpperCamelCase)
													UpperCamelCase__    :		List[Any]     =				None

										with open(UpperCamelCase     ,						encoding='utf-8') as vocab_handle:
													UpperCamelCase__    :		int     =				json.load(UpperCamelCase)
										UpperCamelCase__    :		Dict     =				{v: k for k, v in self.encoder.items()}
										with open(UpperCamelCase     ,						encoding='utf-8') as merges_handle:
													UpperCamelCase__    :		Optional[int]     =				merges_handle.read().split('\n')[1:-1]
										UpperCamelCase__    :		int     =				[tuple(merge.split()) for merge in merges]
										UpperCamelCase__    :		int     =				dict(zip(UpperCamelCase     ,						range(len(UpperCamelCase))))
										UpperCamelCase__    :		Optional[Any]     =				{}
							@property
							def        lowerCAmelCase__ (				self) ->	Dict:
										return True
							@property
							def        lowerCAmelCase__ (				self) ->	Union[str, Any]:
										return len(self.encoder)
							def        lowerCAmelCase__ (				self) ->	Tuple:
										return dict(self.encoder     ,						**self.added_tokens_encoder)
							def        lowerCAmelCase__ (				self     ,						UpperCamelCase) ->	List[Any]:
										UpperCamelCase__    :		Optional[Any]     =				tuple(token[:-1]) + (token[-1] + '</w>',)
										if token in self.cache:
													return self.cache[token]
										UpperCamelCase__    :		int     =				get_pairs(UpperCamelCase)

										if not pairs:
													return token + "</w>"

										while True:
													UpperCamelCase__    :		str     =				min(UpperCamelCase     ,						key=lambda UpperCamelCase: self.bpe_ranks.get(UpperCamelCase     ,						float('inf')))
													if bigram not in self.bpe_ranks:
																break
													UpperCamelCase__   ,			UpperCamelCase__    :		Optional[int]     =				bigram
													UpperCamelCase__    :		str     =				[]
													UpperCamelCase__    :		List[Any]     =				0
													while i < len(UpperCamelCase):
																try:
																			UpperCamelCase__    :		Union[str, Any]     =				word.index(UpperCamelCase     ,						UpperCamelCase)
																except ValueError:
																			new_word.extend(word[i:])
																			break
																else:
																			new_word.extend(word[i:j])
																			UpperCamelCase__    :		Tuple     =				j

																if word[i] == first and i < len(UpperCamelCase) - 1 and word[i + 1] == second:
																			new_word.append(first + second)
																			i += 2
																else:
																			new_word.append(word[i])
																			i += 1
													UpperCamelCase__    :		int     =				tuple(UpperCamelCase)
													UpperCamelCase__    :		List[Any]     =				new_word
													if len(UpperCamelCase) == 1:
																break
													else:
																UpperCamelCase__    :		List[Any]     =				get_pairs(UpperCamelCase)
										UpperCamelCase__    :		Optional[int]     =				' '.join(UpperCamelCase)
										if word == "\n  </w>":
													UpperCamelCase__    :		Optional[int]     =				'\n</w>'
										UpperCamelCase__    :		Optional[Any]     =				word
										return word
							def        lowerCAmelCase__ (				self     ,						UpperCamelCase) ->	List[Any]:
										UpperCamelCase__    :		int     =				[]
										if self.fix_text is None:
													# Using BERT's BasicTokenizer
													UpperCamelCase__    :		List[str]     =				self.nlp.tokenize(UpperCamelCase)
													for token in text:
																split_tokens.extend(list(self.bpe(UpperCamelCase).split(' ')))
										else:
													# Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
													UpperCamelCase__    :		Dict     =				self.nlp(text_standardize(self.fix_text(UpperCamelCase)))
													for token in text:
																split_tokens.extend(list(self.bpe(token.text.lower()).split(' ')))
										return split_tokens
							def        lowerCAmelCase__ (				self     ,						UpperCamelCase) ->	Tuple:
										return self.encoder.get(UpperCamelCase     ,						self.encoder.get(self.unk_token))
							def        lowerCAmelCase__ (				self     ,						UpperCamelCase) ->	List[Any]:
										return self.decoder.get(UpperCamelCase     ,						self.unk_token)
							def        lowerCAmelCase__ (				self     ,						UpperCamelCase) ->	List[str]:
										UpperCamelCase__    :		Optional[Any]     =				''.join(UpperCamelCase).replace('</w>'     ,						' ').strip()
										return out_string






							def        lowerCAmelCase__ (				self     ,						UpperCamelCase     ,						UpperCamelCase = None) ->	Tuple[str]:
										if not os.path.isdir(UpperCamelCase):
													logger.error(F"""Vocabulary path ({save_directory}) should be a directory""")
													return
										UpperCamelCase__    :		int     =				os.path.join(
										    UpperCamelCase     ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])
										UpperCamelCase__    :		List[str]     =				os.path.join(
										    UpperCamelCase     ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'])

										with open(UpperCamelCase     ,						'w'     ,						encoding='utf-8') as f:
													f.write(json.dumps(self.encoder     ,						indent=2     ,						sort_keys=UpperCamelCase     ,						ensure_ascii=UpperCamelCase) + '\n')

										UpperCamelCase__    :		Any     =				0
										with open(UpperCamelCase     ,						'w'     ,						encoding='utf-8') as writer:
													writer.write('#version: 0.2\n')
													for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,						key=lambda UpperCamelCase: kv[1]):
																if index != token_index:
																			logger.warning(
																			    F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
																			    ' Please check that the tokenizer is not corrupted!')
																			UpperCamelCase__    :		Dict     =				token_index
																writer.write(' '.join(UpperCamelCase) + '\n')
																index += 1

										return vocab_file, merge_file



