


'''simple docstring'''







import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import cached_download, hf_hub_download, hf_hub_url
from PIL import Image

from transformers import DetaConfig, DetaForObjectDetection, DetaImageProcessor
from transformers.utils import logging


logging.set_verbosity_info()
_SCREAMING_SNAKE_CASE       :   Any        = logging.get_logger(__name__)



def 						_UpperCamelCase							( ):






				"""simple docstring"""



				__magic_name__				:					Dict						       =       DetaConfig(
				    num_queries=900     ,   encoder_ffn_dim=2048     ,   decoder_ffn_dim=2048     ,   num_feature_levels=5     ,   assign_first_stage=__a     ,   with_box_refine=__a     ,   two_stage=__a     ,   )

				# set labels
				__magic_name__				:					Dict						       =       91
				__magic_name__				:					Dict						       =       '''huggingface/label-files'''
				__magic_name__				:					List[str]						       =       '''coco-detection-id2label.json'''
				__magic_name__				:					Tuple						       =       json.load(open(cached_download(hf_hub_url(__a     ,   __a     ,   repo_type="dataset"		)		)     ,   "r"		)		)
				__magic_name__				:					int						       =       {int(__a		): v for k, v in idalabel.items()}
				__magic_name__				:					Union[str, Any]						       =       idalabel
				__magic_name__				:					Any						       =       {v: k for k, v in idalabel.items()}

				return config



def 						_UpperCamelCase							( UpperCamelCase__		):






				"""simple docstring"""



				__magic_name__				:					int						       =       []

				# stem
				# fmt: off
				rename_keys.append(("backbone.0.body.conv1.weight", "model.backbone.model.embedder.embedder.convolution.weight")		)
				rename_keys.append(("backbone.0.body.bn1.weight", "model.backbone.model.embedder.embedder.normalization.weight")		)
				rename_keys.append(("backbone.0.body.bn1.bias", "model.backbone.model.embedder.embedder.normalization.bias")		)
				rename_keys.append(("backbone.0.body.bn1.running_mean", "model.backbone.model.embedder.embedder.normalization.running_mean")		)
				rename_keys.append(("backbone.0.body.bn1.running_var", "model.backbone.model.embedder.embedder.normalization.running_var")		)
				# stages
				for stage_idx in range(len(config.backbone_config.depths		)		):
								for layer_idx in range(config.backbone_config.depths[stage_idx]		):
												# shortcut
												if layer_idx == 0:
																rename_keys.append(
																    (
																        F"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.0.weight""",
																        F"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.convolution.weight""",
																    )		)
																rename_keys.append(
																    (
																        F"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.weight""",
																        F"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.weight""",
																    )		)
																rename_keys.append(
																    (
																        F"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.bias""",
																        F"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.bias""",
																    )		)
																rename_keys.append(
																    (
																        F"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.running_mean""",
																        F"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.running_mean""",
																    )		)
																rename_keys.append(
																    (
																        F"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.downsample.1.running_var""",
																        F"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.shortcut.normalization.running_var""",
																    )		)
												# 3 convs
												for i in range(3		):
																rename_keys.append(
																    (
																        F"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.conv{i+1}.weight""",
																        F"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.convolution.weight""",
																    )		)
																rename_keys.append(
																    (
																        F"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.weight""",
																        F"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.weight""",
																    )		)
																rename_keys.append(
																    (
																        F"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.bias""",
																        F"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.bias""",
																    )		)
																rename_keys.append(
																    (
																        F"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.running_mean""",
																        F"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.running_mean""",
																    )		)
																rename_keys.append(
																    (
																        F"""backbone.0.body.layer{stage_idx + 1}.{layer_idx}.bn{i+1}.running_var""",
																        F"""model.backbone.model.encoder.stages.{stage_idx}.layers.{layer_idx}.layer.{i}.normalization.running_var""",
																    )		)
    # transformer encoder
				for i in range(config.encoder_layers		):
								rename_keys.append((F"""transformer.encoder.layers.{i}.self_attn.sampling_offsets.weight""", F"""model.encoder.layers.{i}.self_attn.sampling_offsets.weight""")		)
								rename_keys.append((F"""transformer.encoder.layers.{i}.self_attn.sampling_offsets.bias""", F"""model.encoder.layers.{i}.self_attn.sampling_offsets.bias""")		)
								rename_keys.append((F"""transformer.encoder.layers.{i}.self_attn.attention_weights.weight""", F"""model.encoder.layers.{i}.self_attn.attention_weights.weight""")		)
								rename_keys.append((F"""transformer.encoder.layers.{i}.self_attn.attention_weights.bias""", F"""model.encoder.layers.{i}.self_attn.attention_weights.bias""")		)
								rename_keys.append((F"""transformer.encoder.layers.{i}.self_attn.value_proj.weight""", F"""model.encoder.layers.{i}.self_attn.value_proj.weight""")		)
								rename_keys.append((F"""transformer.encoder.layers.{i}.self_attn.value_proj.bias""", F"""model.encoder.layers.{i}.self_attn.value_proj.bias""")		)
								rename_keys.append((F"""transformer.encoder.layers.{i}.self_attn.output_proj.weight""", F"""model.encoder.layers.{i}.self_attn.output_proj.weight""")		)
								rename_keys.append((F"""transformer.encoder.layers.{i}.self_attn.output_proj.bias""", F"""model.encoder.layers.{i}.self_attn.output_proj.bias""")		)
								rename_keys.append((F"""transformer.encoder.layers.{i}.norm1.weight""", F"""model.encoder.layers.{i}.self_attn_layer_norm.weight""")		)
								rename_keys.append((F"""transformer.encoder.layers.{i}.norm1.bias""", F"""model.encoder.layers.{i}.self_attn_layer_norm.bias""")		)
								rename_keys.append((F"""transformer.encoder.layers.{i}.linear1.weight""", F"""model.encoder.layers.{i}.fc1.weight""")		)
								rename_keys.append((F"""transformer.encoder.layers.{i}.linear1.bias""", F"""model.encoder.layers.{i}.fc1.bias""")		)
								rename_keys.append((F"""transformer.encoder.layers.{i}.linear2.weight""", F"""model.encoder.layers.{i}.fc2.weight""")		)
								rename_keys.append((F"""transformer.encoder.layers.{i}.linear2.bias""", F"""model.encoder.layers.{i}.fc2.bias""")		)
								rename_keys.append((F"""transformer.encoder.layers.{i}.norm2.weight""", F"""model.encoder.layers.{i}.final_layer_norm.weight""")		)
								rename_keys.append((F"""transformer.encoder.layers.{i}.norm2.bias""", F"""model.encoder.layers.{i}.final_layer_norm.bias""")		)

				# transformer decoder
				for i in range(config.decoder_layers		):
								rename_keys.append((F"""transformer.decoder.layers.{i}.cross_attn.sampling_offsets.weight""", F"""model.decoder.layers.{i}.encoder_attn.sampling_offsets.weight""")		)
								rename_keys.append((F"""transformer.decoder.layers.{i}.cross_attn.sampling_offsets.bias""", F"""model.decoder.layers.{i}.encoder_attn.sampling_offsets.bias""")		)
								rename_keys.append((F"""transformer.decoder.layers.{i}.cross_attn.attention_weights.weight""", F"""model.decoder.layers.{i}.encoder_attn.attention_weights.weight""")		)
								rename_keys.append((F"""transformer.decoder.layers.{i}.cross_attn.attention_weights.bias""", F"""model.decoder.layers.{i}.encoder_attn.attention_weights.bias""")		)
								rename_keys.append((F"""transformer.decoder.layers.{i}.cross_attn.value_proj.weight""", F"""model.decoder.layers.{i}.encoder_attn.value_proj.weight""")		)
								rename_keys.append((F"""transformer.decoder.layers.{i}.cross_attn.value_proj.bias""", F"""model.decoder.layers.{i}.encoder_attn.value_proj.bias""")		)
								rename_keys.append((F"""transformer.decoder.layers.{i}.cross_attn.output_proj.weight""", F"""model.decoder.layers.{i}.encoder_attn.output_proj.weight""")		)
								rename_keys.append((F"""transformer.decoder.layers.{i}.cross_attn.output_proj.bias""", F"""model.decoder.layers.{i}.encoder_attn.output_proj.bias""")		)
								rename_keys.append((F"""transformer.decoder.layers.{i}.norm1.weight""", F"""model.decoder.layers.{i}.encoder_attn_layer_norm.weight""")		)
								rename_keys.append((F"""transformer.decoder.layers.{i}.norm1.bias""", F"""model.decoder.layers.{i}.encoder_attn_layer_norm.bias""")		)
								rename_keys.append((F"""transformer.decoder.layers.{i}.self_attn.out_proj.weight""", F"""model.decoder.layers.{i}.self_attn.out_proj.weight""")		)
								rename_keys.append((F"""transformer.decoder.layers.{i}.self_attn.out_proj.bias""", F"""model.decoder.layers.{i}.self_attn.out_proj.bias""")		)
								rename_keys.append((F"""transformer.decoder.layers.{i}.norm2.weight""", F"""model.decoder.layers.{i}.self_attn_layer_norm.weight""")		)
								rename_keys.append((F"""transformer.decoder.layers.{i}.norm2.bias""", F"""model.decoder.layers.{i}.self_attn_layer_norm.bias""")		)
								rename_keys.append((F"""transformer.decoder.layers.{i}.linear1.weight""", F"""model.decoder.layers.{i}.fc1.weight""")		)
								rename_keys.append((F"""transformer.decoder.layers.{i}.linear1.bias""", F"""model.decoder.layers.{i}.fc1.bias""")		)
								rename_keys.append((F"""transformer.decoder.layers.{i}.linear2.weight""", F"""model.decoder.layers.{i}.fc2.weight""")		)
								rename_keys.append((F"""transformer.decoder.layers.{i}.linear2.bias""", F"""model.decoder.layers.{i}.fc2.bias""")		)
								rename_keys.append((F"""transformer.decoder.layers.{i}.norm3.weight""", F"""model.decoder.layers.{i}.final_layer_norm.weight""")		)
								rename_keys.append((F"""transformer.decoder.layers.{i}.norm3.bias""", F"""model.decoder.layers.{i}.final_layer_norm.bias""")		)

				# fmt: on

				return rename_keys



def 						_UpperCamelCase							( UpperCamelCase__     ,   UpperCamelCase__     ,   UpperCamelCase__		):






				"""simple docstring"""



				__magic_name__				:					Optional[Any]						       =       dct.pop(__a		)
				__magic_name__				:					List[Any]						       =       val



def 						_UpperCamelCase							( UpperCamelCase__     ,   UpperCamelCase__		):






				"""simple docstring"""



				__magic_name__				:					Tuple						       =       config.d_model
				for i in range(config.decoder_layers		):
								# read in weights + bias of input projection layer of self-attention
								__magic_name__				:					List[str]						       =       state_dict.pop(F"""transformer.decoder.layers.{i}.self_attn.in_proj_weight"""		)
								__magic_name__				:					Optional[Any]						       =       state_dict.pop(F"""transformer.decoder.layers.{i}.self_attn.in_proj_bias"""		)
								# next, add query, keys and values (in that order) to the state dict
								__magic_name__				:					Dict						       =       in_proj_weight[:hidden_size, :]
								__magic_name__				:					List[Any]						       =       in_proj_bias[:hidden_size]
								__magic_name__				:					int						       =       in_proj_weight[
								    hidden_size : hidden_size * 2, :
								]
								__magic_name__				:					Any						       =       in_proj_bias[hidden_size : hidden_size * 2]
								__magic_name__				:					List[str]						       =       in_proj_weight[-hidden_size:, :]
								__magic_name__				:					Dict						       =       in_proj_bias[-hidden_size:]



def 						_UpperCamelCase							( ):






				"""simple docstring"""



				__magic_name__				:					Tuple						       =       '''http://images.cocodataset.org/val2017/000000039769.jpg'''
				__magic_name__				:					Union[str, Any]						       =       Image.open(requests.get(__a     ,   stream=__a		).raw		)

				return im



@torch.no_grad()
def 						_UpperCamelCase							( UpperCamelCase__     ,   UpperCamelCase__     ,   UpperCamelCase__		):






				"""simple docstring"""



				__magic_name__				:					str						       =       get_deta_config()

				# load original state dict
				if model_name == "deta-resnet-50":
								__magic_name__				:					int						       =       '''adet_checkpoint0011.pth'''
				elif model_name == "deta-resnet-50-24-epochs":
								__magic_name__				:					str						       =       '''adet_2x_checkpoint0023.pth'''
				else:
								raise ValueError(F"""Model name {model_name} not supported"""		)
				__magic_name__				:					Optional[int]						       =       hf_hub_download(repo_id="nielsr/deta-checkpoints"     ,   filename=__a		)
				__magic_name__				:					str						       =       torch.load(__a     ,   map_location="cpu"		)['''model''']

				# rename keys
				__magic_name__				:					int						       =       create_rename_keys(__a		)
				for src, dest in rename_keys:
								rename_key(__a     ,   __a     ,   __a		)
				read_in_decoder_q_k_v(__a     ,   __a		)

				# fix some prefixes
				for key in state_dict.copy().keys():
								if "transformer.decoder.class_embed" in key or "transformer.decoder.bbox_embed" in key:
												__magic_name__				:					str						       =       state_dict.pop(__a		)
												__magic_name__				:					List[str]						       =       val
								if "input_proj" in key:
												__magic_name__				:					Tuple						       =       state_dict.pop(__a		)
												__magic_name__				:					Tuple						       =       val
								if "level_embed" in key or "pos_trans" in key or "pix_trans" in key or "enc_output" in key:
												__magic_name__				:					Dict						       =       state_dict.pop(__a		)
												__magic_name__				:					List[str]						       =       val

    # finally, create HuggingFace model and load state dict
				__magic_name__				:					str						       =       DetaForObjectDetection(__a		)
				model.load_state_dict(__a		)
				model.eval()

				__magic_name__				:					int						       =       '''cuda''' if torch.cuda.is_available() else '''cpu'''
				model.to(__a		)

				# load image processor
				__magic_name__				:					Optional[Any]						       =       DetaImageProcessor(format="coco_detection"		)

				# verify our conversion on image
				__magic_name__				:					Union[str, Any]						       =       prepare_img()
				__magic_name__				:					Any						       =       processor(images=__a     ,   return_tensors="pt"		)
				__magic_name__				:					List[Any]						       =       encoding['''pixel_values''']
				__magic_name__				:					Dict						       =       model(pixel_values.to(__a		)		)

				# verify logits
				if model_name == "deta-resnet-50":
								__magic_name__				:					Optional[int]						       =       torch.tensor(
								    [[-7.3978, -2.5406, -4.1668], [-8.2684, -3.9933, -3.8096], [-7.0515, -3.7973, -5.8516]]		)
								__magic_name__				:					Tuple						       =       torch.tensor([[0.5043, 0.4973, 0.9998], [0.2542, 0.5489, 0.4748], [0.5490, 0.2765, 0.0570]]		)
				elif model_name == "deta-resnet-50-24-epochs":
								__magic_name__				:					Any						       =       torch.tensor(
								    [[-7.1688, -2.4857, -4.8669], [-7.8630, -3.8154, -4.2674], [-7.2730, -4.1865, -5.5323]]		)
								__magic_name__				:					List[Any]						       =       torch.tensor([[0.5021, 0.4971, 0.9994], [0.2546, 0.5486, 0.4731], [0.1686, 0.1986, 0.2142]]		)

				assert torch.allclose(outputs.logits[0, :3, :3]     ,   expected_logits.to(__a		)     ,   atol=1E-4		)
				assert torch.allclose(outputs.pred_boxes[0, :3, :3]     ,   expected_boxes.to(__a		)     ,   atol=1E-4		)
				print("Everything ok!"		)

				if pytorch_dump_folder_path:
								# Save model and processor
								logger.info(F"""Saving PyTorch model and processor to {pytorch_dump_folder_path}..."""		)
								Path(__a		).mkdir(exist_ok=__a		)
								model.save_pretrained(__a		)
								processor.save_pretrained(__a		)

				# Push to hub
				if push_to_hub:
								print("Pushing model and processor to hub..."		)
								model.push_to_hub(F"""jozhang97/{model_name}"""		)
								processor.push_to_hub(F"""jozhang97/{model_name}"""		)


if __name__ == "__main__":
		_SCREAMING_SNAKE_CASE       :   Optional[int]        = argparse.ArgumentParser()

		parser.add_argument(
		    "--model_name",
		    type=str,
		    default="deta-resnet-50",
		    choices=["deta-resnet-50", "deta-resnet-50-24-epochs"],
		    help="Name of the model you'd like to convert.",
		)
		parser.add_argument(
		    "--pytorch_dump_folder_path",
		    default=None,
		    type=str,
		    help="Path to the folder to output PyTorch model.",
		)
		parser.add_argument(
		    "--push_to_hub", action="store_true", help="Whether or not to push the converted model to the 🤗 hub."
		)
		_SCREAMING_SNAKE_CASE       :   List[Any]        = parser.parse_args()
		convert_deta_checkpoint(args.model_name, args.pytorch_dump_folder_path, args.push_to_hub)



'''simple docstring'''







import json
import os
import unittest

from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import (
    VOCAB_FILES_NAMES,
    GPTNeoXJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin






@require_tokenizers
class 		_snake_case    (				snake_case_  ,  unittest.TestCase		):






						'''simple docstring'''







						__snake_case   =      GPTNeoXJapaneseTokenizer
						__snake_case   =      False
						__snake_case   =      {"do_clean_text": False, "add_prefix_space": False}







						def lowerCAmelCase__ (  self:	Any     )							->	List[str]:
										super().setUp()

										__magic_name__				:					Union[str, Any]						       =       [
										    "こん",
										    "こんに",
										    "にちは",
										    "ばんは",
										    "世界,㔺界",
										    "、",
										    "。",
										    "<BR>",
										    "<SP>",
										    "<TAB>",
										    "<URL>",
										    "<EMAIL>",
										    "<TEL>",
										    "<DATE>",
										    "<PRICE>",
										    "<BLOCK>",
										    "<KIGOU>",
										    "<U2000U2BFF>",
										    "<|emoji1|>",
										    "<unk>",
										    "<|startoftext|>",
										    "<|endoftext|>",
										]
										__magic_name__				:					List[str]						       =       {"emoji": {"\ud83d\ude00": "<|emoji1|>"}, "emoji_inv": {"<|emoji1|>": "\ud83d\ude00"}}  # 😀
										__magic_name__				:					str						       =       {"unk_token": "<unk>"}

										__magic_name__				:					Dict						       =       os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES["vocab_file"]     )
										__magic_name__				:					Dict						       =       os.path.join(self.tmpdirname      ,							VOCAB_FILES_NAMES["emoji_file"]     )
										with open(self.vocab_file      ,							"w"      ,							encoding="utf-8"     ) as vocab_writer:
														vocab_writer.write("".join([x + "\n" for x in vocab_tokens]     )     )
										with open(self.emoji_file      ,							"w"     ) as emoji_writer:
														emoji_writer.write(json.dumps(__UpperCamelCase     )     )







						def lowerCAmelCase__ (  self:	Tuple      ,							**__UpperCamelCase:	str     )							->	Union[str, Any]:
										kwargs.update(self.special_tokens_map     )
										return GPTNeoXJapaneseTokenizer.from_pretrained(self.tmpdirname      ,							**__UpperCamelCase     )







						def lowerCAmelCase__ (  self:	List[str]      ,							__UpperCamelCase:	str     )							->	List[str]:
										__magic_name__				:					str						       =       "こんにちは、世界。 \nこんばんは、㔺界。😀"
										__magic_name__				:					str						       =       "こんにちは、世界。 \nこんばんは、世界。😀"
										return input_text, output_text







						def lowerCAmelCase__ (  self:	Union[str, Any]      ,							__UpperCamelCase:	List[Any]     )							->	Dict:
										__magic_name__       ,     __magic_name__				:					Union[str, Any]						       =       self.get_input_output_texts(__UpperCamelCase     )
										__magic_name__				:					List[Any]						       =       tokenizer.encode(__UpperCamelCase      ,							add_special_tokens=__UpperCamelCase     )
										__magic_name__				:					Optional[Any]						       =       tokenizer.decode(__UpperCamelCase      ,							clean_up_tokenization_spaces=__UpperCamelCase     )
										return text, ids







						def lowerCAmelCase__ (  self:	Optional[int]     )							->	Union[str, Any]:
										pass  # TODO add if relevant







						def lowerCAmelCase__ (  self:	str     )							->	List[str]:
										pass  # TODO add if relevant







						def lowerCAmelCase__ (  self:	str     )							->	str:
										pass  # TODO add if relevant







						def lowerCAmelCase__ (  self:	Union[str, Any]     )							->	Dict:
										__magic_name__				:					List[str]						       =       self.get_tokenizer()

										# Testing tokenization
										__magic_name__				:					Any						       =       "こんにちは、世界。　こんばんは、㔺界。"
										__magic_name__				:					Optional[int]						       =       ["こん", "にちは", "、", "世界", "。", "<SP>", "こん", "ばんは", "、", "㔺界", "。"]
										__magic_name__				:					List[Any]						       =       tokenizer.tokenize(__UpperCamelCase     )
										self.assertListEqual(__UpperCamelCase      ,							__UpperCamelCase     )

										# Testing conversion to ids without special tokens
										__magic_name__				:					int						       =       [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
										__magic_name__				:					str						       =       tokenizer.convert_tokens_to_ids(__UpperCamelCase     )
										self.assertListEqual(__UpperCamelCase      ,							__UpperCamelCase     )

										# Testing conversion to ids with special tokens
										__magic_name__				:					str						       =       tokens + [tokenizer.unk_token]
										__magic_name__				:					Tuple						       =       [0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
										__magic_name__				:					Dict						       =       tokenizer.convert_tokens_to_ids(__UpperCamelCase     )
										self.assertListEqual(__UpperCamelCase      ,							__UpperCamelCase     )







						@slow
						def lowerCAmelCase__ (  self:	List[Any]     )							->	Union[str, Any]:
										__magic_name__				:					List[str]						       =       self.tokenizer_class.from_pretrained("abeja/gpt-neox-japanese-2.7b"     )

										__magic_name__				:					List[Any]						       =       tokenizer.encode("ありがとう。"      ,							add_special_tokens=__UpperCamelCase     )
										__magic_name__				:					Optional[Any]						       =       tokenizer.encode("どういたしまして。"      ,							add_special_tokens=__UpperCamelCase     )

										__magic_name__				:					List[Any]						       =       tokenizer.build_inputs_with_special_tokens(__UpperCamelCase     )
										__magic_name__				:					str						       =       tokenizer.build_inputs_with_special_tokens(__UpperCamelCase      ,							__UpperCamelCase     )

										assert encoded_sentence == ids_a
										assert encoded_pair == ids_a + ids_a







						def lowerCAmelCase__ (  self:	Tuple     )							->	Dict:
										# Intentionally convert some words to accommodate character fluctuations unique to Japanese
										pass







						def lowerCAmelCase__ (  self:	Dict     )							->	Optional[int]:
										# tokenizer has no padding token
										pass