"""simple docstring"""






import argparse
import collections
from pathlib import Path

import torch
from torch.serialization import default_restore_location

from transformers import BertConfig, DPRConfig, DPRContextEncoder, DPRQuestionEncoder, DPRReader


UpperCamelCase     					=     collections.namedtuple(
    'CheckpointState', ['model_dict', 'optimizer_dict', 'scheduler_dict', 'offset', 'epoch', 'encoder_params']
)

def  lowerCAmelCase_      (_SCREAMING_SNAKE_CASE	:str  )				-> Union[str, Any]:

						'''simple docstring'''





						print(F'''Reading saved model from {model_file}'''  )
						a_							:			str     		=					torch.load(_A		,			map_location=lambda _SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE  : default_restore_location(_A		,			"cpu"  )  )
						return CheckpointState(**_A  )
class   UpperCAmelCase__				:


	"""simple docstring"""







	def __init__(			self       ,     _SCREAMING_SNAKE_CASE )    ->	int:
							a_							:			List[Any]     		=					src_file







	def       A    (			self )    ->	int:
							raise NotImplementedError







	@staticmethod
	def       A    (			_SCREAMING_SNAKE_CASE       ,     *_SCREAMING_SNAKE_CASE       ,     **_SCREAMING_SNAKE_CASE )    ->	"DPRState":






							if comp_type.startswith("c" ):
													return DPRContextEncoderState(*lowercase_       ,     **lowercase_ )
							if comp_type.startswith("q" ):
													return DPRQuestionEncoderState(*lowercase_       ,     **lowercase_ )
							if comp_type.startswith("r" ):
													return DPRReaderState(*lowercase_       ,     **lowercase_ )
							else:
													raise ValueError("Component type must be either \'ctx_encoder\', \'question_encoder\' or \'reader\'." )




class   UpperCAmelCase__				(    lowerCamelCase__			):


	"""simple docstring"""







	def       A    (			self )    ->	Any:
							a_							:			List[str]     		=					DPRContextEncoder(DPRConfig(**BertConfig.get_config_dict("bert-base-uncased" )[0] ) )
							print(f'''Loading DPR biencoder from {self.src_file}''' )
							a_							:			List[str]     		=					load_states_from_checkpoint(self.src_file )
							a_		,					a_							:			int     		=					model.ctx_encoder, "ctx_model."
							# Fix changes from https://github.com/huggingface/transformers/commit/614fef1691edb806de976756d4948ecbcd0c0ca3
							a_							:			Dict     		=					{"bert_model.embeddings.position_ids": model.ctx_encoder.bert_model.embeddings.position_ids}
							for key, value in saved_state.model_dict.items():
													if key.startswith(lowercase_ ):
																			a_							:			Optional[int]     		=					key[len(lowercase_ ) :]
																			if not key.startswith("encode_proj." ):
																									a_							:			Union[str, Any]     		=					"bert_model." + key
																			a_							:			Tuple     		=					value
							encoder.load_state_dict(lowercase_ )
							return model




class   UpperCAmelCase__				(    lowerCamelCase__			):


	"""simple docstring"""







	def       A    (			self )    ->	Tuple:
							a_							:			Any     		=					DPRQuestionEncoder(DPRConfig(**BertConfig.get_config_dict("bert-base-uncased" )[0] ) )
							print(f'''Loading DPR biencoder from {self.src_file}''' )
							a_							:			str     		=					load_states_from_checkpoint(self.src_file )
							a_		,					a_							:			Any     		=					model.question_encoder, "question_model."
							# Fix changes from https://github.com/huggingface/transformers/commit/614fef1691edb806de976756d4948ecbcd0c0ca3
							a_							:			int     		=					{"bert_model.embeddings.position_ids": model.question_encoder.bert_model.embeddings.position_ids}
							for key, value in saved_state.model_dict.items():
													if key.startswith(lowercase_ ):
																			a_							:			Dict     		=					key[len(lowercase_ ) :]
																			if not key.startswith("encode_proj." ):
																									a_							:			Union[str, Any]     		=					"bert_model." + key
																			a_							:			Tuple     		=					value
							encoder.load_state_dict(lowercase_ )
							return model




class   UpperCAmelCase__				(    lowerCamelCase__			):


	"""simple docstring"""







	def       A    (			self )    ->	Union[str, Any]:
							a_							:			List[str]     		=					DPRReader(DPRConfig(**BertConfig.get_config_dict("bert-base-uncased" )[0] ) )
							print(f'''Loading DPR reader from {self.src_file}''' )
							a_							:			Tuple     		=					load_states_from_checkpoint(self.src_file )
							# Fix changes from https://github.com/huggingface/transformers/commit/614fef1691edb806de976756d4948ecbcd0c0ca3
							a_							:			Any     		=					{
							    "encoder.bert_model.embeddings.position_ids": model.span_predictor.encoder.bert_model.embeddings.position_ids
							}
							for key, value in saved_state.model_dict.items():
													if key.startswith("encoder." ) and not key.startswith("encoder.encode_proj" ):
																			a_							:			Optional[Any]     		=					"encoder.bert_model." + key[len("encoder." ) :]
													a_							:			List[str]     		=					value
							model.span_predictor.load_state_dict(lowercase_ )
							return model




def  lowerCAmelCase_      (_SCREAMING_SNAKE_CASE	:str		,			_SCREAMING_SNAKE_CASE	:Path		,			_SCREAMING_SNAKE_CASE	:Path  )				-> Optional[int]:

						'''simple docstring'''





						a_							:			Any     		=					Path(_A  )
						dest_dir.mkdir(exist_ok=_A  )

						a_							:			Optional[Any]     		=					DPRState.from_type(_A		,			src_file=_A  )
						a_							:			Dict     		=					dpr_state.load_dpr_model()
						model.save_pretrained(_A  )
						model.from_pretrained(_A  )  # sanity check


if __name__ == "__main__":
				UpperCamelCase     					=     argparse.ArgumentParser()
				# Required parameters
				parser.add_argument(
				    '--type', type=str, help='Type of the component to convert: \'ctx_encoder\', \'question_encoder\' or \'reader\'.'
				)
				parser.add_argument(
				    '--src',
				    type=str,
				    help=(
				        'Path to the dpr checkpoint file. They can be downloaded from the official DPR repo'
				        ' https://github.com/facebookresearch/DPR. Note that in the official repo, both encoders are stored in the'
				        ' \'retriever\' checkpoints.'
				    ),
				)
				parser.add_argument('--dest', type=str, default=None, help='Path to the output PyTorch model directory.')
				UpperCamelCase     					=     parser.parse_args()

				UpperCamelCase     					=     Path(args.src)
				UpperCamelCase     					=     F'converted-{src_file.name}' if args.dest is None else args.dest
				UpperCamelCase     					=     Path(dest_dir)
				assert src_file.exists()
				assert (
				    args.type is not None
				), "Please specify the component type of the DPR model to convert: 'ctx_encoder', 'question_encoder' or 'reader'."
				convert(args.type, src_file, dest_dir)





"""simple docstring"""






import json
import os
import tempfile

import datasets
from datasets.arrow_writer import ArrowWriter
from datasets.features import ArrayaD
from utils import generate_examples, get_duration


UpperCamelCase     					=     (30, 4_87)
UpperCamelCase     					=     (36, 10_24)
UpperCamelCase     					=     (1_00, 1_00)
UpperCamelCase     					=     1_00

UpperCamelCase     					=     datasets.Features(
    {'text': ArrayaD(SHAPE_TEST_1, dtype='float32'), 'image': ArrayaD(SHAPE_TEST_2, dtype='float32')}
)

UpperCamelCase					,      UpperCamelCase     					=     os.path.split(__file__)
UpperCamelCase     					=     os.path.join(RESULTS_BASEPATH, 'results', RESULTS_FILENAME.replace('.py', '.json'))

@get_duration
def  lowerCAmelCase_      (_SCREAMING_SNAKE_CASE	:Tuple		,			_SCREAMING_SNAKE_CASE	:str		,			_SCREAMING_SNAKE_CASE	:Tuple  )				-> Any:
						with ArrowWriter(features=_SCREAMING_SNAKE_CASE		,			path=os.path.join(_SCREAMING_SNAKE_CASE		,			"beta.arrow"  )  ) as writer:
												for key, record in dummy_data:
																		a_							:			int     		=					my_features.encode_example(_SCREAMING_SNAKE_CASE  )
																		writer.write(_SCREAMING_SNAKE_CASE  )
												a_		,					a_							:			str     		=					writer.finalize()

@get_duration
def  lowerCAmelCase_      (_SCREAMING_SNAKE_CASE	:List[str]		,			_SCREAMING_SNAKE_CASE	:Any  )				-> Tuple:
						a_							:			Dict     		=					datasets.Dataset.from_file(
						    filename=os.path.join(_SCREAMING_SNAKE_CASE		,			"beta.arrow"  )		,			info=datasets.DatasetInfo(features=_SCREAMING_SNAKE_CASE  )  )
						for _ in dataset:
												pass

@get_duration
def  lowerCAmelCase_      (_SCREAMING_SNAKE_CASE	:str		,			_SCREAMING_SNAKE_CASE	:Dict  )				-> Optional[Any]:
						a_							:			List[str]     		=					datasets.Dataset.from_file(
						    filename=os.path.join(_SCREAMING_SNAKE_CASE		,			"beta.arrow"  )		,			info=datasets.DatasetInfo(features=_SCREAMING_SNAKE_CASE  )  )
						dataset.set_format("numpy"  )
						for _ in dataset:
												pass

@get_duration
def  lowerCAmelCase_      (_SCREAMING_SNAKE_CASE	:List[Any]		,			_SCREAMING_SNAKE_CASE	:List[str]  )				-> List[str]:
						a_							:			Optional[Any]     		=					10
						a_							:			Optional[int]     		=					datasets.Dataset.from_file(
						    filename=os.path.join(_SCREAMING_SNAKE_CASE		,			"beta.arrow"  )		,			info=datasets.DatasetInfo(features=_SCREAMING_SNAKE_CASE  )  )
						for i in range(0		,			len(_SCREAMING_SNAKE_CASE  )		,			_SCREAMING_SNAKE_CASE  ):
												a_							:			List[str]     		=					dataset[i : i + batch_size]

@get_duration
def  lowerCAmelCase_      (_SCREAMING_SNAKE_CASE	:Optional[int]		,			_SCREAMING_SNAKE_CASE	:List[str]  )				-> Optional[int]:
						a_							:			List[str]     		=					10
						a_							:			int     		=					datasets.Dataset.from_file(
						    filename=os.path.join(_SCREAMING_SNAKE_CASE		,			"beta.arrow"  )		,			info=datasets.DatasetInfo(features=_SCREAMING_SNAKE_CASE  )  )
						dataset.set_format("numpy"  )
						for i in range(0		,			len(_SCREAMING_SNAKE_CASE  )		,			_SCREAMING_SNAKE_CASE  ):
												a_							:			str     		=					dataset[i : i + batch_size]

@get_duration
def  lowerCAmelCase_      (_SCREAMING_SNAKE_CASE	:List[str]		,			_SCREAMING_SNAKE_CASE	:Optional[Any]  )				-> Dict:
						a_							:			List[str]     		=					datasets.Dataset.from_file(
						    filename=os.path.join(_SCREAMING_SNAKE_CASE		,			"beta.arrow"  )		,			info=datasets.DatasetInfo(features=_SCREAMING_SNAKE_CASE  )  )
						for col in feats:
												a_							:			List[Any]     		=					dataset[col]

@get_duration
def  lowerCAmelCase_      (_SCREAMING_SNAKE_CASE	:str		,			_SCREAMING_SNAKE_CASE	:Optional[Any]  )				-> Dict:
						a_							:			Any     		=					datasets.Dataset.from_file(
						    filename=os.path.join(_SCREAMING_SNAKE_CASE		,			"beta.arrow"  )		,			info=datasets.DatasetInfo(features=_SCREAMING_SNAKE_CASE  )  )
						dataset.set_format("numpy"  )
						for col in feats:
												a_							:			Any     		=					dataset[col]

def  lowerCAmelCase_      ()				-> Optional[Any]:
						a_							:			List[Any]     		=					{}
						a_							:			str     		=					(
						    read_unformated,
						    read_formatted_as_numpy,
						    read_batch_unformated,
						    read_batch_formatted_as_numpy,
						    read_col_unformated,
						    read_col_formatted_as_numpy,
						)
						with tempfile.TemporaryDirectory() as tmp_dir:
												a_							:			Optional[int]     		=					datasets.Features({"image": ArrayaD(_SCREAMING_SNAKE_CASE		,			dtype="float32"  )}  )
												a_							:			Optional[Any]     		=					generate_examples(features=_SCREAMING_SNAKE_CASE		,			num_examples=_SCREAMING_SNAKE_CASE  )
												a_							:			List[str]     		=					write(_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE  )
												for read_func in read_functions:
																		a_							:			Optional[int]     		=					read_func(_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE  )

						with tempfile.TemporaryDirectory() as tmp_dir:
												# don't use fixed length for fair comparison
												# feats = datasets.Features(
												#     {"image": datasets.Sequence(datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[1]), SPEED_TEST_SHAPE[0])}
												# )
												a_							:			Union[str, Any]     		=					datasets.Features({"image": datasets.Sequence(datasets.Sequence(datasets.Value("float32"  )  )  )}  )
												a_							:			List[str]     		=					generate_examples(
												    features=_SCREAMING_SNAKE_CASE		,			num_examples=_SCREAMING_SNAKE_CASE		,			seq_shapes={"image": SPEED_TEST_SHAPE}  )
												a_							:			str     		=					write(_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE  )
												for read_func in read_functions:
																		a_							:			List[str]     		=					read_func(_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE  )

						with tempfile.TemporaryDirectory() as tmp_dir:
												# don't use fixed length for fair comparison
												# feats = datasets.Features(
												#     {"image": datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1])}
												# )
												a_							:			int     		=					datasets.Features({"image": datasets.Sequence(datasets.Value("float32"  )  )}  )
												a_							:			str     		=					generate_examples(
												    features=_SCREAMING_SNAKE_CASE		,			num_examples=_SCREAMING_SNAKE_CASE		,			seq_shapes={"image": [SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1]]}		,			)
												a_							:			List[Any]     		=					write(_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE  )
												for read_func in read_functions:
																		a_							:			str     		=					read_func(_SCREAMING_SNAKE_CASE		,			_SCREAMING_SNAKE_CASE  )

						with open(_SCREAMING_SNAKE_CASE		,			"wb"  ) as f:
												f.write(json.dumps(_SCREAMING_SNAKE_CASE  ).encode("utf-8"  )  )


if __name__ == "__main__":  # useful to run the profiler
				benchmark_array_xd()




