


import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


lowerCAmelCase__						:Any         =							'''block_records.npy'''


lowerCAmelCase__						:Tuple         =							logging.get_logger(__name__)

def  lowerCAmelCase__				(    a__:	str    ,						a__:	int )						->	np.ndarray:


							'''simple docstring'''





							import tensorflow.compat.va as tf

							_UpperCAmelCase					=    tf.data.TFRecordDataset(a__    ,						buffer_size=5_1_2 * 1_0_2_4 * 1_0_2_4 )
							_UpperCAmelCase					=    blocks_dataset.batch(a__    ,						drop_remainder=a__ )
							_UpperCAmelCase					=    next(blocks_dataset.take(1 ).as_numpy_iterator() )

							return np_record

class 	__a   :
		def __init__(    self    ,					_SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE=2    ,					_SCREAMING_SNAKE_CASE=1000    ,					_SCREAMING_SNAKE_CASE=100    ,					_SCREAMING_SNAKE_CASE=100000    ,					)				-> int:




									"""simple docstring"""





									from scann.scann_ops.py.scann_ops_pybind import builder as Builder

									_UpperCAmelCase					=    Builder(db=_SCREAMING_SNAKE_CASE    ,					num_neighbors=_SCREAMING_SNAKE_CASE    ,					distance_measure='dot_product'		)
									_UpperCAmelCase					=    builder.tree(
									    num_leaves=_SCREAMING_SNAKE_CASE    ,					num_leaves_to_search=_SCREAMING_SNAKE_CASE    ,					training_sample_size=_SCREAMING_SNAKE_CASE		)
									_UpperCAmelCase					=    builder.score_ah(dimensions_per_block=_SCREAMING_SNAKE_CASE		)

									_UpperCAmelCase					=    builder.build()






		def 			UpperCAmelCase__       (    self    ,					_SCREAMING_SNAKE_CASE		)				-> Any:




									"""simple docstring"""





									_UpperCAmelCase	, _UpperCAmelCase					=    self.searcher.search_batched(question_projection.detach().cpu()		)
									return retrieved_block_ids.astype('int64'		)



class 	__a   :
		def __init__(    self    ,					_SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE		)				-> List[str]:




									"""simple docstring"""





									super().__init__()
									_UpperCAmelCase					=    block_records
									_UpperCAmelCase					=    tokenizer
		def __call__(    self    ,					_SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE=None    ,					_SCREAMING_SNAKE_CASE="pt"		)				-> int:




									"""simple docstring"""





									_UpperCAmelCase					=    np.take(self.block_records    ,					indices=_SCREAMING_SNAKE_CASE    ,					axis=0		)

									_UpperCAmelCase					=    self.tokenizer.decode(question_input_ids[0]    ,					skip_special_tokens=_SCREAMING_SNAKE_CASE		)

									_UpperCAmelCase					=    []
									_UpperCAmelCase					=    []
									for retrieved_block in retrieved_blocks:
																text.append(_SCREAMING_SNAKE_CASE		)
																text_pair.append(retrieved_block.decode()		)

									_UpperCAmelCase					=    self.tokenizer(
									    _SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE    ,					padding=_SCREAMING_SNAKE_CASE    ,					truncation=_SCREAMING_SNAKE_CASE    ,					return_special_tokens_mask=_SCREAMING_SNAKE_CASE    ,					max_length=_SCREAMING_SNAKE_CASE		)
									_UpperCAmelCase					=    concat_inputs.convert_to_tensors(_SCREAMING_SNAKE_CASE		)

									if answer_ids is not None:
																return self.block_has_answer(_SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE		) + (concat_inputs_tensors,)
									else:
																return (None, None, None, concat_inputs_tensors)
		@classmethod
		def 			UpperCAmelCase__       (    cls    ,					_SCREAMING_SNAKE_CASE    ,					*_SCREAMING_SNAKE_CASE    ,					**_SCREAMING_SNAKE_CASE		)				-> Tuple:




									"""simple docstring"""





									if os.path.isdir(_SCREAMING_SNAKE_CASE		):
																_UpperCAmelCase					=    os.path.join(_SCREAMING_SNAKE_CASE    ,					_REALM_BLOCK_RECORDS_FILENAME		)
									else:
																_UpperCAmelCase					=    hf_hub_download(
																    repo_id=_SCREAMING_SNAKE_CASE    ,					filename=_REALM_BLOCK_RECORDS_FILENAME    ,					**_SCREAMING_SNAKE_CASE		)
									_UpperCAmelCase					=    np.load(_SCREAMING_SNAKE_CASE    ,					allow_pickle=_SCREAMING_SNAKE_CASE		)

									_UpperCAmelCase					=    AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    ,					*_SCREAMING_SNAKE_CASE    ,					**_SCREAMING_SNAKE_CASE		)

									return cls(_SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE		)
		def 			UpperCAmelCase__       (    self    ,					_SCREAMING_SNAKE_CASE		)				-> int:




									"""simple docstring"""





									np.save(os.path.join(_SCREAMING_SNAKE_CASE    ,					_REALM_BLOCK_RECORDS_FILENAME		)    ,					self.block_records		)
									# save tokenizer
									self.tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE		)






		def 			UpperCAmelCase__       (    self    ,					_SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE		)				-> int:




									"""simple docstring"""





									_UpperCAmelCase					=    []
									_UpperCAmelCase					=    []
									_UpperCAmelCase					=    []
									_UpperCAmelCase					=    0

									for input_id in concat_inputs.input_ids:
																_UpperCAmelCase					=    input_id.tolist()
																# Check answers between two [SEP] tokens
																_UpperCAmelCase					=    input_id_list.index(self.tokenizer.sep_token_id		)
																_UpperCAmelCase					=    first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id		)

																start_pos.append([]		)
																end_pos.append([]		)
																for answer in answer_ids:
																							for idx in range(first_sep_idx + 1    ,					_SCREAMING_SNAKE_CASE		):
																														if answer[0] == input_id_list[idx]:
																																					if input_id_list[idx : idx + len(_SCREAMING_SNAKE_CASE		)] == answer:
																																												start_pos[-1].append(_SCREAMING_SNAKE_CASE		)
																																												end_pos[-1].append(idx + len(_SCREAMING_SNAKE_CASE		) - 1		)

																if len(start_pos[-1]		) == 0:
																							has_answers.append(_SCREAMING_SNAKE_CASE		)
																else:
																							has_answers.append(_SCREAMING_SNAKE_CASE		)
																							if len(start_pos[-1]		) > max_answers:
																														_UpperCAmelCase					=    len(start_pos[-1]		)

        # Pad -1 to max_answers
									for start_pos_, end_pos_ in zip(_SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE		):
																if len(start_pos_		) < max_answers:
																							_UpperCAmelCase					=    [-1] * (max_answers - len(start_pos_		))
																							start_pos_ += padded
																							end_pos_ += padded
									return has_answers, start_pos, end_pos




import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding

class 	__a   (       UpperCAmelCase    ):
		_a			:						int    			=			['image_processor', 'tokenizer']
		_a			:						Union[str, Any]    			=			'AutoImageProcessor'
		_a			:						Optional[int]    			=			'AutoTokenizer'
		def __init__(    self    ,					_SCREAMING_SNAKE_CASE=None    ,					_SCREAMING_SNAKE_CASE=None    ,					**_SCREAMING_SNAKE_CASE		)				-> Tuple:




									"""simple docstring"""





									_UpperCAmelCase					=    None
									if "feature_extractor" in kwargs:
																warnings.warn(
																    'The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'
																    ' instead.'    ,					_SCREAMING_SNAKE_CASE    ,					)
																_UpperCAmelCase					=    kwargs.pop('feature_extractor'		)

									_UpperCAmelCase					=    image_processor if image_processor is not None else feature_extractor
									if image_processor is None:
																raise ValueError('You have to specify an image_processor.'		)
									if tokenizer is None:
																raise ValueError('You have to specify a tokenizer.'		)

									super().__init__(_SCREAMING_SNAKE_CASE    ,					_SCREAMING_SNAKE_CASE		)
									_UpperCAmelCase					=    self.image_processor
		def __call__(    self    ,					_SCREAMING_SNAKE_CASE=None    ,					_SCREAMING_SNAKE_CASE=None    ,					_SCREAMING_SNAKE_CASE=None    ,					**_SCREAMING_SNAKE_CASE		)				-> Dict:




									"""simple docstring"""





									if text is None and images is None:
																raise ValueError('You have to specify either text or images. Both cannot be none.'		)

									if text is not None:
																_UpperCAmelCase					=    self.tokenizer(_SCREAMING_SNAKE_CASE    ,					return_tensors=_SCREAMING_SNAKE_CASE    ,					**_SCREAMING_SNAKE_CASE		)

									if images is not None:
																_UpperCAmelCase					=    self.image_processor(_SCREAMING_SNAKE_CASE    ,					return_tensors=_SCREAMING_SNAKE_CASE    ,					**_SCREAMING_SNAKE_CASE		)

									if text is not None and images is not None:
																_UpperCAmelCase					=    image_features.pixel_values
																return encoding
									elif text is not None:
																return encoding
									else:
																return BatchEncoding(data=dict(**_SCREAMING_SNAKE_CASE		)    ,					tensor_type=_SCREAMING_SNAKE_CASE		)
		def 			UpperCAmelCase__       (    self    ,					*_SCREAMING_SNAKE_CASE    ,					**_SCREAMING_SNAKE_CASE		)				-> int:




									"""simple docstring"""





									return self.tokenizer.batch_decode(*_SCREAMING_SNAKE_CASE    ,					**_SCREAMING_SNAKE_CASE		)
		def 			UpperCAmelCase__       (    self    ,					*_SCREAMING_SNAKE_CASE    ,					**_SCREAMING_SNAKE_CASE		)				-> Tuple:




									"""simple docstring"""





									return self.tokenizer.decode(*_SCREAMING_SNAKE_CASE    ,					**_SCREAMING_SNAKE_CASE		)
		@property
		def 			UpperCAmelCase__       (    self		)				-> Optional[int]:




									"""simple docstring"""





									_UpperCAmelCase					=    self.tokenizer.model_input_names
									_UpperCAmelCase					=    self.image_processor.model_input_names
									return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names		)		)
		@property
		def 			UpperCAmelCase__       (    self		)				-> Any:




									"""simple docstring"""





									warnings.warn(
									    '`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'    ,					_SCREAMING_SNAKE_CASE    ,					)
									return self.image_processor_class






		@property
		def 			UpperCAmelCase__       (    self		)				-> Any:




									"""simple docstring"""





									warnings.warn(
									    '`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'    ,					_SCREAMING_SNAKE_CASE    ,					)
									return self.image_processor
