






import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


a    =       'block_records.npy'


a    =       logging.get_logger(__name__)
def       a__						(  A		,				A					):
   import tensorflow.compat.va as tf

   lowercase__						:	Optional[int]         = tf.data.TFRecordDataset(A		,				buffer_size=512 * 1_024 * 1_024					)
   lowercase__						:	List[str]         = blocks_dataset.batch(A		,				drop_remainder=A					)
   lowercase__						:	Union[str, Any]         = next(blocks_dataset.take(1					).as_numpy_iterator()					)

   return np_record
class        __snake_case:



 def __init__(	self   :   Optional[Any] ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   Tuple ,		UpperCAmelCase__   :   Optional[Any]=2 ,		UpperCAmelCase__   :   Tuple=1_000 ,		UpperCAmelCase__   :   Optional[int]=100 ,		UpperCAmelCase__   :   Optional[int]=100_000 ,		):

    """simple docstring"""






    from scann.scann_ops.py.scann_ops_pybind import builder as Builder

    lowercase__						:	Tuple         = Builder(db=UpperCAmelCase__ ,		num_neighbors=UpperCAmelCase__ ,		distance_measure="""dot_product"""	)
    lowercase__						:	Union[str, Any]         = builder.tree(
        num_leaves=UpperCAmelCase__ ,		num_leaves_to_search=UpperCAmelCase__ ,		training_sample_size=UpperCAmelCase__	)
    lowercase__						:	Dict         = builder.score_ah(dimensions_per_block=UpperCAmelCase__	)

    lowercase__						:	Union[str, Any]         = builder.build()







 def 						UpperCAmelCase    (	self   :   Optional[Any] ,		UpperCAmelCase__   :   Union[str, Any]	):

    """simple docstring"""






    lowercase__     ,					lowercase__						:	Dict         = self.searcher.search_batched(question_projection.detach().cpu()	)
    return retrieved_block_ids.astype("""int64"""	)
class        __snake_case:



 def __init__(	self   :   Optional[Any] ,		UpperCAmelCase__   :   Optional[int] ,		UpperCAmelCase__   :   List[str]	):

    """simple docstring"""






    super().__init__()
    lowercase__						:	str         = block_records
    lowercase__						:	Tuple         = tokenizer



 def __call__(	self   :   Optional[int] ,		UpperCAmelCase__   :   Dict ,		UpperCAmelCase__   :   Optional[int] ,		UpperCAmelCase__   :   Dict ,		UpperCAmelCase__   :   List[str]=None ,		UpperCAmelCase__   :   str="pt"	):

    """simple docstring"""






    lowercase__						:	Optional[Any]         = np.take(self.block_records ,		indices=UpperCAmelCase__ ,		axis=0	)

    lowercase__						:	Optional[int]         = self.tokenizer.decode(question_input_ids[0] ,		skip_special_tokens=UpperCAmelCase__	)

    lowercase__						:	List[str]         = []
    lowercase__						:	Optional[int]         = []
    for retrieved_block in retrieved_blocks:
       text.append(UpperCAmelCase__	)
       text_pair.append(retrieved_block.decode()	)

    lowercase__						:	Optional[Any]         = self.tokenizer(
        UpperCAmelCase__ ,		UpperCAmelCase__ ,		padding=UpperCAmelCase__ ,		truncation=UpperCAmelCase__ ,		return_special_tokens_mask=UpperCAmelCase__ ,		max_length=UpperCAmelCase__	)
    lowercase__						:	Union[str, Any]         = concat_inputs.convert_to_tensors(UpperCAmelCase__	)

    if answer_ids is not None:
       return self.block_has_answer(UpperCAmelCase__ ,		UpperCAmelCase__	) + (concat_inputs_tensors,)
    else:
       return (None, None, None, concat_inputs_tensors)



 @classmethod
 def 						UpperCAmelCase    (	cls   :   str ,		UpperCAmelCase__   :   Optional[Union[str, os.PathLike]] ,		*UpperCAmelCase__   :   Any ,		**UpperCAmelCase__   :   List[str]	):

    """simple docstring"""






    if os.path.isdir(UpperCAmelCase__	):
       lowercase__						:	Optional[Any]         = os.path.join(UpperCAmelCase__ ,		_REALM_BLOCK_RECORDS_FILENAME	)
    else:
       lowercase__						:	int         = hf_hub_download(
           repo_id=UpperCAmelCase__ ,		filename=_REALM_BLOCK_RECORDS_FILENAME ,		**UpperCAmelCase__	)
    lowercase__						:	int         = np.load(UpperCAmelCase__ ,		allow_pickle=UpperCAmelCase__	)

    lowercase__						:	Optional[Any]         = AutoTokenizer.from_pretrained(UpperCAmelCase__ ,		*UpperCAmelCase__ ,		**UpperCAmelCase__	)

    return cls(UpperCAmelCase__ ,		UpperCAmelCase__	)



 def 						UpperCAmelCase    (	self   :   Any ,		UpperCAmelCase__   :   List[Any]	):

    """simple docstring"""






    np.save(os.path.join(UpperCAmelCase__ ,		_REALM_BLOCK_RECORDS_FILENAME	) ,		self.block_records	)
    # save tokenizer
    self.tokenizer.save_pretrained(UpperCAmelCase__	)







 def 						UpperCAmelCase    (	self   :   Dict ,		UpperCAmelCase__   :   List[Any] ,		UpperCAmelCase__   :   Union[str, Any]	):

    """simple docstring"""






    lowercase__						:	Dict         = []
    lowercase__						:	Union[str, Any]         = []
    lowercase__						:	Dict         = []
    lowercase__						:	Optional[Any]         = 0

    for input_id in concat_inputs.input_ids:
       lowercase__						:	Dict         = input_id.tolist()
       # Check answers between two [SEP] tokens
       lowercase__						:	Dict         = input_id_list.index(self.tokenizer.sep_token_id	)
       lowercase__						:	List[Any]         = first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id	)

       start_pos.append([]	)
       end_pos.append([]	)
       for answer in answer_ids:
          for idx in range(first_sep_idx + 1 ,		UpperCAmelCase__	):
             if answer[0] == input_id_list[idx]:
                if input_id_list[idx : idx + len(UpperCAmelCase__	)] == answer:
                   start_pos[-1].append(UpperCAmelCase__	)
                   end_pos[-1].append(idx + len(UpperCAmelCase__	) - 1	)

       if len(start_pos[-1]	) == 0:
          has_answers.append(UpperCAmelCase__	)
       else:
          has_answers.append(UpperCAmelCase__	)
          if len(start_pos[-1]	) > max_answers:
             lowercase__						:	str         = len(start_pos[-1]	)

        # Pad -1 to max_answers
    for start_pos_, end_pos_ in zip(UpperCAmelCase__ ,		UpperCAmelCase__	):
       if len(start_pos_	) < max_answers:
          lowercase__						:	Optional[int]         = [-1] * (max_answers - len(start_pos_	))
          start_pos_ += padded
          end_pos_ += padded
    return has_answers, start_pos, end_pos










def       a__						(  A					):

   if not isinstance(A		,				A					) or number < 0:
      raise ValueError("""Input must be a non-negative integer"""					)

   lowercase__						:	str         = number & ~(number - 1)
   lowercase__						:	Optional[int]         = 0
   while intermediate:
      intermediate >>= 1
      index += 1
   return index - 1


if __name__ == "__main__":
 import doctest

 doctest.testmod(verbose=True)


