






import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


a	: Optional[Any]      =				"block_records.npy"


a	: Union[str, Any]      =				logging.get_logger(__name__)






def        lowerCamelCase__   (					__lowerCamelCase    :  str						,	__lowerCamelCase    :  int	):
  import tensorflow.compat.va as tf

  __UpperCAmelCase							:			Dict     =						tf.data.TFRecordDataset(__lowerCamelCase						,	buffer_size=512 * 1024 * 1024	)
  __UpperCAmelCase							:			str     =						blocks_dataset.batch(__lowerCamelCase						,	drop_remainder=__lowerCamelCase	)
  __UpperCAmelCase							:			List[str]     =						next(blocks_dataset.take(1	).as_numpy_iterator()	)

  return np_record
class 					a					:



      """simple docstring"""


      def __init__(   self      :    Any	,   __lowercase      :    Any	,   __lowercase      :    Union[str, Any]	,   __lowercase      :    str=2	,   __lowercase      :    int=1000	,   __lowercase      :    List[str]=100	,   __lowercase      :    List[Any]=100000	,   )							->   Optional[int]:
        from scann.scann_ops.py.scann_ops_pybind import builder as Builder

        __UpperCAmelCase							:			Dict     =						Builder(db=__lowercase	,   num_neighbors=__lowercase	,   distance_measure="""dot_product"""      )
        __UpperCAmelCase							:			List[str]     =						builder.tree(
            num_leaves=__lowercase	,   num_leaves_to_search=__lowercase	,   training_sample_size=__lowercase      )
        __UpperCAmelCase							:			List[Any]     =						builder.score_ah(dimensions_per_block=__lowercase      )

        __UpperCAmelCase							:			Optional[Any]     =						builder.build()




      def 			UpperCAmelCase   (   self      :    Union[str, Any]	,   __lowercase      :    List[str]      )							->   List[str]:
        __UpperCAmelCase   ,				__UpperCAmelCase							:			Optional[Any]     =						self.searcher.search_batched(question_projection.detach().cpu()      )
        return retrieved_block_ids.astype("""int64"""      )


class 					a					:



      """simple docstring"""


      def __init__(   self      :    Any	,   __lowercase      :    Optional[int]	,   __lowercase      :    List[Any]      )							->   Any:
        super().__init__()
        __UpperCAmelCase							:			Union[str, Any]     =						block_records
        __UpperCAmelCase							:			Optional[Any]     =						tokenizer
      def __call__(   self      :    Union[str, Any]	,   __lowercase      :    List[Any]	,   __lowercase      :    Union[str, Any]	,   __lowercase      :    int	,   __lowercase      :    Any=None	,   __lowercase      :    List[Any]="pt"      )							->   List[str]:
        __UpperCAmelCase							:			Dict     =						np.take(self.block_records	,   indices=__lowercase	,   axis=0      )

        __UpperCAmelCase							:			Tuple     =						self.tokenizer.decode(question_input_ids[0]	,   skip_special_tokens=__lowercase      )

        __UpperCAmelCase							:			Tuple     =						[]
        __UpperCAmelCase							:			Union[str, Any]     =						[]
        for retrieved_block in retrieved_blocks:
          text.append(__lowercase      )
          text_pair.append(retrieved_block.decode()      )

        __UpperCAmelCase							:			Union[str, Any]     =						self.tokenizer(
            __lowercase	,   __lowercase	,   padding=__lowercase	,   truncation=__lowercase	,   return_special_tokens_mask=__lowercase	,   max_length=__lowercase      )
        __UpperCAmelCase							:			Tuple     =						concat_inputs.convert_to_tensors(__lowercase      )

        if answer_ids is not None:
          return self.block_has_answer(__lowercase	,   __lowercase      ) + (concat_inputs_tensors,)
        else:
          return (None, None, None, concat_inputs_tensors)
      @classmethod
      def 			UpperCAmelCase   (   cls      :    Dict	,   __lowercase      :    Optional[Union[str, os.PathLike]]	,   *__lowercase      :    Dict	,   **__lowercase      :    List[str]      )							->   Optional[Any]:
        if os.path.isdir(__lowercase      ):
          __UpperCAmelCase							:			int     =						os.path.join(__lowercase	,   _REALM_BLOCK_RECORDS_FILENAME      )
        else:
          __UpperCAmelCase							:			Dict     =						hf_hub_download(
              repo_id=__lowercase	,   filename=_REALM_BLOCK_RECORDS_FILENAME	,   **__lowercase      )
        __UpperCAmelCase							:			Dict     =						np.load(__lowercase	,   allow_pickle=__lowercase      )

        __UpperCAmelCase							:			Union[str, Any]     =						AutoTokenizer.from_pretrained(__lowercase	,   *__lowercase	,   **__lowercase      )

        return cls(__lowercase	,   __lowercase      )
      def 			UpperCAmelCase   (   self      :    Dict	,   __lowercase      :    List[str]      )							->   Optional[Any]:
        # save block records
        np.save(os.path.join(__lowercase	,   _REALM_BLOCK_RECORDS_FILENAME      )	,   self.block_records      )
        # save tokenizer
        self.tokenizer.save_pretrained(__lowercase      )




      def 			UpperCAmelCase   (   self      :    Any	,   __lowercase      :    Union[str, Any]	,   __lowercase      :    Dict      )							->   Tuple:
        __UpperCAmelCase							:			Union[str, Any]     =						[]
        __UpperCAmelCase							:			Any     =						[]
        __UpperCAmelCase							:			List[str]     =						[]
        __UpperCAmelCase							:			Tuple     =						0

        for input_id in concat_inputs.input_ids:
          __UpperCAmelCase							:			Optional[Any]     =						input_id.tolist()
          # Check answers between two [SEP] tokens
          __UpperCAmelCase							:			Union[str, Any]     =						input_id_list.index(self.tokenizer.sep_token_id      )
          __UpperCAmelCase							:			Dict     =						first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id      )

          start_pos.append([]      )
          end_pos.append([]      )
          for answer in answer_ids:
            for idx in range(first_sep_idx + 1	,   __lowercase      ):
              if answer[0] == input_id_list[idx]:
                if input_id_list[idx : idx + len(__lowercase      )] == answer:
                  start_pos[-1].append(__lowercase      )
                  end_pos[-1].append(idx + len(__lowercase      ) - 1      )

          if len(start_pos[-1]      ) == 0:
            has_answers.append(__lowercase      )
          else:
            has_answers.append(__lowercase      )
            if len(start_pos[-1]      ) > max_answers:
              __UpperCAmelCase							:			Dict     =						len(start_pos[-1]      )

        # Pad -1 to max_answers
        for start_pos_, end_pos_ in zip(__lowercase	,   __lowercase      ):
          if len(start_pos_      ) < max_answers:
            __UpperCAmelCase							:			str     =						[-1] * (max_answers - len(start_pos_      ))
            start_pos_ += padded
            end_pos_ += padded
        return has_answers, start_pos, end_pos










from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


a	: str      =				{"configuration_hubert": ["HUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "HubertConfig"]}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       a	: str      =				[
           "HUBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
           "HubertForCTC",
           "HubertForSequenceClassification",
           "HubertModel",
           "HubertPreTrainedModel",
       ]


try:
       if not is_tf_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       a	: List[str]      =				[
           "TF_HUBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
           "TFHubertForCTC",
           "TFHubertModel",
           "TFHubertPreTrainedModel",
       ]

if TYPE_CHECKING:
       from .configuration_hubert import HUBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, HubertConfig

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_hubert import (
                  HUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
                  HubertForCTC,
                  HubertForSequenceClassification,
                  HubertModel,
                  HubertPreTrainedModel,
              )

       try:
              if not is_tf_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_tf_hubert import (
                  TF_HUBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
                  TFHubertForCTC,
                  TFHubertModel,
                  TFHubertPreTrainedModel,
              )


else:
       import sys

       a	: List[Any]      =				_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)


