




'''simple docstring'''



import collections
import types

import numpy as np

from ..utils import (
    add_end_docstrings,
    is_tensorflow_probability_available,
    is_tf_available,
    is_torch_available,
    requires_backends,
)
from .base import PIPELINE_INIT_ARGS, ArgumentHandler, Dataset, Pipeline, PipelineException


if is_torch_available():
     import torch

     from ..models.auto.modeling_auto import (
         MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
         MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
     )

if is_tf_available() and is_tensorflow_probability_available():
     import tensorflow as tf
     import tensorflow_probability as tfp

     from ..models.auto.modeling_tf_auto import (
         TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING,
         TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING,
     )





class     lowerCAmelCase   ( __lowerCAmelCase):


       def __call__(   self    ,  __SCREAMING_SNAKE_CASE=None    ,  __SCREAMING_SNAKE_CASE=None    ,  **__SCREAMING_SNAKE_CASE  )      ->  List[Any]:



            '''simple docstring'''




            requires_backends(self    ,  '''pandas'''  )
            import pandas as pd

            if table is None:
                 raise ValueError('''Keyword argument `table` cannot be None.'''  )
            elif query is None:
                 if isinstance(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  ) and table.get('''query'''  ) is not None and table.get('''table'''  ) is not None:
                      __snake_case    					=  [table]
                 elif isinstance(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  ) and len(__SCREAMING_SNAKE_CASE  ) > 0:
                      if not all(isinstance(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  ) for d in table  ):
                           raise ValueError(
                               F'''Keyword argument `table` should be a list of dict, but is {(type(__SCREAMING_SNAKE_CASE  ) for d in table)}'''  )

                      if table[0].get('''query'''  ) is not None and table[0].get('''table'''  ) is not None:
                           __snake_case    					=  table
                      else:
                           raise ValueError(
                               '''If keyword argument `table` is a list of dictionaries, each dictionary should have a `table`'''
                               F''' and `query` key, but only dictionary has keys {table[0].keys()} `table` and `query` keys.'''  )
                 elif Dataset is not None and isinstance(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  ) or isinstance(__SCREAMING_SNAKE_CASE    ,  types.GeneratorType  ):
                      return table
                 else:
                      raise ValueError(
                          '''Invalid input. Keyword argument `table` should be either of type `dict` or `list`, but '''
                          F'''is {type(__SCREAMING_SNAKE_CASE  )})'''  )
            else:
                 __snake_case    					=  [{'''table''': table, '''query''': query}]

            for tqa_pipeline_input in tqa_pipeline_inputs:
                 if not isinstance(tqa_pipeline_input['''table''']    ,  pd.DataFrame  ):
                      if tqa_pipeline_input["table"] is None:
                           raise ValueError('''Table cannot be None.'''  )

                      __snake_case    					=  pd.DataFrame(tqa_pipeline_input['''table''']  )

            return tqa_pipeline_inputs





@add_end_docstrings(__lowerCAmelCase)
class     lowerCAmelCase   ( __lowerCAmelCase):
       __lowercase  :      Union[str, Any]        =			'''table,query'''


       def __init__(   self    ,  __SCREAMING_SNAKE_CASE=TableQuestionAnsweringArgumentHandler()    ,  *__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )      ->  Any:



            '''simple docstring'''




            super().__init__(*__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )
            __snake_case    					=  args_parser

            self.check_model_type(
                dict(
                    TF_MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items()
                    + TF_MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()  )
                if self.framework == '''tf'''
                else dict(
                    MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING.items() + MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING.items()  )  )

            __snake_case    					=  bool(getattr(self.model.config    ,  '''aggregation_labels'''    ,  __SCREAMING_SNAKE_CASE  )  ) and bool(
                getattr(self.model.config    ,  '''num_aggregation_labels'''    ,  __SCREAMING_SNAKE_CASE  )  )
            __snake_case    					=  '''tapas''' if hasattr(self.model.config    ,  '''aggregation_labels'''  ) else None


       def       lowerCAmelCase      (   self    ,  **__SCREAMING_SNAKE_CASE  )      ->  Optional[int]:



            '''simple docstring'''




            return self.model(**__SCREAMING_SNAKE_CASE  )


       def       lowerCAmelCase      (   self    ,  **__SCREAMING_SNAKE_CASE  )      ->  List[str]:



            '''simple docstring'''




            if self.framework == "pt":
                 __snake_case    					=  []
                 __snake_case    					=  []
                 __snake_case    					=  None
                 __snake_case    					=  inputs['''input_ids'''].shape[0]

                 __snake_case    					=  inputs['''input_ids'''].to(self.device  )
                 __snake_case    					=  inputs['''attention_mask'''].to(self.device  )
                 __snake_case    					=  inputs['''token_type_ids'''].to(self.device  )
                 __snake_case    					=  None

                 for index in range(__SCREAMING_SNAKE_CASE  ):
                      # If sequences have already been processed, the token type IDs will be created according to the previous
                      # answer.
                      if prev_answers is not None:
                           __snake_case    					=  token_type_ids_example[:, 3]  # shape (seq_len,)
                           __snake_case    					=  np.zeros_like(prev_labels_example.cpu().numpy()  )  # shape (seq_len,)

                           __snake_case    					=  token_type_ids[index]  # shape (seq_len, 7)
                           for i in range(model_labels.shape[0]  ):
                                __snake_case    					=  token_type_ids_example[:, 0].tolist()[i]
                                __snake_case    					=  token_type_ids_example[:, 1].tolist()[i] - 1
                                __snake_case    					=  token_type_ids_example[:, 2].tolist()[i] - 1

                                if row_id >= 0 and col_id >= 0 and segment_id == 1:
                                     __snake_case    					=  int(prev_answers[(col_id, row_id)]  )

                           __snake_case    					=  torch.from_numpy(__SCREAMING_SNAKE_CASE  ).type(torch.long  ).to(self.device  )

                      __snake_case    					=  input_ids[index]
                      __snake_case    					=  attention_mask[index]  # shape (seq_len,)
                      __snake_case    					=  token_type_ids[index]  # shape (seq_len, 7)
                      __snake_case    					=  self.model(
                          input_ids=input_ids_example.unsqueeze(0  )    ,  attention_mask=attention_mask_example.unsqueeze(0  )    ,  token_type_ids=token_type_ids_example.unsqueeze(0  )    ,  )
                      __snake_case    					=  outputs.logits

                      if self.aggregate:
                           all_aggregations.append(outputs.logits_aggregation  )

                      all_logits.append(__SCREAMING_SNAKE_CASE  )

                      __snake_case    					=  torch.distributions.Bernoulli(logits=__SCREAMING_SNAKE_CASE  )
                      __snake_case    					=  dist_per_token.probs * attention_mask_example.type(torch.floataa  ).to(
                          dist_per_token.probs.device  )

                      __snake_case    					=  collections.defaultdict(__SCREAMING_SNAKE_CASE  )
                      for i, p in enumerate(probabilities.squeeze().tolist()  ):
                           __snake_case    					=  token_type_ids_example[:, 0].tolist()[i]
                           __snake_case    					=  token_type_ids_example[:, 1].tolist()[i] - 1
                           __snake_case    					=  token_type_ids_example[:, 2].tolist()[i] - 1
                           if col >= 0 and row >= 0 and segment_id == 1:
                                coords_to_probs[(col, row)].append(__SCREAMING_SNAKE_CASE  )

                      __snake_case    					=  {key: np.array(coords_to_probs[key]  ).mean() > 0.5 for key in coords_to_probs}

                 __snake_case    					=  torch.cat(tuple(__SCREAMING_SNAKE_CASE  )    ,  0  )

                 return (logits_batch,) if not self.aggregate else (logits_batch, torch.cat(tuple(__SCREAMING_SNAKE_CASE  )    ,  0  ))
            else:
                 __snake_case    					=  []
                 __snake_case    					=  []
                 __snake_case    					=  None
                 __snake_case    					=  inputs['''input_ids'''].shape[0]

                 __snake_case    					=  inputs['''input_ids''']
                 __snake_case    					=  inputs['''attention_mask''']
                 __snake_case    					=  inputs['''token_type_ids'''].numpy()
                 __snake_case    					=  None

                 for index in range(__SCREAMING_SNAKE_CASE  ):
                      # If sequences have already been processed, the token type IDs will be created according to the previous
                      # answer.
                      if prev_answers is not None:
                           __snake_case    					=  token_type_ids_example[:, 3]  # shape (seq_len,)
                           __snake_case    					=  np.zeros_like(__SCREAMING_SNAKE_CASE    ,  dtype=np.intaa  )  # shape (seq_len,)

                           __snake_case    					=  token_type_ids[index]  # shape (seq_len, 7)
                           for i in range(model_labels.shape[0]  ):
                                __snake_case    					=  token_type_ids_example[:, 0].tolist()[i]
                                __snake_case    					=  token_type_ids_example[:, 1].tolist()[i] - 1
                                __snake_case    					=  token_type_ids_example[:, 2].tolist()[i] - 1

                                if row_id >= 0 and col_id >= 0 and segment_id == 1:
                                     __snake_case    					=  int(prev_answers[(col_id, row_id)]  )

                           __snake_case    					=  model_labels

                      __snake_case    					=  input_ids[index]
                      __snake_case    					=  attention_mask[index]  # shape (seq_len,)
                      __snake_case    					=  token_type_ids[index]  # shape (seq_len, 7)
                      __snake_case    					=  self.model(
                          input_ids=np.expand_dims(__SCREAMING_SNAKE_CASE    ,  axis=0  )    ,  attention_mask=np.expand_dims(__SCREAMING_SNAKE_CASE    ,  axis=0  )    ,  token_type_ids=np.expand_dims(__SCREAMING_SNAKE_CASE    ,  axis=0  )    ,  )
                      __snake_case    					=  outputs.logits

                      if self.aggregate:
                           all_aggregations.append(outputs.logits_aggregation  )

                      all_logits.append(__SCREAMING_SNAKE_CASE  )

                      __snake_case    					=  tfp.distributions.Bernoulli(logits=__SCREAMING_SNAKE_CASE  )
                      __snake_case    					=  dist_per_token.probs_parameter() * tf.cast(__SCREAMING_SNAKE_CASE    ,  tf.floataa  )

                      __snake_case    					=  collections.defaultdict(__SCREAMING_SNAKE_CASE  )
                      __snake_case    					=  token_type_ids_example
                      for i, p in enumerate(tf.squeeze(__SCREAMING_SNAKE_CASE  ).numpy().tolist()  ):
                           __snake_case    					=  token_type_ids_example[:, 0].tolist()[i]
                           __snake_case    					=  token_type_ids_example[:, 1].tolist()[i] - 1
                           __snake_case    					=  token_type_ids_example[:, 2].tolist()[i] - 1
                           if col >= 0 and row >= 0 and segment_id == 1:
                                coords_to_probs[(col, row)].append(__SCREAMING_SNAKE_CASE  )

                      __snake_case    					=  {key: np.array(coords_to_probs[key]  ).mean() > 0.5 for key in coords_to_probs}

                 __snake_case    					=  tf.concat(tuple(__SCREAMING_SNAKE_CASE  )    ,  0  )

                 return (logits_batch,) if not self.aggregate else (logits_batch, tf.concat(tuple(__SCREAMING_SNAKE_CASE  )    ,  0  ))


       def __call__(   self    ,  *__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )      ->  Optional[int]:



            '''simple docstring'''




            __snake_case    					=  self._args_parser(*__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )

            __snake_case    					=  super().__call__(__SCREAMING_SNAKE_CASE    ,  **__SCREAMING_SNAKE_CASE  )
            if len(__SCREAMING_SNAKE_CASE  ) == 1:
                 return results[0]
            return results


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE=None    ,  __SCREAMING_SNAKE_CASE=None    ,  __SCREAMING_SNAKE_CASE=None    ,  **__SCREAMING_SNAKE_CASE  )      ->  Union[str, Any]:



            '''simple docstring'''




            __snake_case    					=  {}
            if padding is not None:
                 __snake_case    					=  padding
            if truncation is not None:
                 __snake_case    					=  truncation

            __snake_case    					=  {}
            if sequential is not None:
                 __snake_case    					=  sequential
            return preprocess_params, forward_params, {}


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE=None    ,  __SCREAMING_SNAKE_CASE=True    ,  __SCREAMING_SNAKE_CASE=None  )      ->  List[str]:



            '''simple docstring'''




            if truncation is None:
                 if self.type == "tapas":
                      __snake_case    					=  '''drop_rows_to_fit'''
                 else:
                      __snake_case    					=  '''do_not_truncate'''

            __snake_case					,    __snake_case    					=  pipeline_input['''table'''], pipeline_input['''query''']
            if table.empty:
                 raise ValueError('''table is empty'''  )
            if query is None or query == "":
                 raise ValueError('''query is empty'''  )
            __snake_case    					=  self.tokenizer(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE    ,  return_tensors=self.framework    ,  truncation=__SCREAMING_SNAKE_CASE    ,  padding=__SCREAMING_SNAKE_CASE  )
            __snake_case    					=  table
            return inputs


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE=False  )      ->  Optional[Any]:



            '''simple docstring'''




            __snake_case    					=  model_inputs.pop('''table'''  )

            if self.type == "tapas":
                 if sequential:
                      __snake_case    					=  self.sequential_inference(**__SCREAMING_SNAKE_CASE  )
                 else:
                      __snake_case    					=  self.batch_inference(**__SCREAMING_SNAKE_CASE  )
            else:
                 __snake_case    					=  self.model.generate(**__SCREAMING_SNAKE_CASE  )
            __snake_case    					=  {'''model_inputs''': model_inputs, '''table''': table, '''outputs''': outputs}
            return model_outputs




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Any:



            '''simple docstring'''




            __snake_case    					=  model_outputs['''model_inputs''']
            __snake_case    					=  model_outputs['''table''']
            __snake_case    					=  model_outputs['''outputs''']
            if self.type == "tapas":
                 if self.aggregate:
                      __snake_case					,    __snake_case    					=  outputs[:2]
                      __snake_case    					=  self.tokenizer.convert_logits_to_predictions(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )
                      __snake_case					,    __snake_case    					=  predictions
                      __snake_case    					=  {i: self.model.config.aggregation_labels[pred] for i, pred in enumerate(__SCREAMING_SNAKE_CASE  )}

                      __snake_case    					=  self.model.config.no_aggregation_label_index
                      __snake_case    					=  {
                          i: aggregators[i] + ''' > ''' for i, pred in enumerate(__SCREAMING_SNAKE_CASE  ) if pred != no_agg_label_index
                      }
                 else:
                      __snake_case    					=  outputs[0]
                      __snake_case    					=  self.tokenizer.convert_logits_to_predictions(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )
                      __snake_case    					=  predictions[0]
                      __snake_case    					=  {}
                      __snake_case    					=  {}
                 __snake_case    					=  []
                 for index, coordinates in enumerate(__SCREAMING_SNAKE_CASE  ):
                      __snake_case    					=  [table.iat[coordinate] for coordinate in coordinates]
                      __snake_case    					=  aggregators.get(__SCREAMING_SNAKE_CASE    ,  ''''''  )
                      __snake_case    					=  aggregators_prefix.get(__SCREAMING_SNAKE_CASE    ,  ''''''  )
                      __snake_case    					=  {
                          '''answer''': aggregator_prefix + ''', '''.join(__SCREAMING_SNAKE_CASE  ),
                          '''coordinates''': coordinates,
                          '''cells''': [table.iat[coordinate] for coordinate in coordinates],
                      }
                      if aggregator:
                           __snake_case    					=  aggregator

                      answers.append(__SCREAMING_SNAKE_CASE  )
                 if len(__SCREAMING_SNAKE_CASE  ) == 0:
                      raise PipelineException('''Empty answer'''  )
            else:
                 __snake_case    					=  [{'''answer''': answer} for answer in self.tokenizer.batch_decode(__SCREAMING_SNAKE_CASE    ,  skip_special_tokens=__SCREAMING_SNAKE_CASE  )]

            return answers if len(__SCREAMING_SNAKE_CASE  ) > 1 else answers[0]









'''simple docstring'''



import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


UpperCAmelCase_       :  str      =		datasets.utils.logging.get_logger(__name__)





@dataclass
class     lowerCAmelCase   ( datasets.BuilderConfig):
       __lowercase  :      Optional[datasets.Features]        =			None
       __lowercase  :      str        =			"utf-8"
       __lowercase  :      Optional[str]        =			None
       __lowercase  :      int        =			10 << 20  # 10MB
       __lowercase  :      bool        =			False
       __lowercase  :      str        =			"line"





class     lowerCAmelCase   ( datasets.ArrowBasedBuilder):
       __lowercase  :      Dict        =			TextConfig


       def       lowerCAmelCase      (   self  )      ->  Dict:



            '''simple docstring'''




            return datasets.DatasetInfo(features=self.config.features  )


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Optional[Any]:



            '''simple docstring'''




            if not self.config.data_files:
                 raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''  )
            __snake_case    					=  dl_manager.download_and_extract(self.config.data_files  )
            if isinstance(__SCREAMING_SNAKE_CASE    ,  (str, list, tuple)  ):
                 __snake_case    					=  data_files
                 if isinstance(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  ):
                      __snake_case    					=  [files]
                 __snake_case    					=  [dl_manager.iter_files(__SCREAMING_SNAKE_CASE  ) for file in files]
                 return [datasets.SplitGenerator(name=datasets.Split.TRAIN    ,  gen_kwargs={'''files''': files}  )]
            __snake_case    					=  []
            for split_name, files in data_files.items():
                 if isinstance(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  ):
                      __snake_case    					=  [files]
                 __snake_case    					=  [dl_manager.iter_files(__SCREAMING_SNAKE_CASE  ) for file in files]
                 splits.append(datasets.SplitGenerator(name=__SCREAMING_SNAKE_CASE    ,  gen_kwargs={'''files''': files}  )  )
            return splits


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  pa.Table:



            '''simple docstring'''




            if self.config.features is not None:
                 __snake_case    					=  self.config.features.arrow_schema
                 if all(not require_storage_cast(__SCREAMING_SNAKE_CASE  ) for feature in self.config.features.values()  ):
                      # cheaper cast
                      __snake_case    					=  pa_table.cast(__SCREAMING_SNAKE_CASE  )
                 else:
                      # more expensive cast; allows str <-> int/float or str to Audio for example
                      __snake_case    					=  table_cast(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )
                 return pa_table
            else:
                 return pa_table.cast(pa.schema({'''text''': pa.string()}  )  )




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Optional[Any]:



            '''simple docstring'''




            __snake_case    					=  list(self.config.features  ) if self.config.features is not None else ['''text''']
            for file_idx, file in enumerate(itertools.chain.from_iterable(__SCREAMING_SNAKE_CASE  )  ):
                 # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
                 with open(__SCREAMING_SNAKE_CASE    ,  encoding=self.config.encoding    ,  errors=self.config.errors  ) as f:
                      if self.config.sample_by == "line":
                           __snake_case    					=  0
                           while True:
                                __snake_case    					=  f.read(self.config.chunksize  )
                                if not batch:
                                     break
                                batch += f.readline()  # finish current line
                                # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                                __snake_case    					=  StringIO(__SCREAMING_SNAKE_CASE  ).readlines()
                                if not self.config.keep_linebreaks:
                                     __snake_case    					=  [line.rstrip('''\n'''  ) for line in batch]
                                __snake_case    					=  pa.Table.from_arrays([pa.array(__SCREAMING_SNAKE_CASE  )]    ,  names=__SCREAMING_SNAKE_CASE  )
                                # Uncomment for debugging (will print the Arrow table size and elements)
                                # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE  )
                                batch_idx += 1
                      elif self.config.sample_by == "paragraph":
                           __snake_case    					=  0
                           __snake_case    					=  ''''''
                           while True:
                                __snake_case    					=  f.read(self.config.chunksize  )
                                if not new_batch:
                                     break
                                batch += new_batch
                                batch += f.readline()  # finish current line
                                __snake_case    					=  batch.split('''\n\n'''  )
                                __snake_case    					=  pa.Table.from_arrays(
                                    [pa.array([example for example in batch[:-1] if example]  )]    ,  names=__SCREAMING_SNAKE_CASE  )
                                # Uncomment for debugging (will print the Arrow table size and elements)
                                # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE  )
                                batch_idx += 1
                                __snake_case    					=  batch[-1]
                           if batch:
                                __snake_case    					=  pa.Table.from_arrays([pa.array([batch]  )]    ,  names=__SCREAMING_SNAKE_CASE  )
                                yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE  )
                      elif self.config.sample_by == "document":
                           __snake_case    					=  f.read()
                           __snake_case    					=  pa.Table.from_arrays([pa.array([text]  )]    ,  names=__SCREAMING_SNAKE_CASE  )
                           yield file_idx, self._cast_table(__SCREAMING_SNAKE_CASE  )






