

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_splinter import SplinterTokenizer


lowerCAmelCase							:						List[Any]     			=		logging.get_logger(__name__)

lowerCAmelCase							:						int     			=		{"""vocab_file""": """vocab.txt"""}

lowerCAmelCase							:						List[str]     			=		{
    """vocab_file""": {
        """tau/splinter-base""": """https://huggingface.co/tau/splinter-base/resolve/main/vocab.txt""",
        """tau/splinter-base-qass""": """https://huggingface.co/tau/splinter-base-qass/resolve/main/vocab.txt""",
        """tau/splinter-large""": """https://huggingface.co/tau/splinter-large/resolve/main/vocab.txt""",
        """tau/splinter-large-qass""": """https://huggingface.co/tau/splinter-large-qass/resolve/main/vocab.txt""",
    }
}

lowerCAmelCase							:						Tuple     			=		{
    """tau/splinter-base""": 512,
    """tau/splinter-base-qass""": 512,
    """tau/splinter-large""": 512,
    """tau/splinter-large-qass""": 512,
}

lowerCAmelCase							:						int     			=		{
    """tau/splinter-base""": {"""do_lower_case""": False},
    """tau/splinter-base-qass""": {"""do_lower_case""": False},
    """tau/splinter-large""": {"""do_lower_case""": False},
    """tau/splinter-large-qass""": {"""do_lower_case""": False},
}



class 						__lowercase       (				UpperCAmelCase_			):




   """simple docstring"""







   _UpperCAmelCase     :							Union[str, Any]    							=  VOCAB_FILES_NAMES
   _UpperCAmelCase     :							Tuple    							=  PRETRAINED_VOCAB_FILES_MAP
   _UpperCAmelCase     :							str    							=  PRETRAINED_INIT_CONFIGURATION
   _UpperCAmelCase     :							Any    							=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _UpperCAmelCase     :							Optional[Any]    							=  SplinterTokenizer



   def __init__(				self						:  Union[str, Any]     ,		lowerCAmelCase__						:  Union[str, Any]=None     ,		lowerCAmelCase__						:  Any=None     ,		lowerCAmelCase__						:  List[Any]=True     ,		lowerCAmelCase__						:  Tuple="[UNK]"     ,		lowerCAmelCase__						:  int="[SEP]"     ,		lowerCAmelCase__						:  Optional[Any]="[PAD]"     ,		lowerCAmelCase__						:  Any="[CLS]"     ,		lowerCAmelCase__						:  str="[MASK]"     ,		lowerCAmelCase__						:  str="[QUESTION]"     ,		lowerCAmelCase__						:  Optional[Any]=True     ,		lowerCAmelCase__						:  List[str]=None     ,		**lowerCAmelCase__						:  List[str]     ,		):
         super().__init__(
             lowerCAmelCase__     ,		tokenizer_file=lowerCAmelCase__     ,		do_lower_case=lowerCAmelCase__     ,		unk_token=lowerCAmelCase__     ,		sep_token=lowerCAmelCase__     ,		pad_token=lowerCAmelCase__     ,		cls_token=lowerCAmelCase__     ,		mask_token=lowerCAmelCase__     ,		tokenize_chinese_chars=lowerCAmelCase__     ,		strip_accents=lowerCAmelCase__     ,		additional_special_tokens=(question_token,)     ,		**lowerCAmelCase__     ,		)

         SCREAMING_SNAKE_CASE_:				Tuple							= json.loads(self.backend_tokenizer.normalizer.__getstate__())
         if (
             pre_tok_state.get("lowercase"     ,		lowerCAmelCase__) != do_lower_case
             or pre_tok_state.get("strip_accents"     ,		lowerCAmelCase__) != strip_accents
         ):
               SCREAMING_SNAKE_CASE_:				int							= getattr(lowerCAmelCase__     ,		pre_tok_state.pop("type"))
               SCREAMING_SNAKE_CASE_:				List[Any]							= do_lower_case
               SCREAMING_SNAKE_CASE_:				Any							= strip_accents
               SCREAMING_SNAKE_CASE_:				Dict							= pre_tok_class(**lowerCAmelCase__)

         SCREAMING_SNAKE_CASE_:				Any							= do_lower_case



   @property
   def  _SCREAMING_SNAKE_CASE    (				self						:  int):
         return self.convert_tokens_to_ids(self.question_token)



   def  _SCREAMING_SNAKE_CASE    (				self						:  Any     ,		lowerCAmelCase__						:  List[int]     ,		lowerCAmelCase__						:  Optional[List[int]] = None):
         if token_ids_a is None:
               return [self.cls_token_id] + token_ids_a + [self.sep_token_id]

         SCREAMING_SNAKE_CASE_:				int							= [self.cls_token_id]
         SCREAMING_SNAKE_CASE_:				List[Any]							= [self.sep_token_id]
         SCREAMING_SNAKE_CASE_:				str							= [self.question_token_id] + [self.convert_tokens_to_ids(".")]
         if self.padding_side == "right":
               # Input is question-then-context
               return cls + token_ids_a + question_suffix + sep + token_ids_a + sep
         else:
               # Input is context-then-question
               return cls + token_ids_a + sep + token_ids_a + question_suffix + sep



   def  _SCREAMING_SNAKE_CASE    (				self						:  int     ,		lowerCAmelCase__						:  List[int]     ,		lowerCAmelCase__						:  Optional[List[int]] = None):
         SCREAMING_SNAKE_CASE_:				Tuple							= [self.sep_token_id]
         SCREAMING_SNAKE_CASE_:				List[str]							= [self.cls_token_id]
         SCREAMING_SNAKE_CASE_:				Optional[Any]							= [self.question_token_id] + [self.convert_tokens_to_ids(".")]
         if token_ids_a is None:
               return len(cls + token_ids_a + sep) * [0]

         if self.padding_side == "right":
               # Input is question-then-context
               return len(cls + token_ids_a + question_suffix + sep) * [0] + len(token_ids_a + sep) * [1]
         else:
               # Input is context-then-question
               return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + question_suffix + sep) * [1]




   def  _SCREAMING_SNAKE_CASE    (				self						:  Optional[int]     ,		lowerCAmelCase__						:  str     ,		lowerCAmelCase__						:  Optional[str] = None):
         SCREAMING_SNAKE_CASE_:				Any							= self._tokenizer.model.save(lowerCAmelCase__     ,		name=lowerCAmelCase__)
         return tuple(lowerCAmelCase__)









import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


lowerCAmelCase							:						Tuple     			=		datasets.utils.logging.get_logger(__name__)



@dataclass
class 						__lowercase       (				datasets.BuilderConfig			):




			"""simple docstring"""







			_UpperCAmelCase     :							Optional[datasets.Features]    							=  None
			_UpperCAmelCase     :							str    							=  "utf-8"
			_UpperCAmelCase     :							Optional[str]    							=  None
			_UpperCAmelCase     :							int    							=  10 << 20  # 10MB
			_UpperCAmelCase     :							bool    							=  False
			_UpperCAmelCase     :							str    							=  "line"







class 						__lowercase       (				datasets.ArrowBasedBuilder			):




			"""simple docstring"""







			_UpperCAmelCase     :							List[Any]    							=  TextConfig



			def  _SCREAMING_SNAKE_CASE    (				self						:  Optional[Any]):
									return datasets.DatasetInfo(features=self.config.features)



			def  _SCREAMING_SNAKE_CASE    (				self						:  List[Any]     ,		lowerCAmelCase__						:  Tuple):
									if not self.config.data_files:
															raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}")
									SCREAMING_SNAKE_CASE_:				Dict							= dl_manager.download_and_extract(self.config.data_files)
									if isinstance(lowerCAmelCase__     ,		(str, list, tuple)):
															SCREAMING_SNAKE_CASE_:				Dict							= data_files
															if isinstance(lowerCAmelCase__     ,		lowerCAmelCase__):
																					SCREAMING_SNAKE_CASE_:				Dict							= [files]
															SCREAMING_SNAKE_CASE_:				List[str]							= [dl_manager.iter_files(lowerCAmelCase__) for file in files]
															return [datasets.SplitGenerator(name=datasets.Split.TRAIN     ,		gen_kwargs={"files": files})]
									SCREAMING_SNAKE_CASE_:				Any							= []
									for split_name, files in data_files.items():
															if isinstance(lowerCAmelCase__     ,		lowerCAmelCase__):
																					SCREAMING_SNAKE_CASE_:				List[Any]							= [files]
															SCREAMING_SNAKE_CASE_:				Union[str, Any]							= [dl_manager.iter_files(lowerCAmelCase__) for file in files]
															splits.append(datasets.SplitGenerator(name=lowerCAmelCase__     ,		gen_kwargs={"files": files}))
									return splits



			def  _SCREAMING_SNAKE_CASE    (				self						:  Optional[int]     ,		lowerCAmelCase__						:  pa.Table):
									if self.config.features is not None:
															SCREAMING_SNAKE_CASE_:				Optional[int]							= self.config.features.arrow_schema
															if all(not require_storage_cast(lowerCAmelCase__) for feature in self.config.features.values()):
																					# cheaper cast
																					SCREAMING_SNAKE_CASE_:				List[str]							= pa_table.cast(lowerCAmelCase__)
															else:
																					# more expensive cast; allows str <-> int/float or str to Audio for example
																					SCREAMING_SNAKE_CASE_:				List[Any]							= table_cast(lowerCAmelCase__     ,		lowerCAmelCase__)
															return pa_table
									else:
															return pa_table.cast(pa.schema({"text": pa.string()}))




			def  _SCREAMING_SNAKE_CASE    (				self						:  int     ,		lowerCAmelCase__						:  Dict):
									SCREAMING_SNAKE_CASE_:				str							= list(self.config.features) if self.config.features is not None else ["text"]
									for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCAmelCase__)):
															# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
															with open(lowerCAmelCase__     ,		encoding=self.config.encoding     ,		errors=self.config.errors) as f:
																					if self.config.sample_by == "line":
																											SCREAMING_SNAKE_CASE_:				Any							= 0
																											while True:
																																	SCREAMING_SNAKE_CASE_:				str							= f.read(self.config.chunksize)
																																	if not batch:
																																							break
																																	batch += f.readline()  # finish current line
																																	# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																																	SCREAMING_SNAKE_CASE_:				Optional[Any]							= StringIO(lowerCAmelCase__).readlines()
																																	if not self.config.keep_linebreaks:
																																							SCREAMING_SNAKE_CASE_:				Optional[Any]							= [line.rstrip("\n") for line in batch]
																																	SCREAMING_SNAKE_CASE_:				Any							= pa.Table.from_arrays([pa.array(lowerCAmelCase__)]     ,		names=lowerCAmelCase__)
																																	# Uncomment for debugging (will print the Arrow table size and elements)
																																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																	yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase__)
																																	batch_idx += 1
																					elif self.config.sample_by == "paragraph":
																											SCREAMING_SNAKE_CASE_:				List[str]							= 0
																											SCREAMING_SNAKE_CASE_:				Tuple							= ""
																											while True:
																																	SCREAMING_SNAKE_CASE_:				Dict							= f.read(self.config.chunksize)
																																	if not new_batch:
																																							break
																																	batch += new_batch
																																	batch += f.readline()  # finish current line
																																	SCREAMING_SNAKE_CASE_:				List[str]							= batch.split("\n\n")
																																	SCREAMING_SNAKE_CASE_:				Optional[int]							= pa.Table.from_arrays(
																																	    [pa.array([example for example in batch[:-1] if example])]     ,		names=lowerCAmelCase__)
																																	# Uncomment for debugging (will print the Arrow table size and elements)
																																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																	yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase__)
																																	batch_idx += 1
																																	SCREAMING_SNAKE_CASE_:				Optional[Any]							= batch[-1]
																											if batch:
																																	SCREAMING_SNAKE_CASE_:				int							= pa.Table.from_arrays([pa.array([batch])]     ,		names=lowerCAmelCase__)
																																	yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase__)
																					elif self.config.sample_by == "document":
																											SCREAMING_SNAKE_CASE_:				List[Any]							= f.read()
																											SCREAMING_SNAKE_CASE_:				Optional[int]							= pa.Table.from_arrays([pa.array([text])]     ,		names=lowerCAmelCase__)
																											yield file_idx, self._cast_table(lowerCAmelCase__)






