


'''simple docstring'''


import os
from functools import partial
from glob import glob

import faiss
from datasets import Features, Sequence, Value, concatenate_datasets, load_dataset, load_from_disk

from transformers import DPRContextEncoder, DPRContextEncoderTokenizerFast


def 		_lowercase   (SCREAMING_SNAKE_CASE					,				SCREAMING_SNAKE_CASE=100					,				SCREAMING_SNAKE_CASE=" "       ):




					'''simple docstring'''





					__A  :					List[Any]						     =       text.split(SCREAMING_SNAKE_CASE       )
					return [character.join(text[i : i + n]       ).strip() for i in range(0					,				len(SCREAMING_SNAKE_CASE       )					,				SCREAMING_SNAKE_CASE       )]


def 		_lowercase   (SCREAMING_SNAKE_CASE       ):




					'''simple docstring'''





					__A  :					List[Any]						     =       [], []
					for title, text in zip(documents["title"]					,				documents["text"]       ):
										if text is not None:
															for passage in split_text(SCREAMING_SNAKE_CASE       ):
																				titles.append(title if title is not None else ""       )
																				texts.append(SCREAMING_SNAKE_CASE       )
					return {"title": titles, "text": texts}


def 		_lowercase   (SCREAMING_SNAKE_CASE					,				SCREAMING_SNAKE_CASE					,				SCREAMING_SNAKE_CASE					,				SCREAMING_SNAKE_CASE					,				SCREAMING_SNAKE_CASE					,				SCREAMING_SNAKE_CASE       ):




					'''simple docstring'''





					__A  :					str						     =       load_dataset(
					    "csv"					,				data_files=[csv_path]					,				split="train"					,				delimiter="\t"					,				column_names=["title", "text"]       )
					__A  :					Optional[Any]						     =       kb_dataset.map(
					    SCREAMING_SNAKE_CASE					,				batched=SCREAMING_SNAKE_CASE					,				num_proc=1       )  # if you want you can load already splitted csv.
					__A  :					int						     =       [kb_dataset.shard(SCREAMING_SNAKE_CASE					,				SCREAMING_SNAKE_CASE					,				contiguous=SCREAMING_SNAKE_CASE       ) for i in range(SCREAMING_SNAKE_CASE       )]
					__A  :					List[Any]						     =       kb_list[process_num]

					__A  :					str						     =       "data_" + str(SCREAMING_SNAKE_CASE       )
					__A  :					List[str]						     =       os.path.join(SCREAMING_SNAKE_CASE					,				SCREAMING_SNAKE_CASE       )

					__A  :					Dict						     =       DPRContextEncoderTokenizerFast.from_pretrained("facebook/dpr-ctx_encoder-multiset-base"       )
					__A  :					Any						     =       ctx_encoder.to(device=SCREAMING_SNAKE_CASE       )

					def embed(
					    SCREAMING_SNAKE_CASE					,				SCREAMING_SNAKE_CASE					,				SCREAMING_SNAKE_CASE					,				SCREAMING_SNAKE_CASE       ) -> dict:
										__A  :					Dict						     =       ctx_tokenizer(
										    documents["title"]					,				documents["text"]					,				truncation=SCREAMING_SNAKE_CASE					,				padding="longest"					,				return_tensors="pt"       )["input_ids"]
										__A  :					Any						     =       ctx_encoder(input_ids.to(device=SCREAMING_SNAKE_CASE       )					,				return_dict=SCREAMING_SNAKE_CASE       ).pooler_output
										return {"embeddings": embeddings.detach().cpu().numpy()}

					__A  :					str						     =       Features(
					    {"text": Value("string"       ), "title": Value("string"       ), "embeddings": Sequence(Value("float32"       )       )}       )  # optional, save as float32 instead of float64 to save space

					__A  :					Tuple						     =       data_shrad.map(
					    partial(SCREAMING_SNAKE_CASE					,				ctx_encoder=SCREAMING_SNAKE_CASE					,				ctx_tokenizer=SCREAMING_SNAKE_CASE					,				device=SCREAMING_SNAKE_CASE       )					,				batched=SCREAMING_SNAKE_CASE					,				batch_size=16					,				features=SCREAMING_SNAKE_CASE					,				)
					dataset.save_to_disk(SCREAMING_SNAKE_CASE       )






def 		_lowercase   (SCREAMING_SNAKE_CASE					,				SCREAMING_SNAKE_CASE       ):




					'''simple docstring'''





					__A  :					str						     =       []

					for shard_address in glob(str(SCREAMING_SNAKE_CASE       ) + "/*/"       ):
										data_shard_list.append(load_from_disk(SCREAMING_SNAKE_CASE       )       )

					__A  :					int						     =       concatenate_datasets(SCREAMING_SNAKE_CASE       )
					faiss.omp_set_num_threads(96       )

					__A  :					List[str]						     =       faiss.IndexHNSWFlat(768					,				128					,				faiss.METRIC_INNER_PRODUCT       )
					concat.add_faiss_index("embeddings"					,				custom_index=SCREAMING_SNAKE_CASE       )
					concat.get_index("embeddings"       ).save(
					    SCREAMING_SNAKE_CASE       )  # since we load the index in to memory,we can directly update the index in the disk





'''simple docstring'''


import json
import os
import re
import unicodedata
from typing import Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer, _is_control, _is_punctuation, _is_whitespace
from ...utils import logging


_UpperCamelCase          =     logging.get_logger(__name__)

_UpperCamelCase          =     {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

_UpperCamelCase          =     {
    """vocab_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/vocab.json"""},
    """merges_file""": {"""openai-gpt""": """https://huggingface.co/openai-gpt/resolve/main/merges.txt"""},
}

_UpperCamelCase          =     {
    """openai-gpt""": 512,
}


def 		_lowercase   (SCREAMING_SNAKE_CASE       ):




     '''simple docstring'''





     __A  :					List[str]						     =       text.strip()
     if not text:
          return []
     __A  :					Any						     =       text.split()
     return tokens







class        __magic_name__  (					lowerCAmelCase   ):

     """simple docstring"""






     def __init__(      self ,    lowerCamelCase=True ,    lowerCamelCase=None ,    lowerCamelCase=True ,    lowerCamelCase=None ,    lowerCamelCase=True ,    ):


          '''simple docstring'''





          if never_split is None:
               __A  :					List[Any]						     =       []
          __A  :					str						     =       do_lower_case
          __A  :					str						     =       set(lowerCamelCase		)
          __A  :					Tuple						     =       tokenize_chinese_chars
          __A  :					Any						     =       strip_accents
          __A  :					Dict						     =       do_split_on_punc


     def lowerCAmelCase__     (      self ,    lowerCamelCase ,    lowerCamelCase=None		):


          '''simple docstring'''





          __A  :					str						     =       self.never_split.union(set(lowerCamelCase		)		) if never_split else self.never_split
          __A  :					Optional[Any]						     =       self._clean_text(lowerCamelCase		)

          # This was added on November 1st, 2018 for the multilingual and Chinese
          # models. This is also applied to the English models now, but it doesn't
          # matter since the English models were not trained on any Chinese data
          # and generally don't have any Chinese data in them (there are Chinese
          # characters in the vocabulary because Wikipedia does have some Chinese
          # words in the English Wikipedia.).
          if self.tokenize_chinese_chars:
               __A  :					Tuple						     =       self._tokenize_chinese_chars(lowerCamelCase		)
          # prevents treating the same character with different unicode codepoints as different characters
          __A  :					str						     =       unicodedata.normalize("NFC" ,    lowerCamelCase		)
          __A  :					Tuple						     =       whitespace_tokenize(lowerCamelCase		)
          __A  :					Union[str, Any]						     =       []
          for token in orig_tokens:
               if token not in never_split:
                    if self.do_lower_case:
                         __A  :					str						     =       token.lower()
                         if self.strip_accents is not False:
                              __A  :					str						     =       self._run_strip_accents(lowerCamelCase		)
                    elif self.strip_accents:
                         __A  :					Tuple						     =       self._run_strip_accents(lowerCamelCase		)
               split_tokens.extend(self._run_split_on_punc(lowerCamelCase ,    lowerCamelCase		)		)

          __A  :					Tuple						     =       whitespace_tokenize(" ".join(lowerCamelCase		)		)
          return output_tokens


     def lowerCAmelCase__     (      self ,    lowerCamelCase		):


          '''simple docstring'''





          __A  :					Tuple						     =       unicodedata.normalize("NFD" ,    lowerCamelCase		)
          __A  :					Tuple						     =       []
          for char in text:
               __A  :					Dict						     =       unicodedata.category(lowerCamelCase		)
               if cat == "Mn":
                    continue
               output.append(lowerCamelCase		)
          return "".join(lowerCamelCase		)


     def lowerCAmelCase__     (      self ,    lowerCamelCase ,    lowerCamelCase=None		):


          '''simple docstring'''





          if not self.do_split_on_punc or (never_split is not None and text in never_split):
               return [text]
          __A  :					int						     =       list(lowerCamelCase		)
          __A  :					int						     =       0
          __A  :					int						     =       True
          __A  :					Tuple						     =       []
          while i < len(lowerCamelCase		):
               __A  :					str						     =       chars[i]
               if _is_punctuation(lowerCamelCase		):
                    output.append([char]		)
                    __A  :					Optional[int]						     =       True
               else:
                    if start_new_word:
                         output.append([]		)
                    __A  :					List[str]						     =       False
                    output[-1].append(lowerCamelCase		)
               i += 1

          return ["".join(lowerCamelCase		) for x in output]


     def lowerCAmelCase__     (      self ,    lowerCamelCase		):


          '''simple docstring'''





          __A  :					int						     =       []
          for char in text:
               __A  :					Union[str, Any]						     =       ord(lowerCamelCase		)
               if self._is_chinese_char(lowerCamelCase		):
                    output.append(" "		)
                    output.append(lowerCamelCase		)
                    output.append(" "		)
               else:
                    output.append(lowerCamelCase		)
          return "".join(lowerCamelCase		)


     def lowerCAmelCase__     (      self ,    lowerCamelCase		):


          '''simple docstring'''





          if (
              (cp >= 0x4E00 and cp <= 0x9FFF)
              or (cp >= 0x3400 and cp <= 0x4DBF)  #
              or (cp >= 0x2_0000 and cp <= 0x2_A6DF)  #
              or (cp >= 0x2_A700 and cp <= 0x2_B73F)  #
              or (cp >= 0x2_B740 and cp <= 0x2_B81F)  #
              or (cp >= 0x2_B820 and cp <= 0x2_CEAF)  #
              or (cp >= 0xF900 and cp <= 0xFAFF)
              or (cp >= 0x2_F800 and cp <= 0x2_FA1F)  #
          ):  #
               return True

          return False


     def lowerCAmelCase__     (      self ,    lowerCamelCase		):


          '''simple docstring'''





          __A  :					List[str]						     =       []
          for char in text:
               __A  :					Dict						     =       ord(lowerCamelCase		)
               if cp == 0 or cp == 0xFFFD or _is_control(lowerCamelCase		):
                    continue
               if _is_whitespace(lowerCamelCase		):
                    output.append(" "		)
               else:
                    output.append(lowerCamelCase		)
          return "".join(lowerCamelCase		)


def 		_lowercase   (SCREAMING_SNAKE_CASE       ):




     '''simple docstring'''





     __A  :					Dict						     =       set()
     __A  :					Tuple						     =       word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)       )
          __A  :					str						     =       char
     return pairs


def 		_lowercase   (SCREAMING_SNAKE_CASE       ):




     '''simple docstring'''





     __A  :					Dict						     =       text.replace("—"					,				"-"       )
     __A  :					Optional[int]						     =       text.replace("–"					,				"-"       )
     __A  :					Dict						     =       text.replace("―"					,				"-"       )
     __A  :					Optional[int]						     =       text.replace("…"					,				"..."       )
     __A  :					int						     =       text.replace("´"					,				"'"       )
     __A  :					List[Any]						     =       re.sub(r"(-+|~+|!+|\"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)"					,				r" \1 "					,				SCREAMING_SNAKE_CASE       )
     __A  :					Union[str, Any]						     =       re.sub(r"\s*\n\s*"					,				" \n "					,				SCREAMING_SNAKE_CASE       )
     __A  :					List[Any]						     =       re.sub(r"[^\S\n]+"					,				" "					,				SCREAMING_SNAKE_CASE       )
     return text.strip()







class        __magic_name__  (					lowerCAmelCase   ):

     """simple docstring"""






     lowerCamelCase__						=							VOCAB_FILES_NAMES
     lowerCamelCase__						=							PRETRAINED_VOCAB_FILES_MAP
     lowerCamelCase__						=							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCamelCase__						=							['input_ids', 'attention_mask']


     def __init__(      self ,    lowerCamelCase ,    lowerCamelCase ,    lowerCamelCase="<unk>" ,    **lowerCamelCase		):


          '''simple docstring'''





          super().__init__(unk_token=lowerCamelCase ,    **lowerCamelCase		)

          try:
               import ftfy
               from spacy.lang.en import English

               __A  :					Union[str, Any]						     =       English()
               __A  :					Tuple						     =       _nlp.tokenizer
               __A  :					Any						     =       ftfy.fix_text
          except ImportError:
               logger.warning("ftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy."		)
               __A  :					List[Any]						     =       BasicTokenizer(do_lower_case=lowerCamelCase		)
               __A  :					Any						     =       None

          with open(lowerCamelCase ,    encoding="utf-8"		) as vocab_handle:
               __A  :					int						     =       json.load(lowerCamelCase		)
          __A  :					Union[str, Any]						     =       {v: k for k, v in self.encoder.items()}
          with open(lowerCamelCase ,    encoding="utf-8"		) as merges_handle:
               __A  :					Tuple						     =       merges_handle.read().split("\n"		)[1:-1]
          __A  :					List[Any]						     =       [tuple(merge.split()		) for merge in merges]
          __A  :					Optional[int]						     =       dict(zip(lowerCamelCase ,    range(len(lowerCamelCase		)		)		)		)
          __A  :					Union[str, Any]						     =       {}


     @property
     def lowerCAmelCase__     (      self		):


          '''simple docstring'''





          return True


     @property
     def lowerCAmelCase__     (      self		):


          '''simple docstring'''





          return len(self.encoder		)


     def lowerCAmelCase__     (      self		):


          '''simple docstring'''





          return dict(self.encoder ,    **self.added_tokens_encoder		)


     def lowerCAmelCase__     (      self ,    lowerCamelCase		):


          '''simple docstring'''





          __A  :					Any						     =       tuple(token[:-1]		) + (token[-1] + "</w>",)
          if token in self.cache:
               return self.cache[token]
          __A  :					Optional[Any]						     =       get_pairs(lowerCamelCase		)

          if not pairs:
               return token + "</w>"

          while True:
               __A  :					List[Any]						     =       min(lowerCamelCase ,    key=lambda lowerCamelCase		: self.bpe_ranks.get(lowerCamelCase ,    float("inf"		)		)		)
               if bigram not in self.bpe_ranks:
                    break
               __A     ,__A  :					Tuple						     =       bigram
               __A  :					List[Any]						     =       []
               __A  :					Optional[int]						     =       0
               while i < len(lowerCamelCase		):
                    try:
                         __A  :					Dict						     =       word.index(lowerCamelCase ,    lowerCamelCase		)
                    except ValueError:
                         new_word.extend(word[i:]		)
                         break
                    else:
                         new_word.extend(word[i:j]		)
                         __A  :					int						     =       j

                    if word[i] == first and i < len(lowerCamelCase		) - 1 and word[i + 1] == second:
                         new_word.append(first + second		)
                         i += 2
                    else:
                         new_word.append(word[i]		)
                         i += 1
               __A  :					List[str]						     =       tuple(lowerCamelCase		)
               __A  :					int						     =       new_word
               if len(lowerCamelCase		) == 1:
                    break
               else:
                    __A  :					List[str]						     =       get_pairs(lowerCamelCase		)
          __A  :					Tuple						     =       " ".join(lowerCamelCase		)
          if word == "\n  </w>":
               __A  :					List[str]						     =       "\n</w>"
          __A  :					List[Any]						     =       word
          return word


     def lowerCAmelCase__     (      self ,    lowerCamelCase		):


          '''simple docstring'''





          __A  :					Dict						     =       []
          if self.fix_text is None:
               # Using BERT's BasicTokenizer
               __A  :					Tuple						     =       self.nlp.tokenize(lowerCamelCase		)
               for token in text:
                    split_tokens.extend(list(self.bpe(lowerCamelCase		).split(" "		)		)		)
          else:
               # Using SpaCy & ftfy (original tokenization process of OpenAI GPT)
               __A  :					List[Any]						     =       self.nlp(text_standardize(self.fix_text(lowerCamelCase		)		)		)
               for token in text:
                    split_tokens.extend(list(self.bpe(token.text.lower()		).split(" "		)		)		)
          return split_tokens


     def lowerCAmelCase__     (      self ,    lowerCamelCase		):


          '''simple docstring'''





          return self.encoder.get(lowerCamelCase ,    self.encoder.get(self.unk_token		)		)


     def lowerCAmelCase__     (      self ,    lowerCamelCase		):


          '''simple docstring'''





          return self.decoder.get(lowerCamelCase ,    self.unk_token		)


     def lowerCAmelCase__     (      self ,    lowerCamelCase		):


          '''simple docstring'''





          __A  :					Any						     =       "".join(lowerCamelCase		).replace("</w>" ,    " "		).strip()
          return out_string


     def lowerCAmelCase__     (      self ,    lowerCamelCase ,    lowerCamelCase = None		):


          '''simple docstring'''





          if not os.path.isdir(lowerCamelCase		):
               logger.error(f"Vocabulary path ({save_directory}) should be a directory"		)
               return
          __A  :					Optional[Any]						     =       os.path.join(
              lowerCamelCase ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)
          __A  :					str						     =       os.path.join(
              lowerCamelCase ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]		)

          with open(lowerCamelCase ,    "w" ,    encoding="utf-8"		) as f:
               f.write(json.dumps(self.encoder ,    indent=2 ,    sort_keys=lowerCamelCase ,    ensure_ascii=lowerCamelCase		) + "\n"		)

          __A  :					Tuple						     =       0
          with open(lowerCamelCase ,    "w" ,    encoding="utf-8"		) as writer:
               writer.write("#version: 0.2\n"		)
               for bpe_tokens, token_index in sorted(self.bpe_ranks.items() ,    key=lambda lowerCamelCase		: kv[1]		):
                    if index != token_index:
                         logger.warning(
                             f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
                             " Please check that the tokenizer is not corrupted!"		)
                         __A  :					Tuple						     =       token_index
                    writer.write(" ".join(lowerCamelCase		) + "\n"		)
                    index += 1

          return vocab_file, merge_file

