




'''simple docstring'''



import json
import os
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_       :  Tuple      =		logging.get_logger(__name__)

UpperCAmelCase_       :  List[Any]      =		{
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
}

UpperCAmelCase_       :  Any      =		{
    """vocab_file""": {
        """microsoft/biogpt""": """https://huggingface.co/microsoft/biogpt/resolve/main/vocab.json""",
    },
    """merges_file""": {"""microsoft/biogpt""": """https://huggingface.co/microsoft/biogpt/resolve/main/merges.txt"""},
}

UpperCAmelCase_       :  List[Any]      =		{
    """microsoft/biogpt""": 1_0_2_4,
}







def 			_UpperCamelCase (_lowerCamelCase							:     Tuple	)->   Optional[Any]:




     '''simple docstring'''





     __snake_case    					=  set()
     __snake_case    					=  word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)	)
          __snake_case    					=  char
     return pairs







class     lowerCAmelCase   ( _a):
       __lowercase  :      Optional[int]        =			VOCAB_FILES_NAMES
       __lowercase  :      Tuple        =			PRETRAINED_VOCAB_FILES_MAP
       __lowercase  :      Dict        =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       __lowercase  :      List[str]        =			['''input_ids''', '''attention_mask''']


       def __init__(   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE="<unk>"    ,  __SCREAMING_SNAKE_CASE="<s>"    ,  __SCREAMING_SNAKE_CASE="</s>"    ,  __SCREAMING_SNAKE_CASE="</s>"    ,  __SCREAMING_SNAKE_CASE="<pad>"    ,  **__SCREAMING_SNAKE_CASE    ,  )      ->  Any:



            '''simple docstring'''




            super().__init__(
                bos_token=lowerCamelCase__    ,  eos_token=lowerCamelCase__    ,  sep_token=lowerCamelCase__    ,  unk_token=lowerCamelCase__    ,  pad_token=lowerCamelCase__    ,  **lowerCamelCase__    ,  )

            try:
                 import sacremoses
            except ImportError:
                 raise ImportError(
                     '''You need to install sacremoses to use BioGptTokenizer. '''
                     '''See https://pypi.org/project/sacremoses/ for installation.'''  )

            __snake_case    					=  "en"
            __snake_case    					=  sacremoses
            # cache of sm.MosesTokenizer instance
            __snake_case    					=  {}
            __snake_case    					=  {}
            with open(lowerCamelCase__    ,  encoding='''utf-8'''  ) as vocab_handle:
                 __snake_case    					=  json.load(lowerCamelCase__  )
            __snake_case    					=  {v: k for k, v in self.encoder.items()}
            with open(lowerCamelCase__    ,  encoding='''utf-8'''  ) as merges_handle:
                 __snake_case    					=  merges_handle.read().split('''\n'''  )[:-1]
            __snake_case    					=  [tuple(merge.split()[:2]  ) for merge in merges]
            __snake_case    					=  dict(zip(lowerCamelCase__    ,  range(len(lowerCamelCase__  )  )  )  )
            __snake_case    					=  {}


       @property
       def       lowerCAmelCase      (   self  )      ->  int:



            '''simple docstring'''




            return len(self.encoder  )


       def       lowerCAmelCase      (   self  )      ->  str:



            '''simple docstring'''




            return dict(self.encoder    ,  **self.added_tokens_encoder  )


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )      ->  Tuple:



            '''simple docstring'''




            if lang not in self.cache_moses_tokenizer:
                 __snake_case    					=  self.sm.MosesTokenizer(lang=lowerCamelCase__  )
                 __snake_case    					=  moses_tokenizer
            return self.cache_moses_tokenizer[lang].tokenize(
                lowerCamelCase__    ,  aggressive_dash_splits=lowerCamelCase__    ,  return_str=lowerCamelCase__    ,  escape=lowerCamelCase__  )


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )      ->  Any:



            '''simple docstring'''




            if lang not in self.cache_moses_detokenizer:
                 __snake_case    					=  self.sm.MosesDetokenizer(lang=lowerCamelCase__  )
                 __snake_case    					=  moses_detokenizer
            return self.cache_moses_detokenizer[lang].detokenize(lowerCamelCase__  )


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  List[Any]:



            '''simple docstring'''




            __snake_case    					=  tuple(token[:-1]  ) + (token[-1] + "</w>",)
            if token in self.cache:
                 return self.cache[token]
            __snake_case    					=  get_pairs(lowerCamelCase__  )

            if not pairs:
                 return token + "</w>"

            while True:
                 __snake_case    					=  min(lowerCamelCase__    ,  key=lambda __SCREAMING_SNAKE_CASE  : self.bpe_ranks.get(lowerCamelCase__    ,  float('''inf'''  )  )  )
                 if bigram not in self.bpe_ranks:
                      break
                 __snake_case    					=  bigram
                 __snake_case    					=  []
                 __snake_case    					=  0
                 while i < len(lowerCamelCase__  ):
                      try:
                           __snake_case    					=  word.index(lowerCamelCase__    ,  lowerCamelCase__  )
                      except ValueError:
                           new_word.extend(word[i:]  )
                           break
                      else:
                           new_word.extend(word[i:j]  )
                           __snake_case    					=  j

                      if word[i] == first and i < len(lowerCamelCase__  ) - 1 and word[i + 1] == second:
                           new_word.append(first + second  )
                           i += 2
                      else:
                           new_word.append(word[i]  )
                           i += 1
                 __snake_case    					=  tuple(lowerCamelCase__  )
                 __snake_case    					=  new_word
                 if len(lowerCamelCase__  ) == 1:
                      break
                 else:
                      __snake_case    					=  get_pairs(lowerCamelCase__  )
            __snake_case    					=  " ".join(lowerCamelCase__  )
            if word == "\n  </w>":
                 __snake_case    					=  "\n</w>"
            __snake_case    					=  word
            return word


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE=False  )      ->  Tuple:



            '''simple docstring'''




            if bypass_tokenizer:
                 __snake_case    					=  text.split()
            else:
                 __snake_case    					=  self.moses_tokenize(lowerCamelCase__    ,  self.lang  )

            __snake_case    					=  []
            for token in text:
                 if token:
                      split_tokens.extend(list(self.bpe(lowerCamelCase__  ).split(''' '''  )  )  )

            return split_tokens


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Union[str, Any]:



            '''simple docstring'''




            return self.encoder.get(lowerCamelCase__    ,  self.encoder.get(self.unk_token  )  )


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  str:



            '''simple docstring'''




            return self.decoder.get(lowerCamelCase__    ,  self.unk_token  )


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Dict:



            '''simple docstring'''




            __snake_case    					=  [t.replace(''' '''    ,  ''''''  ).replace('''</w>'''    ,  ''' '''  ) for t in tokens]
            __snake_case    					=  "".join(lowerCamelCase__  ).split()
            # detokenize
            __snake_case    					=  self.moses_detokenize(lowerCamelCase__    ,  self.lang  )
            return text


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE = None  )      ->  Union[str, Any]:



            '''simple docstring'''




            if token_ids_a is None:
                 return [self.sep_token_id] + token_ids_a
            __snake_case    					=  [self.sep_token_id]
            return sep + token_ids_a + sep + token_ids_a


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE = None    ,  __SCREAMING_SNAKE_CASE = False  )      ->  str:



            '''simple docstring'''




            if already_has_special_tokens:
                 return super().get_special_tokens_mask(
                     token_ids_a=lowerCamelCase__    ,  token_ids_a=lowerCamelCase__    ,  already_has_special_tokens=lowerCamelCase__  )
            # no bos used in fairseq
            if token_ids_a is not None:
                 return [1] + ([0] * len(lowerCamelCase__  )) + [1] + ([0] * len(lowerCamelCase__  ))
            return [1] + ([0] * len(lowerCamelCase__  ))


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE = None  )      ->  Dict:



            '''simple docstring'''




            __snake_case    					=  [self.sep_token_id]

            # no bos used in fairseq
            if token_ids_a is None:
                 return len(token_ids_a + sep  ) * [0]
            return len(token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE = None  )      ->  Optional[int]:



            '''simple docstring'''




            if not os.path.isdir(lowerCamelCase__  ):
                 logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''  )
                 return
            __snake_case    					=  os.path.join(
                lowerCamelCase__    ,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']  )
            __snake_case    					=  os.path.join(
                lowerCamelCase__    ,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']  )

            with open(lowerCamelCase__    ,  '''w'''    ,  encoding='''utf-8'''  ) as f:
                 f.write(json.dumps(self.encoder    ,  indent=2    ,  sort_keys=lowerCamelCase__    ,  ensure_ascii=lowerCamelCase__  ) + '''\n'''  )

            __snake_case    					=  0
            with open(lowerCamelCase__    ,  '''w'''    ,  encoding='''utf-8'''  ) as writer:
                 for bpe_tokens, token_index in sorted(self.bpe_ranks.items()    ,  key=lambda __SCREAMING_SNAKE_CASE  : kv[1]  ):
                      if index != token_index:
                           logger.warning(
                               F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                               ''' Please check that the tokenizer is not corrupted!'''  )
                           __snake_case    					=  token_index
                      writer.write(''' '''.join(lowerCamelCase__  ) + '''\n'''  )
                      index += 1

            return vocab_file, merge_file


       def __getstate__(   self  )      ->  Any:



            '''simple docstring'''




            __snake_case    					=  self.__dict__.copy()
            __snake_case    					=  None
            return state




       def __setstate__(   self    ,  __SCREAMING_SNAKE_CASE  )      ->  List[str]:



            '''simple docstring'''




            __snake_case    					=  d

            try:
                 import sacremoses
            except ImportError:
                 raise ImportError(
                     '''You need to install sacremoses to use XLMTokenizer. '''
                     '''See https://pypi.org/project/sacremoses/ for installation.'''  )

            __snake_case    					=  sacremoses









'''simple docstring'''



import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


UpperCAmelCase_       :  str      =		datasets.utils.logging.get_logger(__name__)





@dataclass
class     lowerCAmelCase   ( datasets.BuilderConfig):
       __lowercase  :      Optional[datasets.Features]        =			None
       __lowercase  :      str        =			"utf-8"
       __lowercase  :      Optional[str]        =			None
       __lowercase  :      int        =			10 << 20  # 10MB
       __lowercase  :      bool        =			False
       __lowercase  :      str        =			"line"





class     lowerCAmelCase   ( datasets.ArrowBasedBuilder):
       __lowercase  :      Dict        =			TextConfig


       def       lowerCAmelCase      (   self  )      ->  Dict:



            '''simple docstring'''




            return datasets.DatasetInfo(features=self.config.features  )


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Optional[Any]:



            '''simple docstring'''




            if not self.config.data_files:
                 raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''  )
            __snake_case    					=  dl_manager.download_and_extract(self.config.data_files  )
            if isinstance(__SCREAMING_SNAKE_CASE    ,  (str, list, tuple)  ):
                 __snake_case    					=  data_files
                 if isinstance(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  ):
                      __snake_case    					=  [files]
                 __snake_case    					=  [dl_manager.iter_files(__SCREAMING_SNAKE_CASE  ) for file in files]
                 return [datasets.SplitGenerator(name=datasets.Split.TRAIN    ,  gen_kwargs={'''files''': files}  )]
            __snake_case    					=  []
            for split_name, files in data_files.items():
                 if isinstance(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  ):
                      __snake_case    					=  [files]
                 __snake_case    					=  [dl_manager.iter_files(__SCREAMING_SNAKE_CASE  ) for file in files]
                 splits.append(datasets.SplitGenerator(name=__SCREAMING_SNAKE_CASE    ,  gen_kwargs={'''files''': files}  )  )
            return splits


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  pa.Table:



            '''simple docstring'''




            if self.config.features is not None:
                 __snake_case    					=  self.config.features.arrow_schema
                 if all(not require_storage_cast(__SCREAMING_SNAKE_CASE  ) for feature in self.config.features.values()  ):
                      # cheaper cast
                      __snake_case    					=  pa_table.cast(__SCREAMING_SNAKE_CASE  )
                 else:
                      # more expensive cast; allows str <-> int/float or str to Audio for example
                      __snake_case    					=  table_cast(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )
                 return pa_table
            else:
                 return pa_table.cast(pa.schema({'''text''': pa.string()}  )  )




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Optional[Any]:



            '''simple docstring'''




            __snake_case    					=  list(self.config.features  ) if self.config.features is not None else ['''text''']
            for file_idx, file in enumerate(itertools.chain.from_iterable(__SCREAMING_SNAKE_CASE  )  ):
                 # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
                 with open(__SCREAMING_SNAKE_CASE    ,  encoding=self.config.encoding    ,  errors=self.config.errors  ) as f:
                      if self.config.sample_by == "line":
                           __snake_case    					=  0
                           while True:
                                __snake_case    					=  f.read(self.config.chunksize  )
                                if not batch:
                                     break
                                batch += f.readline()  # finish current line
                                # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                                __snake_case    					=  StringIO(__SCREAMING_SNAKE_CASE  ).readlines()
                                if not self.config.keep_linebreaks:
                                     __snake_case    					=  [line.rstrip('''\n'''  ) for line in batch]
                                __snake_case    					=  pa.Table.from_arrays([pa.array(__SCREAMING_SNAKE_CASE  )]    ,  names=__SCREAMING_SNAKE_CASE  )
                                # Uncomment for debugging (will print the Arrow table size and elements)
                                # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE  )
                                batch_idx += 1
                      elif self.config.sample_by == "paragraph":
                           __snake_case    					=  0
                           __snake_case    					=  ''''''
                           while True:
                                __snake_case    					=  f.read(self.config.chunksize  )
                                if not new_batch:
                                     break
                                batch += new_batch
                                batch += f.readline()  # finish current line
                                __snake_case    					=  batch.split('''\n\n'''  )
                                __snake_case    					=  pa.Table.from_arrays(
                                    [pa.array([example for example in batch[:-1] if example]  )]    ,  names=__SCREAMING_SNAKE_CASE  )
                                # Uncomment for debugging (will print the Arrow table size and elements)
                                # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE  )
                                batch_idx += 1
                                __snake_case    					=  batch[-1]
                           if batch:
                                __snake_case    					=  pa.Table.from_arrays([pa.array([batch]  )]    ,  names=__SCREAMING_SNAKE_CASE  )
                                yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE  )
                      elif self.config.sample_by == "document":
                           __snake_case    					=  f.read()
                           __snake_case    					=  pa.Table.from_arrays([pa.array([text]  )]    ,  names=__SCREAMING_SNAKE_CASE  )
                           yield file_idx, self._cast_table(__SCREAMING_SNAKE_CASE  )






