




'''simple docstring'''



# Lint as: python3

import hashlib
from typing import Union







def 			_UpperCamelCase (_lowerCamelCase							:     Union[str, int, bytes]	)->   bytes:




     '''simple docstring'''





     if isinstance(_lowerCamelCase   ,	_lowerCamelCase	):
          # Data already in bytes, returns as it as
          return hash_data
     elif isinstance(_lowerCamelCase   ,	_lowerCamelCase	):
          # We keep the data as it as for it ot be later encoded to UTF-8
          # However replace `\\` with `/` for Windows compatibility
          __snake_case    					=  hash_data.replace('''\\'''   ,	'''/'''	)
     elif isinstance(_lowerCamelCase   ,	_lowerCamelCase	):
          __snake_case    					=  str(_lowerCamelCase	)
     else:
          # If data is not of the required type, raise error
          raise InvalidKeyError(_lowerCamelCase	)

     return hash_data.encode('''utf-8'''	)





class     lowerCAmelCase   ( __lowerCAmelCase):


       def __init__(   self    ,  __SCREAMING_SNAKE_CASE  )      ->  str:



            '''simple docstring'''




            __snake_case    					=  '''\nFAILURE TO GENERATE DATASET: Invalid key type detected'''
            __snake_case    					=  F'''\nFound Key {hash_data} of type {type(__SCREAMING_SNAKE_CASE  )}'''
            __snake_case    					=  '''\nKeys should be either str, int or bytes type'''
            super().__init__(F'''{self.prefix}{self.err_msg}{self.suffix}'''  )





class     lowerCAmelCase   ( __lowerCAmelCase):


       def __init__(   self    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE=""  )      ->  List[str]:



            '''simple docstring'''




            __snake_case    					=  key
            __snake_case    					=  duplicate_key_indices
            __snake_case    					=  fix_msg
            __snake_case    					=  '''Found multiple examples generated with the same key'''
            if len(__SCREAMING_SNAKE_CASE  ) <= 20:
                 __snake_case    					=  F'''\nThe examples at index {", ".join(__SCREAMING_SNAKE_CASE  )} have the key {key}'''
            else:
                 __snake_case    					=  F'''\nThe examples at index {", ".join(duplicate_key_indices[:20]  )}... ({len(__SCREAMING_SNAKE_CASE  ) - 20} more) have the key {key}'''
            __snake_case    					=  '''\n''' + fix_msg if fix_msg else ''''''
            super().__init__(F'''{self.prefix}{self.err_msg}{self.suffix}'''  )





class     lowerCAmelCase   :


       def __init__(   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Any:



            '''simple docstring'''




            __snake_case    					=  hashlib.mda(_as_bytes(__SCREAMING_SNAKE_CASE  )  )




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  int:



            '''simple docstring'''




            __snake_case    					=  self._split_mda.copy()
            __snake_case    					=  _as_bytes(__SCREAMING_SNAKE_CASE  )
            mda.update(__SCREAMING_SNAKE_CASE  )
            # Convert to integer with hexadecimal conversion
            return int(mda.hexdigest()    ,  16  )












'''simple docstring'''



import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


UpperCAmelCase_       :  str      =		datasets.utils.logging.get_logger(__name__)





@dataclass
class     lowerCAmelCase   ( datasets.BuilderConfig):
       __lowercase  :      Optional[datasets.Features]        =			None
       __lowercase  :      str        =			"utf-8"
       __lowercase  :      Optional[str]        =			None
       __lowercase  :      int        =			10 << 20  # 10MB
       __lowercase  :      bool        =			False
       __lowercase  :      str        =			"line"





class     lowerCAmelCase   ( datasets.ArrowBasedBuilder):
       __lowercase  :      Dict        =			TextConfig


       def       lowerCAmelCase      (   self  )      ->  Dict:



            '''simple docstring'''




            return datasets.DatasetInfo(features=self.config.features  )


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Optional[Any]:



            '''simple docstring'''




            if not self.config.data_files:
                 raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''  )
            __snake_case    					=  dl_manager.download_and_extract(self.config.data_files  )
            if isinstance(__SCREAMING_SNAKE_CASE    ,  (str, list, tuple)  ):
                 __snake_case    					=  data_files
                 if isinstance(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  ):
                      __snake_case    					=  [files]
                 __snake_case    					=  [dl_manager.iter_files(__SCREAMING_SNAKE_CASE  ) for file in files]
                 return [datasets.SplitGenerator(name=datasets.Split.TRAIN    ,  gen_kwargs={'''files''': files}  )]
            __snake_case    					=  []
            for split_name, files in data_files.items():
                 if isinstance(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  ):
                      __snake_case    					=  [files]
                 __snake_case    					=  [dl_manager.iter_files(__SCREAMING_SNAKE_CASE  ) for file in files]
                 splits.append(datasets.SplitGenerator(name=__SCREAMING_SNAKE_CASE    ,  gen_kwargs={'''files''': files}  )  )
            return splits


       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  pa.Table:



            '''simple docstring'''




            if self.config.features is not None:
                 __snake_case    					=  self.config.features.arrow_schema
                 if all(not require_storage_cast(__SCREAMING_SNAKE_CASE  ) for feature in self.config.features.values()  ):
                      # cheaper cast
                      __snake_case    					=  pa_table.cast(__SCREAMING_SNAKE_CASE  )
                 else:
                      # more expensive cast; allows str <-> int/float or str to Audio for example
                      __snake_case    					=  table_cast(__SCREAMING_SNAKE_CASE    ,  __SCREAMING_SNAKE_CASE  )
                 return pa_table
            else:
                 return pa_table.cast(pa.schema({'''text''': pa.string()}  )  )




       def       lowerCAmelCase      (   self    ,  __SCREAMING_SNAKE_CASE  )      ->  Optional[Any]:



            '''simple docstring'''




            __snake_case    					=  list(self.config.features  ) if self.config.features is not None else ['''text''']
            for file_idx, file in enumerate(itertools.chain.from_iterable(__SCREAMING_SNAKE_CASE  )  ):
                 # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
                 with open(__SCREAMING_SNAKE_CASE    ,  encoding=self.config.encoding    ,  errors=self.config.errors  ) as f:
                      if self.config.sample_by == "line":
                           __snake_case    					=  0
                           while True:
                                __snake_case    					=  f.read(self.config.chunksize  )
                                if not batch:
                                     break
                                batch += f.readline()  # finish current line
                                # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
                                __snake_case    					=  StringIO(__SCREAMING_SNAKE_CASE  ).readlines()
                                if not self.config.keep_linebreaks:
                                     __snake_case    					=  [line.rstrip('''\n'''  ) for line in batch]
                                __snake_case    					=  pa.Table.from_arrays([pa.array(__SCREAMING_SNAKE_CASE  )]    ,  names=__SCREAMING_SNAKE_CASE  )
                                # Uncomment for debugging (will print the Arrow table size and elements)
                                # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE  )
                                batch_idx += 1
                      elif self.config.sample_by == "paragraph":
                           __snake_case    					=  0
                           __snake_case    					=  ''''''
                           while True:
                                __snake_case    					=  f.read(self.config.chunksize  )
                                if not new_batch:
                                     break
                                batch += new_batch
                                batch += f.readline()  # finish current line
                                __snake_case    					=  batch.split('''\n\n'''  )
                                __snake_case    					=  pa.Table.from_arrays(
                                    [pa.array([example for example in batch[:-1] if example]  )]    ,  names=__SCREAMING_SNAKE_CASE  )
                                # Uncomment for debugging (will print the Arrow table size and elements)
                                # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE  )
                                batch_idx += 1
                                __snake_case    					=  batch[-1]
                           if batch:
                                __snake_case    					=  pa.Table.from_arrays([pa.array([batch]  )]    ,  names=__SCREAMING_SNAKE_CASE  )
                                yield (file_idx, batch_idx), self._cast_table(__SCREAMING_SNAKE_CASE  )
                      elif self.config.sample_by == "document":
                           __snake_case    					=  f.read()
                           __snake_case    					=  pa.Table.from_arrays([pa.array([text]  )]    ,  names=__SCREAMING_SNAKE_CASE  )
                           yield file_idx, self._cast_table(__SCREAMING_SNAKE_CASE  )






