




'''simple docstring'''




import os
import tempfile
from pathlib import Path
from unittest import TestCase

import pyarrow as pa
import pytest

from datasets.arrow_dataset import Dataset
from datasets.arrow_reader import ArrowReader, BaseReader, FileInstructions, ReadInstruction, make_file_instructions
from datasets.info import DatasetInfo
from datasets.splits import NamedSplit, Split, SplitDict, SplitInfo

from .utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases


class 						__UpperCamelCase (				lowerCamelCase__       ):






       def   lowercase__  (							self,	lowerCAmelCase,	lowerCAmelCase=False						):



              """simple docstring"""







              lowerCamelCase_,   lowerCamelCase_,   lowerCamelCase_							=(
                  filename_skip_take['''filename'''],
                  filename_skip_take['''skip'''] if '''skip''' in filename_skip_take else None,
                  filename_skip_take['''take'''] if '''take''' in filename_skip_take else None,
              )
              open(os.path.join(lowerCAmelCase						),	'''wb'''						).close()
              lowerCamelCase_							=pa.Table.from_pydict({'''filename''': [Path(lowerCAmelCase						).name] * 100}						)
              if take == -1:
                     lowerCamelCase_							=len(lowerCAmelCase						) - skip
              if skip is not None and take is not None:
                     lowerCamelCase_							=pa_table.slice(lowerCAmelCase,	lowerCAmelCase						)
              return pa_table





class 						__UpperCamelCase (				lowerCamelCase__       ):






       def   lowercase__  (							self						):



              """simple docstring"""







              lowerCamelCase_							='''my_name'''
              lowerCamelCase_							=SplitInfo(name='''train''',	num_examples=100						)
              lowerCamelCase_							=SplitInfo(name='''test''',	num_examples=100						)
              lowerCamelCase_							=[train_info, test_info]
              lowerCamelCase_							=SplitDict()
              split_dict.add(lowerCAmelCase						)
              split_dict.add(lowerCAmelCase						)
              lowerCamelCase_							=DatasetInfo(splits=lowerCAmelCase						)

              with tempfile.TemporaryDirectory() as tmp_dir:
                     lowerCamelCase_							=ReaderTest(lowerCAmelCase,	lowerCAmelCase						)

                     lowerCamelCase_							='''test[:33%]'''
                     lowerCamelCase_							=Dataset(**reader.read(lowerCAmelCase,	lowerCAmelCase,	lowerCAmelCase						)						)
                     self.assertEqual(dset['''filename'''][0],	f'''{name}-test'''						)
                     self.assertEqual(dset.num_rows,	33						)
                     self.assertEqual(dset.num_columns,	1						)

                     lowerCamelCase_							=['''train''', '''test[:33%]''']
                     lowerCamelCase_							=[Split.TRAIN, ReadInstruction.from_spec('''test[:33%]'''						)]
                     for instructions in [instructionsa, instructionsa]:
                            lowerCamelCase_							=[reader.read(lowerCAmelCase,	lowerCAmelCase,	lowerCAmelCase						) for instr in instructions]
                            lowerCamelCase_,   lowerCamelCase_							=(Dataset(**lowerCAmelCase						) for dataset_kwargs in datasets_kwargs)
                            self.assertEqual(train_dset['''filename'''][0],	f'''{name}-train'''						)
                            self.assertEqual(train_dset.num_rows,	100						)
                            self.assertEqual(train_dset.num_columns,	1						)
                            self.assertIsInstance(train_dset.split,	lowerCAmelCase						)
                            self.assertEqual(str(train_dset.split						),	'''train'''						)
                            self.assertEqual(test_dset['''filename'''][0],	f'''{name}-test'''						)
                            self.assertEqual(test_dset.num_rows,	33						)
                            self.assertEqual(test_dset.num_columns,	1						)
                            self.assertIsInstance(test_dset.split,	lowerCAmelCase						)
                            self.assertEqual(str(test_dset.split						),	'''test[:33%]'''						)
                            del train_dset, test_dset






       def   lowercase__  (							self						):



              """simple docstring"""







              lowerCamelCase_							='''my_name'''
              lowerCamelCase_							=SplitInfo(name='''train''',	num_examples=1_000,	shard_lengths=[100] * 10						)
              lowerCamelCase_							=[train_info]
              lowerCamelCase_							=SplitDict()
              split_dict.add(lowerCAmelCase						)
              lowerCamelCase_							=DatasetInfo(splits=lowerCAmelCase						)

              with tempfile.TemporaryDirectory() as tmp_dir:
                     lowerCamelCase_							=ReaderTest(lowerCAmelCase,	lowerCAmelCase						)

                     lowerCamelCase_							='''train[:33%]'''
                     lowerCamelCase_							=Dataset(**reader.read(lowerCAmelCase,	lowerCAmelCase,	lowerCAmelCase						)						)
                     self.assertEqual(dset['''filename'''][0],	f'''{name}-train-00000-of-00010'''						)
                     self.assertEqual(dset['''filename'''][-1],	f'''{name}-train-00003-of-00010'''						)
                     self.assertEqual(dset.num_rows,	330						)
                     self.assertEqual(dset.num_columns,	1						)






       def   lowercase__  (							self						):



              """simple docstring"""







              lowerCamelCase_							=SplitInfo(name='''train''',	num_examples=100						)
              lowerCamelCase_							=SplitInfo(name='''test''',	num_examples=100						)
              lowerCamelCase_							=SplitDict()
              split_dict.add(lowerCAmelCase						)
              split_dict.add(lowerCAmelCase						)
              lowerCamelCase_							=DatasetInfo(splits=lowerCAmelCase						)

              with tempfile.TemporaryDirectory() as tmp_dir:
                     lowerCamelCase_							=ReaderTest(lowerCAmelCase,	lowerCAmelCase						)

                     lowerCamelCase_							=[
                         {'''filename''': os.path.join(lowerCAmelCase,	'''train'''						)},
                         {'''filename''': os.path.join(lowerCAmelCase,	'''test'''						), '''skip''': 10, '''take''': 10},
                     ]
                     lowerCamelCase_							=Dataset(**reader.read_files(lowerCAmelCase,	original_instructions='''train+test[10:20]'''						)						)
                     self.assertEqual(dset.num_rows,	110						)
                     self.assertEqual(dset.num_columns,	1						)
                     del dset





@pytest.mark.parametrize('''in_memory'''     , [False, True]     )
def   a_							(   __snake_case							:    Union[str, Any]     , __snake_case							:    List[Any]     , __snake_case							:    Union[str, Any]     )							->		Dict:



       """simple docstring"""
       lowerCamelCase_							=arrow_file
       with assert_arrow_memory_increases() if in_memory else assert_arrow_memory_doesnt_increase():
              lowerCamelCase_							=ArrowReader.read_table(__snake_case     , in_memory=__snake_case     )
       assert table.shape == dataset.data.shape
       assert set(table.column_names     ) == set(dataset.data.column_names     )
       assert dict(table.to_pydict()     ) == dict(dataset.data.to_pydict()     )  # to_pydict returns OrderedDict





@pytest.mark.parametrize('''in_memory'''     , [False, True]     )
def   a_							(   __snake_case							:    str     , __snake_case							:    Union[str, Any]     , __snake_case							:    Optional[int]     )							->		List[Any]:



       """simple docstring"""
       lowerCamelCase_							=arrow_file
       lowerCamelCase_							=ArrowReader(''''''     , __snake_case     )
       with assert_arrow_memory_increases() if in_memory else assert_arrow_memory_doesnt_increase():
              lowerCamelCase_							=reader.read_files([{'''filename''': filename}]     , in_memory=__snake_case     )
       assert dataset_kwargs.keys() == {"arrow_table", "info", "split"}
       lowerCamelCase_							=dataset_kwargs['''arrow_table''']
       assert table.shape == dataset.data.shape
       assert set(table.column_names     ) == set(dataset.data.column_names     )
       assert dict(table.to_pydict()     ) == dict(dataset.data.to_pydict()     )  # to_pydict returns OrderedDict





def   a_							(   )							->		Any:



       """simple docstring"""
       assert ReadInstruction('''train'''     , to=10     , unit='''abs'''     ).to_spec() == "train[:10]"
       assert ReadInstruction('''train'''     , from_=-80     , to=10     , unit='''%'''     ).to_spec() == "train[-80%:10%]"

       lowerCamelCase_							='''train+test'''
       assert ReadInstruction.from_spec(__snake_case     ).to_spec() == spec_train_test

       lowerCamelCase_							='''train[2:10]'''
       assert ReadInstruction.from_spec(__snake_case     ).to_spec() == spec_train_abs

       lowerCamelCase_							='''train[15%:-20%]'''
       assert ReadInstruction.from_spec(__snake_case     ).to_spec() == spec_train_pct

       lowerCamelCase_							='''train[:10%](closest)'''
       assert ReadInstruction.from_spec(__snake_case     ).to_spec() == "train[:10%]"

       lowerCamelCase_							='''train[:10%](pct1_dropremainder)'''
       assert ReadInstruction.from_spec(__snake_case     ).to_spec() == spec_train_pct_rounding

       lowerCamelCase_							='''train[:10%](pct1_dropremainder)+test[-10%:](pct1_dropremainder)'''
       assert ReadInstruction.from_spec(__snake_case     ).to_spec() == spec_train_test_pct_rounding





def   a_							(   )							->		List[str]:



       """simple docstring"""
       lowerCamelCase_							='''dummy'''
       lowerCamelCase_							=[SplitInfo(name='''train'''     , num_examples=100     )]
       lowerCamelCase_							='''train[:33%]'''
       lowerCamelCase_							='''arrow'''
       lowerCamelCase_							='''prefix'''

       lowerCamelCase_							=make_file_instructions(__snake_case     , __snake_case     , __snake_case     , __snake_case     , __snake_case     )
       assert isinstance(__snake_case     , __snake_case     )
       assert file_instructions.num_examples == 33
       assert file_instructions.file_instructions == [
           {"filename": os.path.join(__snake_case     , F'''{name}-train.arrow'''     ), "skip": 0, "take": 33}
       ]

       lowerCamelCase_							=[SplitInfo(name='''train'''     , num_examples=100     , shard_lengths=[10] * 10     )]
       lowerCamelCase_							=make_file_instructions(__snake_case     , __snake_case     , __snake_case     , __snake_case     , __snake_case     )
       assert isinstance(__snake_case     , __snake_case     )
       assert file_instructions.num_examples == 33
       assert file_instructions.file_instructions == [
           {"filename": os.path.join(__snake_case     , F'''{name}-train-00000-of-00010.arrow'''     ), "skip": 0, "take": -1},
           {"filename": os.path.join(__snake_case     , F'''{name}-train-00001-of-00010.arrow'''     ), "skip": 0, "take": -1},
           {"filename": os.path.join(__snake_case     , F'''{name}-train-00002-of-00010.arrow'''     ), "skip": 0, "take": -1},
           {"filename": os.path.join(__snake_case     , F'''{name}-train-00003-of-00010.arrow'''     ), "skip": 0, "take": 3},
       ]





@pytest.mark.parametrize('''name, expected_exception'''     , [(None, TypeError), ('''''', ValueError)]     )
def   a_							(   __snake_case							:    List[str]     , __snake_case							:    Optional[Any]     )							->		Dict:



       """simple docstring"""
       lowerCamelCase_							=[SplitInfo(name='''train'''     , num_examples=100     )]
       lowerCamelCase_							='''train'''
       lowerCamelCase_							='''arrow'''
       lowerCamelCase_							='''prefix_path'''
       with pytest.raises(__snake_case     ):
              lowerCamelCase_							=make_file_instructions(__snake_case     , __snake_case     , __snake_case     , __snake_case     , __snake_case     )










'''simple docstring'''




from .hash_table import HashTable


class 						__UpperCamelCase (				lowerCamelCase__       ):






       def __init__(							self,	*lowerCAmelCase,	**lowerCAmelCase						):



              """simple docstring"""







              super().__init__(*lowerCAmelCase,	**lowerCAmelCase						)






       def   lowercase__  (							self,	lowerCAmelCase,	lowerCAmelCase=None						):



              """simple docstring"""







              lowerCamelCase_							=1
              lowerCamelCase_							=self.hash_function(key + i * i						)

              while self.values[new_key] is not None and self.values[new_key] != key:
                     i += 1
                     lowerCamelCase_							=(
                         self.hash_function(key + i * i						)
                         if not self.balanced_factor() >= self.lim_charge
                         else None
                     )

                     if new_key is None:
                            break

              return new_key




