


"""simple docstring"""






import csv
import os

import pytest

from datasets import Dataset, DatasetDict, Features, NamedSplit, Value
from datasets.io.csv import CsvDatasetReader, CsvDatasetWriter

from ..utils import assert_arrow_memory_doesnt_increase, assert_arrow_memory_increases





def 							A_					(   snake_case__					,				snake_case__		)				->     str:
			assert isinstance(snake_case__					,				snake_case__		)
			assert dataset.num_rows == 4
			assert dataset.num_columns == 3
			assert dataset.column_names == ["col_1", "col_2", "col_3"]
			for feature, expected_dtype in expected_features.items():
						assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize('''keep_in_memory'''					,				[False, True]		)
def 							A_					(   snake_case__					,				snake_case__					,				snake_case__		)				->     Optional[int]:
			_UpperCamelCase				:List[Any]				    =  tmp_path / '''cache'''
			_UpperCamelCase				:Optional[int]				    =  {'''col_1''': '''int64''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
			with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
						_UpperCamelCase				:Any				    =  CsvDatasetReader(snake_case__					,				cache_dir=snake_case__					,				keep_in_memory=snake_case__		).read()
			_check_csv_dataset(snake_case__					,				snake_case__		)





@pytest.mark.parametrize(
    '''features'''					,				[
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ]					,				)
def 							A_					(   snake_case__					,				snake_case__					,				snake_case__		)				->     Union[str, Any]:
			_UpperCamelCase				:Optional[int]				    =  tmp_path / '''cache'''
			# CSV file loses col_1 string dtype information: default now is "int64" instead of "string"
			_UpperCamelCase				:Optional[Any]				    =  {'''col_1''': '''int64''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
			_UpperCamelCase				:List[Any]				    =  features.copy() if features else default_expected_features
			_UpperCamelCase				:Union[str, Any]				    =  (
			    Features({feature: Value(snake_case__		) for feature, dtype in features.items()}		) if features is not None else None
			)
			_UpperCamelCase				:Optional[Any]				    =  CsvDatasetReader(snake_case__					,				features=snake_case__					,				cache_dir=snake_case__		).read()
			_check_csv_dataset(snake_case__					,				snake_case__		)





@pytest.mark.parametrize('''split'''					,				[None, NamedSplit('''train'''		), '''train''', '''test''']		)
def 							A_					(   snake_case__					,				snake_case__					,				snake_case__		)				->     Any:
			_UpperCamelCase				:Dict				    =  tmp_path / '''cache'''
			_UpperCamelCase				:Dict				    =  {'''col_1''': '''int64''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
			_UpperCamelCase				:List[Any]				    =  CsvDatasetReader(snake_case__					,				cache_dir=snake_case__					,				split=snake_case__		).read()
			_check_csv_dataset(snake_case__					,				snake_case__		)
			assert dataset.split == split if split else "train"





@pytest.mark.parametrize('''path_type'''					,				[str, list]		)
def 							A_					(   snake_case__					,				snake_case__					,				snake_case__		)				->     Union[str, Any]:
			if issubclass(snake_case__					,				snake_case__		):
						_UpperCamelCase				:List[str]				    =  csv_path
			elif issubclass(snake_case__					,				snake_case__		):
						_UpperCamelCase				:Tuple				    =  [csv_path]
			_UpperCamelCase				:List[str]				    =  tmp_path / '''cache'''
			_UpperCamelCase				:Tuple				    =  {'''col_1''': '''int64''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
			_UpperCamelCase				:Optional[int]				    =  CsvDatasetReader(snake_case__					,				cache_dir=snake_case__		).read()
			_check_csv_dataset(snake_case__					,				snake_case__		)





def 							A_					(   snake_case__					,				snake_case__					,				snake_case__=("train",)		)				->     List[Any]:
			assert isinstance(snake_case__					,				snake_case__		)
			for split in splits:
						_UpperCamelCase				:Any				    =  dataset_dict[split]
						assert dataset.num_rows == 4
						assert dataset.num_columns == 3
						assert dataset.column_names == ["col_1", "col_2", "col_3"]
						for feature, expected_dtype in expected_features.items():
									assert dataset.features[feature].dtype == expected_dtype





@pytest.mark.parametrize('''keep_in_memory'''					,				[False, True]		)
def 							A_					(   snake_case__					,				snake_case__					,				snake_case__		)				->     Tuple:
			_UpperCamelCase				:str				    =  tmp_path / '''cache'''
			_UpperCamelCase				:Tuple				    =  {'''col_1''': '''int64''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
			with assert_arrow_memory_increases() if keep_in_memory else assert_arrow_memory_doesnt_increase():
						_UpperCamelCase				:str				    =  CsvDatasetReader({'''train''': csv_path}					,				cache_dir=snake_case__					,				keep_in_memory=snake_case__		).read()
			_check_csv_datasetdict(snake_case__					,				snake_case__		)





@pytest.mark.parametrize(
    '''features'''					,				[
        None,
        {'''col_1''': '''string''', '''col_2''': '''int64''', '''col_3''': '''float64'''},
        {'''col_1''': '''string''', '''col_2''': '''string''', '''col_3''': '''string'''},
        {'''col_1''': '''int32''', '''col_2''': '''int32''', '''col_3''': '''int32'''},
        {'''col_1''': '''float32''', '''col_2''': '''float32''', '''col_3''': '''float32'''},
    ]					,				)
def 							A_					(   snake_case__					,				snake_case__					,				snake_case__		)				->     Any:
			_UpperCamelCase				:int				    =  tmp_path / '''cache'''
			# CSV file loses col_1 string dtype information: default now is "int64" instead of "string"
			_UpperCamelCase				:str				    =  {'''col_1''': '''int64''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
			_UpperCamelCase				:List[Any]				    =  features.copy() if features else default_expected_features
			_UpperCamelCase				:Optional[Any]				    =  (
			    Features({feature: Value(snake_case__		) for feature, dtype in features.items()}		) if features is not None else None
			)
			_UpperCamelCase				:Optional[int]				    =  CsvDatasetReader({'''train''': csv_path}					,				features=snake_case__					,				cache_dir=snake_case__		).read()
			_check_csv_datasetdict(snake_case__					,				snake_case__		)





@pytest.mark.parametrize('''split'''					,				[None, NamedSplit('''train'''		), '''train''', '''test''']		)
def 							A_					(   snake_case__					,				snake_case__					,				snake_case__		)				->     str:
			if split:
						_UpperCamelCase				:str				    =  {split: csv_path}
			else:
						_UpperCamelCase				:Optional[Any]				    =  {'''train''': csv_path, '''test''': csv_path}
			_UpperCamelCase				:Any				    =  tmp_path / '''cache'''
			_UpperCamelCase				:str				    =  {'''col_1''': '''int64''', '''col_2''': '''int64''', '''col_3''': '''float64'''}
			_UpperCamelCase				:Optional[int]				    =  CsvDatasetReader(snake_case__					,				cache_dir=snake_case__		).read()
			_check_csv_datasetdict(snake_case__					,				snake_case__					,				splits=list(path.keys()		)		)
			assert all(dataset[split].split == split for split in path.keys()		)





def 							A_					(   snake_case__		)				->     Any:
			with open(snake_case__					,				encoding='''utf-8'''		) as csvfile:
						yield from csv.reader(snake_case__		)





def 							A_					(   snake_case__					,				snake_case__		)				->     str:
			_UpperCamelCase				:Tuple				    =  tmp_path / '''cache'''
			_UpperCamelCase				:Union[str, Any]				    =  os.path.join(snake_case__					,				'''tmp.csv'''		)
			_UpperCamelCase				:Optional[int]				    =  CsvDatasetReader({'''train''': csv_path}					,				cache_dir=snake_case__		).read()
			CsvDatasetWriter(dataset['''train''']					,				snake_case__					,				num_proc=1		).write()

			_UpperCamelCase				:List[Any]				    =  iter_csv_file(snake_case__		)
			_UpperCamelCase				:int				    =  iter_csv_file(snake_case__		)

			for rowa, rowa in zip(snake_case__					,				snake_case__		):
						assert rowa == rowa





def 							A_					(   snake_case__					,				snake_case__		)				->     str:
			_UpperCamelCase				:Any				    =  tmp_path / '''cache'''
			_UpperCamelCase				:Optional[Any]				    =  os.path.join(snake_case__					,				'''tmp.csv'''		)
			_UpperCamelCase				:Tuple				    =  CsvDatasetReader({'''train''': csv_path}					,				cache_dir=snake_case__		).read()
			CsvDatasetWriter(dataset['''train''']					,				snake_case__					,				num_proc=2		).write()

			_UpperCamelCase				:Union[str, Any]				    =  iter_csv_file(snake_case__		)
			_UpperCamelCase				:str				    =  iter_csv_file(snake_case__		)

			for rowa, rowa in zip(snake_case__					,				snake_case__		):
						assert rowa == rowa





def 							A_					(   snake_case__					,				snake_case__		)				->     List[str]:
			_UpperCamelCase				:Optional[Any]				    =  tmp_path / '''cache'''
			_UpperCamelCase				:List[Any]				    =  os.path.join(snake_case__					,				'''tmp.csv'''		)
			_UpperCamelCase				:int				    =  CsvDatasetReader({'''train''': csv_path}					,				cache_dir=snake_case__		).read()
			with pytest.raises(snake_case__		):
						CsvDatasetWriter(dataset['''train''']					,				snake_case__					,				num_proc=0		)





"""simple docstring"""






import textwrap

import pyarrow as pa
import pytest

from datasets import Features, Value
from datasets.packaged_modules.json.json import Json





@pytest.fixture
def 							A_					(   snake_case__		)				->     Union[str, Any]:
			_UpperCamelCase				:int				    =  tmp_path / '''file.jsonl'''
			_UpperCamelCase				:Dict				    =  textwrap.dedent(
			    '''\
        {"col_1": -1}
        {"col_1": 1, "col_2": 2}
        {"col_1": 10, "col_2": 20}
        '''		)
			with open(snake_case__					,				'''w'''		) as f:
						f.write(snake_case__		)
			return str(snake_case__		)





@pytest.fixture
def 							A_					(   snake_case__		)				->     Optional[int]:
			_UpperCamelCase				:Dict				    =  tmp_path / '''file_utf16_encoded.jsonl'''
			_UpperCamelCase				:Dict				    =  textwrap.dedent(
			    '''\
        {"col_1": -1}
        {"col_1": 1, "col_2": 2}
        {"col_1": 10, "col_2": 20}
        '''		)
			with open(snake_case__					,				'''w'''					,				encoding='''utf-16'''		) as f:
						f.write(snake_case__		)
			return str(snake_case__		)





@pytest.fixture
def 							A_					(   snake_case__		)				->     Optional[int]:
			_UpperCamelCase				:Optional[Any]				    =  tmp_path / '''file_with_list_of_dicts.json'''
			_UpperCamelCase				:str				    =  textwrap.dedent(
			    '''\
        [
            {"col_1": -1},
            {"col_1": 1, "col_2": 2},
            {"col_1": 10, "col_2": 20}
        ]
        '''		)
			with open(snake_case__					,				'''w'''		) as f:
						f.write(snake_case__		)
			return str(snake_case__		)





@pytest.fixture
def 							A_					(   snake_case__		)				->     Union[str, Any]:
			_UpperCamelCase				:Dict				    =  tmp_path / '''file_with_list_of_dicts_field.json'''
			_UpperCamelCase				:Optional[int]				    =  textwrap.dedent(
			    '''\
        {
            "field1": 1,
            "field2": "aabb",
            "field3": [
                {"col_1": -1},
                {"col_1": 1, "col_2": 2},
                {"col_1": 10, "col_2": 20}
            ]
        }
        '''		)
			with open(snake_case__					,				'''w'''		) as f:
						f.write(snake_case__		)
			return str(snake_case__		)





@pytest.mark.parametrize(
    '''file_fixture, config_kwargs'''					,				[
        ('''jsonl_file''', {}),
        ('''jsonl_file_utf16_encoded''', {'''encoding''': '''utf-16'''}),
        ('''json_file_with_list_of_dicts''', {}),
        ('''json_file_with_list_of_dicts_field''', {'''field''': '''field3'''}),
    ]					,				)
def 							A_					(   snake_case__					,				snake_case__					,				snake_case__		)				->     int:
			_UpperCamelCase				:Any				    =  Json(**snake_case__		)
			_UpperCamelCase				:str				    =  json._generate_tables([[request.getfixturevalue(snake_case__		)]]		)
			_UpperCamelCase				:List[Any]				    =  pa.concat_tables([table for _, table in generator]		)
			assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20]}





@pytest.mark.parametrize(
    '''file_fixture, config_kwargs'''					,				[
        (
            '''jsonl_file''',
            {'''features''': Features({'''col_1''': Value('''int64'''		), '''col_2''': Value('''int64'''		), '''missing_col''': Value('''string'''		)}		)},
        ),
        (
            '''json_file_with_list_of_dicts''',
            {'''features''': Features({'''col_1''': Value('''int64'''		), '''col_2''': Value('''int64'''		), '''missing_col''': Value('''string'''		)}		)},
        ),
        (
            '''json_file_with_list_of_dicts_field''',
            {
                '''field''': '''field3''',
                '''features''': Features(
                    {'''col_1''': Value('''int64'''		), '''col_2''': Value('''int64'''		), '''missing_col''': Value('''string'''		)}		),
            },
        ),
    ]					,				)
def 							A_					(   snake_case__					,				snake_case__					,				snake_case__		)				->     Optional[int]:
			_UpperCamelCase				:Union[str, Any]				    =  Json(**snake_case__		)
			_UpperCamelCase				:List[str]				    =  json._generate_tables([[request.getfixturevalue(snake_case__		)]]		)
			_UpperCamelCase				:str				    =  pa.concat_tables([table for _, table in generator]		)
			assert pa_table.to_pydict() == {"col_1": [-1, 1, 10], "col_2": [None, 2, 20], "missing_col": [None, None, None]}

