c4 dataset streaming fails with `FileNotFoundError`

Also encountering this issue for every dataset I try to stream! Installed datasets from main:
```
- `datasets` version: 2.10.1.dev0
- Platform: macOS-13.1-arm64-arm-64bit
- Python version: 3.9.13
- PyArrow version: 10.0.1
- Pandas version: 1.5.2
```

Repro:
```python
from datasets import load_dataset

spigi = load_dataset("kensho/spgispeech", "dev", split="validation", streaming=True, use_auth_token=True)
sample = next(iter(spigi))
```

<details>
<summary> Traceback </summary>

```python
---------------------------------------------------------------------------
ClientResponseError                       Traceback (most recent call last)
File ~/venv/lib/python3.9/site-packages/fsspec/implementations/http.py:407, in HTTPFileSystem._info(self, url, **kwargs)
    405 try:
    406     info.update(
--> 407         await _file_info(
    408             self.encode_url(url),
    409             size_policy=policy,
    410             session=session,
    411             **self.kwargs,
    412             **kwargs,
    413         )
    414     )
    415     if info.get("size") is not None:

File ~/venv/lib/python3.9/site-packages/fsspec/implementations/http.py:792, in _file_info(url, session, size_policy, **kwargs)
    791 async with r:
--> 792     r.raise_for_status()
    794     # TODO:
    795     #  recognise lack of 'Accept-Ranges',
    796     #                 or 'Accept-Ranges': 'none' (not 'bytes')
    797     #  to mean streaming only, no random access => return None

File ~/venv/lib/python3.9/site-packages/aiohttp/client_reqrep.py:1005, in ClientResponse.raise_for_status(self)
   1004 self.release()
-> 1005 raise ClientResponseError(
   1006     self.request_info,
   1007     self.history,
   1008     status=self.status,
   1009     message=self.reason,
   1010     headers=self.headers,
   1011 )

ClientResponseError: 403, message='Forbidden', url=URL('[https://cdn-lfs.huggingface.co/repos/e2/89/e28905247d6f48bb4edad5baf9b1bb4158e897a13fdf18bf3b8ee89ff8387ab8/46eca7431a7b6bad344bf451800e5b10cea1dd168f26d1027a6d9eb374b7fac3?response-content-disposition=attachment%3B+filename*%3DUTF-8''dev.csv%3B+filename%3D%22dev.csv%22%3B&response-content-type=text/csv&Expires=1677494732&Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9jZG4tbGZzLmh1Z2dpbmdmYWNlLmNvL3JlcG9zL2UyLzg5L2UyODkwNTI0N2Q2ZjQ4YmI0ZWRhZDViYWY5YjFiYjQxNThlODk3YTEzZmRmMThiZjNiOGVlODlmZjgzODdhYjgvNDZlY2E3NDMxYTdiNmJhZDM0NGJmNDUxODAwZTViMTBjZWExZGQxNjhmMjZkMTAyN2E2ZDllYjM3NGI3ZmFjMz9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSomcmVzcG9uc2UtY29udGVudC10eXBlPXRleHQlMkZjc3YiLCJDb25kaXRpb24iOnsiRGF0ZUxlc3NUaGFuIjp7IkFXUzpFcG9jaFRpbWUiOjE2Nzc0OTQ3MzJ9fX1dfQ__&Signature=EzQB9f7xPckvqfFB6LzcyR-wzTnQCqtPDdWtQUzZ3QJ-gY-IHG5mxQITJgMr1nVTbJZrPmGAaDngMcPFUfSQa8RmCqYH~dZl-UGE8CO4neKNUT1DvA2WEvLDS4WaAJ3SN-9rX0uFb03~c1QS78cIgIRboYvf6ugKiJz86Bd7Vs~tcp201JFR0A6jIMseqApOnkb9d8dHMP3Ny~F6gO3Qf2QpEWM-QsDIyw2Kz2QV55nq8TsDpRYZCZo50~WwD~73Hej0PoDhEA1K37d19pa0CQhkaN-gjCrbT9xLabbvhJWa~ZkWcMdD0teCgjYqv1wKyvFXDAxukxLGEc7OBXVbYw__&Key-Pair-Id=KVTP0A1DKRTAX](https://cdn-lfs.huggingface.co/repos/e2/89/e28905247d6f48bb4edad5baf9b1bb4158e897a13fdf18bf3b8ee89ff8387ab8/46eca7431a7b6bad344bf451800e5b10cea1dd168f26d1027a6d9eb374b7fac3?response-content-disposition=attachment%3B+filename*%3DUTF-8%27%27dev.csv%3B+filename%3D%22dev.csv%22%3B&response-content-type=text/csv&Expires=1677494732&Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9jZG4tbGZzLmh1Z2dpbmdmYWNlLmNvL3JlcG9zL2UyLzg5L2UyODkwNTI0N2Q2ZjQ4YmI0ZWRhZDViYWY5YjFiYjQxNThlODk3YTEzZmRmMThiZjNiOGVlODlmZjgzODdhYjgvNDZlY2E3NDMxYTdiNmJhZDM0NGJmNDUxODAwZTViMTBjZWExZGQxNjhmMjZkMTAyN2E2ZDllYjM3NGI3ZmFjMz9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSomcmVzcG9uc2UtY29udGVudC10eXBlPXRleHQlMkZjc3YiLCJDb25kaXRpb24iOnsiRGF0ZUxlc3NUaGFuIjp7IkFXUzpFcG9jaFRpbWUiOjE2Nzc0OTQ3MzJ9fX1dfQ__&Signature=EzQB9f7xPckvqfFB6LzcyR-wzTnQCqtPDdWtQUzZ3QJ-gY-IHG5mxQITJgMr1nVTbJZrPmGAaDngMcPFUfSQa8RmCqYH~dZl-UGE8CO4neKNUT1DvA2WEvLDS4WaAJ3SN-9rX0uFb03~c1QS78cIgIRboYvf6ugKiJz86Bd7Vs~tcp201JFR0A6jIMseqApOnkb9d8dHMP3Ny~F6gO3Qf2QpEWM-QsDIyw2Kz2QV55nq8TsDpRYZCZo50~WwD~73Hej0PoDhEA1K37d19pa0CQhkaN-gjCrbT9xLabbvhJWa~ZkWcMdD0teCgjYqv1wKyvFXDAxukxLGEc7OBXVbYw__&Key-Pair-Id=KVTP0A1DKRTAX)')

The above exception was the direct cause of the following exception:

FileNotFoundError                         Traceback (most recent call last)
Cell In[5], line 4
      1 from datasets import load_dataset
      3 spigi = load_dataset("kensho/spgispeech", "dev", split="validation", streaming=True)
----> 4 sample = next(iter(spigi))

File ~/datasets/src/datasets/iterable_dataset.py:937, in IterableDataset.__iter__(self)
    934         yield from self._iter_pytorch(ex_iterable)
    935         return
--> 937 for key, example in ex_iterable:
    938     if self.features:
    939         # `IterableDataset` automatically fills missing columns with None.
    940         # This is done with `_apply_feature_types_on_example`.
    941         yield _apply_feature_types_on_example(
    942             example, self.features, token_per_repo_id=self._token_per_repo_id
    943         )

File ~/datasets/src/datasets/iterable_dataset.py:113, in ExamplesIterable.__iter__(self)
    112 def __iter__(self):
--> 113     yield from self.generate_examples_fn(**self.kwargs)

File ~/.cache/huggingface/modules/datasets_modules/datasets/kensho--spgispeech/5fbf75dd9ef795a9b5a673457d2cbaf0b8fa0de8fb62acbd1da338d83a41e2f0/spgispeech.py:186, in Spgispeech._generate_examples(self, local_extracted_archive_paths, archives, meta_path)
    183 dict_keys = ["wav_filename", "wav_filesize", "transcript"]
    185 logging.info("Reading metadata...")
--> 186 with open(meta_path, encoding="utf-8") as f:
    187     csvreader = csv.DictReader(f, delimiter="|")
    188     metadata = {x["wav_filename"]: dict((k, x[k]) for k in dict_keys) for x in csvreader}

File ~/datasets/src/datasets/streaming.py:70, in extend_module_for_streaming.<locals>.wrap_auth.<locals>.wrapper(*args, **kwargs)
     68 @wraps(function)
     69 def wrapper(*args, **kwargs):
---> 70     return function(*args, use_auth_token=use_auth_token, **kwargs)

File ~/datasets/src/datasets/download/streaming_download_manager.py:495, in xopen(file, mode, use_auth_token, *args, **kwargs)
    493 kwargs = {**kwargs, **new_kwargs}
    494 try:
--> 495     file_obj = fsspec.open(file, mode=mode, *args, **kwargs).open()
    496 except ValueError as e:
    497     if str(e) == "Cannot seek streaming HTTP file":

File ~/venv/lib/python3.9/site-packages/fsspec/core.py:135, in OpenFile.open(self)
    128 def open(self):
    129     """Materialise this as a real open file without context
    130
    131     The OpenFile object should be explicitly closed to avoid enclosed file
    132     instances persisting. You must, therefore, keep a reference to the OpenFile
    133     during the life of the file-like it generates.
    134     """
--> 135     return self.__enter__()

File ~/venv/lib/python3.9/site-packages/fsspec/core.py:103, in OpenFile.__enter__(self)
    100 def __enter__(self):
    101     mode = self.mode.replace("t", "").replace("b", "") + "b"
--> 103     f = self.fs.open(self.path, mode=mode)
    105     self.fobjects = [f]
    107     if self.compression is not None:

File ~/venv/lib/python3.9/site-packages/fsspec/spec.py:1106, in AbstractFileSystem.open(self, path, mode, block_size, cache_options, compression, **kwargs)
   1104 else:
   1105     ac = kwargs.pop("autocommit", not self._intrans)
-> 1106     f = self._open(
   1107         path,
   1108         mode=mode,
   1109         block_size=block_size,
   1110         autocommit=ac,
   1111         cache_options=cache_options,
   1112         **kwargs,
   1113     )
   1114     if compression is not None:
   1115         from fsspec.compression import compr

File ~/venv/lib/python3.9/site-packages/fsspec/implementations/http.py:346, in HTTPFileSystem._open(self, path, mode, block_size, autocommit, cache_type, cache_options, size, **kwargs)
    344 kw["asynchronous"] = self.asynchronous
    345 kw.update(kwargs)
--> 346 size = size or self.info(path, **kwargs)["size"]
    347 session = sync(self.loop, self.set_session)
    348 if block_size and size:

File ~/venv/lib/python3.9/site-packages/fsspec/asyn.py:113, in sync_wrapper.<locals>.wrapper(*args, **kwargs)
    110 @functools.wraps(func)
    111 def wrapper(*args, **kwargs):
    112     self = obj or args[0]
--> 113     return sync(self.loop, func, *args, **kwargs)

File ~/venv/lib/python3.9/site-packages/fsspec/asyn.py:98, in sync(loop, func, timeout, *args, **kwargs)
     96     raise FSTimeoutError from return_result
     97 elif isinstance(return_result, BaseException):
---> 98     raise return_result
     99 else:
    100     return return_result

File ~/venv/lib/python3.9/site-packages/fsspec/asyn.py:53, in _runner(event, coro, result, timeout)
     51     coro = asyncio.wait_for(coro, timeout=timeout)
     52 try:
---> 53     result[0] = await coro
     54 except Exception as ex:
     55     result[0] = ex

File ~/venv/lib/python3.9/site-packages/fsspec/implementations/http.py:420, in HTTPFileSystem._info(self, url, **kwargs)
    417     except Exception as exc:
    418         if policy == "get":
    419             # If get failed, then raise a FileNotFoundError
--> 420             raise FileNotFoundError(url) from exc
    421         logger.debug(str(exc))
    423 return {"name": url, "size": None, **info, "type": "file"}

FileNotFoundError: https://huggingface.co/datasets/kensho/spgispeech/resolve/main/data/meta/dev.csv
```
</details>

Hi ! We're investigating this issue, sorry for the inconvenience

This has been resolved ! Thanks for reporting

This problem now appears again, this time with an underlying HTTP 502 status code:

```
aiohttp.client_exceptions.ClientResponseError: 502, message='Bad Gateway', url=URL('https://huggingface.co/datasets/allenai/c4/resolve/1ddc917116b730e1859edef32896ec5c16be51d0/en/c4-validation.00002-of-00008.json.gz')
```

I'm facing the same problem. Interestingly using `wget` I can download the file.

> It's been resolved again ;)

I'm experiencing the same issue when trying to load this dataset, `FileNotFoundError: https://huggingface.co/datasets/allenai/c4/resolve/1ddc917116b730e1859edef32896ec5c16be51d0/realnewslike/c4-train.00000-of-00512.json.gz`

Experiencing the same issues as above : `FileNotFoundError: https://huggingface.co/datasets/allenai/c4/resolve/1ddc917116b730e1859edef32896ec5c16be51d0/en/c4-train.00000-of-01024.json.gz
If the repo is private or gated, make sure to log in with `huggingface-cli login`.`

Have made sure to login as well, issue persists.

> Experiencing the same issues as above : `FileNotFoundError: https://huggingface.co/datasets/allenai/c4/resolve/1ddc917116b730e1859edef32896ec5c16be51d0/en/c4-train.00000-of-01024.json.gz If the repo is private or gated, make sure to log in with `huggingface-cli login`.`
>
> Have made sure to login as well, issue persists.

I meet the same issue

### Describe the bug

Loading the `c4` dataset in streaming mode with `load_dataset("c4", "en", split="validation", streaming=True)` and then using it fails with a `FileNotFoundException`.

### Steps to reproduce the bug

```python
from datasets import load_dataset

dataset = load_dataset("c4", "en", split="train", streaming=True)
next(iter(dataset))
```

causes a

```
FileNotFoundError: https://huggingface.co/datasets/allenai/c4/resolve/1ddc917116b730e1859edef32896ec5c16be51d0/en/c4-train.00000-of-01024.json.gz
```

I can download this file manually though e.g. by entering this URL in a browser.

There is an underlying HTTP 403 status code:

```
aiohttp.client_exceptions.ClientResponseError: 403, message='Forbidden', url=URL('https://cdn-lfs.huggingface.co/datasets/allenai/c4/8ef8d75b0e045dec4aa5123a671b4564466b0707086a7ed1ba8721626dfffbc9?response-content-disposition=attachment%3B+filename*%3DUTF-8''c4-train.00000-of-01024.json.gz%3B+filename%3D%22c4-train.00000-of-01024.json.gz%22%3B&response-content-type=application/gzip&Expires=1677483770&Policy=eyJTdGF0ZW1lbnQiOlt7IlJlc291cmNlIjoiaHR0cHM6Ly9jZG4tbGZzLmh1Z2dpbmdmYWNlLmNvL2RhdGFzZXRzL2FsbGVuYWkvYzQvOGVmOGQ3NWIwZTA0NWRlYzRhYTUxMjNhNjcxYjQ1NjQ0NjZiMDcwNzA4NmE3ZWQxYmE4NzIxNjI2ZGZmZmJjOT9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSomcmVzcG9uc2UtY29udGVudC10eXBlPWFwcGxpY2F0aW9uJTJGZ3ppcCIsIkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTY3NzQ4Mzc3MH19fV19&Signature=yjL3UeY72cf2xpnvPvD68eAYOEe2qtaUJV55sB-jnPskBJEMwpMJcBZvg2~GqXZdM3O-GWV-Z3CI~d4u5VCb4YZ-HlmOjr3VBYkvox2EKiXnBIhjMecf2UVUPtxhTa9kBVlWjqu4qKzB9gKXZF2Cwpp5ctLzapEaT2nnqF84RAL-rsqMA3I~M8vWWfivQsbBK63hMfgZqqKMgdWM0iKMaItveDl0ufQ29azMFmsR7qd8V7sU2Z-F1fAeohS8HpN9OOnClW34yi~YJ2AbgZJJBXA~qsylfVA0Qp7Q~yX~q4P8JF1vmJ2BjkiSbGrj3bAXOGugpOVU5msI52DT88yMdA__&Key-Pair-Id=KVTP0A1DKRTAX')
```

### Expected behavior

This should retrieve the first example from the C4 validation set. This worked a few days ago but stopped working now.

### Environment info

- `datasets` version: 2.9.0
- Platform: Linux-5.15.0-60-generic-x86_64-with-glibc2.31
- Python version: 3.9.16
- PyArrow version: 11.0.0
- Pandas version: 1.5.3


Use soundfile for mp3 decoding instead of torchaudio

_The documentation is not available anymore as the PR was closed or merged._

@mariosasko thank you for the review! do you have any idea why `test_hash_torch_tensor` fails on "ubuntu-latest deps-minimum"? I removed the `torchaudio<0.12.0` test dependency so it uses the latest `torch` now, might it be connected?

@polinaeterna The failure is due to `torch.from_numpy` not being picklable in newer versions of PyTorch. You can replace the current definition of `_save_tensor` in `utils/py_utils.py` with the following one to fix it:

```python
@pklregister(obj_type)
def _save_tensor(pickler, obj):
    # `torch.from_numpy` is not picklable in `torch>=1.11.0`
    def _create_tensor(np_array):
        return torch.from_numpy(np_array)

    dill_log(pickler, f"To: {obj}")
    args = (obj.detach().cpu().numpy(),)
    pickler.save_reduce(_create_tensor, args, obj=obj)
    dill_log(pickler, "# To")
    return
```

(doing a patch release now - please wait before merging ^^)

@mariosasko génial, merci!! i've integrated all your changes, can you pls take a look one more time?

Patch release is done (I did it from another branch than `main` anyway)

Show benchmarks

PyArrow==6.0.0

Show updated benchmarks!

### Benchmark: benchmark_array_xd.json

| metric | read_batch_formatted_as_numpy after write_array2d | read_batch_formatted_as_numpy after write_flattened_sequence | read_batch_formatted_as_numpy after write_nested_sequence | read_batch_unformated after write_array2d | read_batch_unformated after write_flattened_sequence | read_batch_unformated after write_nested_sequence | read_col_formatted_as_numpy after write_array2d | read_col_formatted_as_numpy after write_flattened_sequence | read_col_formatted_as_numpy after write_nested_sequence | read_col_unformated after write_array2d | read_col_unformated after write_flattened_sequence | read_col_unformated after write_nested_sequence | read_formatted_as_numpy after write_array2d | read_formatted_as_numpy after write_flattened_sequence | read_formatted_as_numpy after write_nested_sequence | read_unformated after write_array2d | read_unformated after write_flattened_sequence | read_unformated after write_nested_sequence | write_array2d | write_flattened_sequence | write_nested_sequence |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.010927 / 0.011353 (-0.000426) | 0.006232 / 0.011008 (-0.004776) | 0.119815 / 0.038508 (0.081307) | 0.034138 / 0.023109 (0.011029) | 0.349945 / 0.275898 (0.074047) | 0.404967 / 0.323480 (0.081487) | 0.008672 / 0.007986 (0.000687) | 0.005010 / 0.004328 (0.000681) | 0.091931 / 0.004250 (0.087680) | 0.042534 / 0.037052 (0.005482) | 0.374701 / 0.258489 (0.116212) | 0.401027 / 0.293841 (0.107186) | 0.053523 / 0.128546 (-0.075024) | 0.019704 / 0.075646 (-0.055942) | 0.384207 / 0.419271 (-0.035064) | 0.065350 / 0.043533 (0.021817) | 0.375074 / 0.255139 (0.119935) | 0.390458 / 0.283200 (0.107259) | 0.110549 / 0.141683 (-0.031134) | 1.719812 / 1.452155 (0.267657) | 1.748906 / 1.492716 (0.256190) |

### Benchmark: benchmark_getitem\_100B.json

| metric | get_batch_of\_1024\_random_rows | get_batch_of\_1024\_rows | get_first_row | get_last_row |
|--------|---|---|---|---|
| new / old (diff) | 0.210051 / 0.018006 (0.192045) | 0.546503 / 0.000490 (0.546013) | 0.004078 / 0.000200 (0.003878) | 0.000111 / 0.000054 (0.000056) |

### Benchmark: benchmark_indices_mapping.json

| metric | select | shard | shuffle | sort | train_test_split |
|--------|---|---|---|---|---|
| new / old (diff) | 0.030212 / 0.037411 (-0.007199) | 0.121845 / 0.014526 (0.107319) | 0.136309 / 0.176557 (-0.040247) | 0.204667 / 0.737135 (-0.532468) | 0.157327 / 0.296338 (-0.139012) |

### Benchmark: benchmark_iterating.json

| metric | read 5000 | read 50000 | read_batch 50000 10 | read_batch 50000 100 | read_batch 50000 1000 | read_formatted numpy 5000 | read_formatted pandas 5000 | read_formatted tensorflow 5000 | read_formatted torch 5000 | read_formatted_batch numpy 5000 10 | read_formatted_batch numpy 5000 1000 | shuffled read 5000 | shuffled read 50000 | shuffled read_batch 50000 10 | shuffled read_batch 50000 100 | shuffled read_batch 50000 1000 | shuffled read_formatted numpy 5000 | shuffled read_formatted_batch numpy 5000 10 | shuffled read_formatted_batch numpy 5000 1000 |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.672548 / 0.215209 (0.457339) | 6.239409 / 2.077655 (4.161754) | 2.462441 / 1.504120 (0.958322) | 2.063985 / 1.541195 (0.522791) | 2.098858 / 1.468490 (0.630368) | 1.262600 / 4.584777 (-3.322177) | 5.478462 / 3.745712 (1.732750) | 5.454672 / 5.269862 (0.184810) | 2.991866 / 4.565676 (-1.573810) | 0.153415 / 0.424275 (-0.270861) | 0.015061 / 0.007607 (0.007454) | 0.796115 / 0.226044 (0.570071) | 8.206858 / 2.268929 (5.937930) | 3.226395 / 55.444624 (-52.218229) | 2.503522 / 6.876477 (-4.372955) | 2.547489 / 2.142072 (0.405417) | 1.504776 / 4.805227 (-3.300451) | 0.256536 / 6.500664 (-6.244128) | 0.078543 / 0.075469 (0.003073) |

### Benchmark: benchmark_map_filter.json

| metric | filter | map fast-tokenizer batched | map identity | map identity batched | map no-op batched | map no-op batched numpy | map no-op batched pandas | map no-op batched pytorch | map no-op batched tensorflow |
|--------|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 1.591109 / 1.841788 (-0.250678) | 18.153317 / 8.074308 (10.079008) | 20.465684 / 10.191392 (10.274292) | 0.229808 / 0.680424 (-0.450616) | 0.045263 / 0.534201 (-0.488938) | 0.556760 / 0.579283 (-0.022524) | 0.614985 / 0.434364 (0.180622) | 0.635675 / 0.540337 (0.095337) | 0.729817 / 1.386936 (-0.657119) |

PyArrow==latest

Show updated benchmarks!

### Benchmark: benchmark_array_xd.json

| metric | read_batch_formatted_as_numpy after write_array2d | read_batch_formatted_as_numpy after write_flattened_sequence | read_batch_formatted_as_numpy after write_nested_sequence | read_batch_unformated after write_array2d | read_batch_unformated after write_flattened_sequence | read_batch_unformated after write_nested_sequence | read_col_formatted_as_numpy after write_array2d | read_col_formatted_as_numpy after write_flattened_sequence | read_col_formatted_as_numpy after write_nested_sequence | read_col_unformated after write_array2d | read_col_unformated after write_flattened_sequence | read_col_unformated after write_nested_sequence | read_formatted_as_numpy after write_array2d | read_formatted_as_numpy after write_flattened_sequence | read_formatted_as_numpy after write_nested_sequence | read_unformated after write_array2d | read_unformated after write_flattened_sequence | read_unformated after write_nested_sequence | write_array2d | write_flattened_sequence | write_nested_sequence |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.011247 / 0.011353 (-0.000106) | 0.006823 / 0.011008 (-0.004185) | 0.101989 / 0.038508 (0.063481) | 0.036077 / 0.023109 (0.012968) | 0.413469 / 0.275898 (0.137571) | 0.505560 / 0.323480 (0.182080) | 0.007506 / 0.007986 (-0.000480) | 0.006369 / 0.004328 (0.002040) | 0.099597 / 0.004250 (0.095346) | 0.058115 / 0.037052 (0.021063) | 0.414735 / 0.258489 (0.156246) | 0.466801 / 0.293841 (0.172960) | 0.064771 / 0.128546 (-0.063775) | 0.021100 / 0.075646 (-0.054546) | 0.135407 / 0.419271 (-0.283864) | 0.068784 / 0.043533 (0.025251) | 0.410467 / 0.255139 (0.155328) | 0.465993 / 0.283200 (0.182794) | 0.119404 / 0.141683 (-0.022279) | 1.767107 / 1.452155 (0.314952) | 1.938342 / 1.492716 (0.445626) |

### Benchmark: benchmark_getitem\_100B.json

| metric | get_batch_of\_1024\_random_rows | get_batch_of\_1024\_rows | get_first_row | get_last_row |
|--------|---|---|---|---|
| new / old (diff) | 0.227038 / 0.018006 (0.209032) | 0.511389 / 0.000490 (0.510899) | 0.006723 / 0.000200 (0.006523) | 0.000118 / 0.000054 (0.000064) |

### Benchmark: benchmark_indices_mapping.json

| metric | select | shard | shuffle | sort | train_test_split |
|--------|---|---|---|---|---|
| new / old (diff) | 0.033078 / 0.037411 (-0.004333) | 0.133159 / 0.014526 (0.118633) | 0.147928 / 0.176557 (-0.028629) | 0.214005 / 0.737135 (-0.523130) | 0.151655 / 0.296338 (-0.144683) |

### Benchmark: benchmark_iterating.json

| metric | read 5000 | read 50000 | read_batch 50000 10 | read_batch 50000 100 | read_batch 50000 1000 | read_formatted numpy 5000 | read_formatted pandas 5000 | read_formatted tensorflow 5000 | read_formatted torch 5000 | read_formatted_batch numpy 5000 10 | read_formatted_batch numpy 5000 1000 | shuffled read 5000 | shuffled read 50000 | shuffled read_batch 50000 10 | shuffled read_batch 50000 100 | shuffled read_batch 50000 1000 | shuffled read_formatted numpy 5000 | shuffled read_formatted_batch numpy 5000 10 | shuffled read_formatted_batch numpy 5000 1000 |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.634829 / 0.215209 (0.419620) | 6.578640 / 2.077655 (4.500985) | 2.673598 / 1.504120 (1.169478) | 2.338671 / 1.541195 (0.797476) | 2.389104 / 1.468490 (0.920614) | 1.274938 / 4.584777 (-3.309839) | 5.746524 / 3.745712 (2.000812) | 5.992084 / 5.269862 (0.722222) | 3.092090 / 4.565676 (-1.473587) | 0.150375 / 0.424275 (-0.273900) | 0.015470 / 0.007607 (0.007863) | 0.792962 / 0.226044 (0.566918) | 8.057491 / 2.268929 (5.788563) | 3.483966 / 55.444624 (-51.960659) | 2.715038 / 6.876477 (-4.161438) | 2.747186 / 2.142072 (0.605114) | 1.532951 / 4.805227 (-3.272276) | 0.262214 / 6.500664 (-6.238450) | 0.081308 / 0.075469 (0.005839) |

### Benchmark: benchmark_map_filter.json

| metric | filter | map fast-tokenizer batched | map identity | map identity batched | map no-op batched | map no-op batched numpy | map no-op batched pandas | map no-op batched pytorch | map no-op batched tensorflow |
|--------|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 1.698448 / 1.841788 (-0.143340) | 18.590002 / 8.074308 (10.515694) | 20.584508 / 10.191392 (10.393116) | 0.227237 / 0.680424 (-0.453187) | 0.028445 / 0.534201 (-0.505756) | 0.527874 / 0.579283 (-0.051409) | 0.602844 / 0.434364 (0.168480) | 0.672948 / 0.540337 (0.132611) | 0.788103 / 1.386936 (-0.598833) |

![](https://cml.dev/watermark.png#f96547708a889c09ca8a02ed7aadd8c5690503c5 "CML watermark")

I've removed `torchaudio` completely and switched to use `soundfile` for everything. With the new version of `soundfile` package this should work smoothly because the `libsndfile` C library is bundled, in Linux wheels too.

Let me know if you think it's too harsh and we should continue to support `torchaudio` decoding.

I decided that we can drop it completely because:
1. it's always something wrong with `torchaudio` (for example recently https://github.com/huggingface/datasets/issues/5488 )
2. the results of mp3 decoding are different depending on `torchaudio` version
3. `soundfile` is slightly faster then the latest `torchaudio`
4. anyway users can pass any custom decoding function with any library they want if needed (worth putting a snippet in the docs).

cc @sanchit-gandhi @vaibhavad


Datasets 2.10.0 does not reuse the dataset cache

### Describe the bug

download_mode="reuse_dataset_if_exists" will always consider that a dataset doesn't exist.

Specifically, upon losing an internet connection trying to load a dataset for a second time in ten seconds, a connection error results, showing a breakpoint of:

```
File ~/jupyterlab/.direnv/python-3.9.6/lib/python3.9/site-packages/datasets/load.py:1174, in dataset_module_factory(path, revision, download_config, download_mode, dynamic_modules_path, data_dir, data_files, **download_kwargs)
   1165 except Exception as e:  # noqa: catch any exception of hf_hub and consider that the dataset doesn't exist
   1166     if isinstance(
   1167         e,
   1168         (
   (...)
   1172         ),
   1173     ):
-> 1174         raise ConnectionError(f"Couldn't reach '{path}' on the Hub ({type(e).__name__})")
   1175     elif "404" in str(e):
   1176         msg = f"Dataset '{path}' doesn't exist on the Hub"

ConnectionError: Couldn't reach 'lsb/tenk' on the Hub (ConnectionError)
```

This has been around since at least v2.0.

### Steps to reproduce the bug

```
from datasets import load_dataset
import numpy as np

tenk = load_dataset("lsb/tenk") # ten thousand integers
print(np.average(tenk['train']['a'])) # prints 4999.5

### now disconnect your internet

tenk_too = load_dataset("lsb/tenk", download_mode="reuse_dataset_if_exists")
# Raises ConnectionError: Couldn't reach 'lsb/tenk' on the Hub (ConnectionError)
```

### Expected behavior

I expected that I would be able to reuse the dataset I just downloaded.

### Environment info

- `datasets` version: 2.10.0
- Platform: macOS-13.1-arm64-arm-64bit
- Python version: 3.9.6
- PyArrow version: 7.0.0
- Pandas version: 1.5.2


load_dataset fails for JSON in windows

Hi!

You need to pass an input json file explicitly as `data_files` to `load_dataset` to avoid this error:
```python
 ds = load_dataset("json", data_files=args.input_json)
```

### Describe the bug

Steps:
1. Created a dataset in a Linux VM and created a small sample using dataset.to_json() method.
2. Downloaded the JSON file to my local Windows machine for working and saved in say - r"C:\Users\name\file.json"
3. I am reading the file in my local PyCharm - the location of python file is different than the location of the JSON.
4. When I read using load_dataset("json",args.input_json), it throws and error from builder.py.
raise InvalidConfigName(
                    f"Bad characters from black list '{invalid_windows_characters}' found in '{self.name}'. "
                    f"They could create issues when creating a directory for this config on Windows filesystem."
6. When I bring the data to the current directory, it works fine.

### Steps to reproduce the bug

Steps:
1. Created a dataset in a Linux VM and created a small sample using dataset.to_json() method.
2. Downloaded the JSON file to my local Windows machine for working and saved in say - r"C:\Users\name\file.json"
3. I am reading the file in my local PyCharm - the location of python file is different than the location of the JSON.
4. When I read using load_dataset("json",args.input_json), it throws and error from builder.py.
raise InvalidConfigName(
                    f"Bad characters from black list '{invalid_windows_characters}' found in '{self.name}'. "
                    f"They could create issues when creating a directory for this config on Windows filesystem."
6. When I bring the data to the current directory, it works fine.

### Expected behavior

Should be able to read from a path different than current directory in Windows machine.

### Environment info

datasets version: 2.3.1
python version: 3.8
Windows OS


load_dataset gives FileNotFoundError on imagenet-1k if license is not accepted on the hub

Hi, thanks for the feedback! Would it help to add a tip or note saying the dataset is gated and you need to accept the license before downloading it?

The error is now more informative:
```
FileNotFoundError: Couldn't find a dataset script at /content/imagenet-1k/imagenet-1k.py or any data file in the same directory. Couldn't find 'imagenet-1k' on the Hugging Face Hub either: FileNotFoundError: Dataset 'imagenet-1k' doesn't exist on the Hub. If the repo is private or gated, make sure to log in with `huggingface-cli login`.
```

### Describe the bug

When calling ```load_dataset('imagenet-1k')``` FileNotFoundError is raised, if not logged in and if logged in with huggingface-cli but not having accepted the licence on the hub. There is no error once accepting.

### Steps to reproduce the bug

```
from datasets import load_dataset

imagenet = load_dataset("imagenet-1k", split="train", streaming=True)

FileNotFoundError: Couldn't find a dataset script at /content/imagenet-1k/imagenet-1k.py or any data file in the same directory. Couldn't find 'imagenet-1k' on the Hugging Face Hub either: FileNotFoundError: Dataset 'imagenet-1k' doesn't exist on the Hub
```
tested on a colab notebook.

### Expected behavior

I would expect a specific error indicating that I have to login then accept the dataset licence.

I find this bug very relevant as this code is on a guide on the [Huggingface documentation for Datasets](https://huggingface.co/docs/datasets/about_mapstyle_vs_iterable)

### Environment info

google colab cpu-only instance


pass the dataset features to the IterableDataset.from_generator function

Show benchmarks

PyArrow==6.0.0

Show updated benchmarks!

### Benchmark: benchmark_array_xd.json

| metric | read_batch_formatted_as_numpy after write_array2d | read_batch_formatted_as_numpy after write_flattened_sequence | read_batch_formatted_as_numpy after write_nested_sequence | read_batch_unformated after write_array2d | read_batch_unformated after write_flattened_sequence | read_batch_unformated after write_nested_sequence | read_col_formatted_as_numpy after write_array2d | read_col_formatted_as_numpy after write_flattened_sequence | read_col_formatted_as_numpy after write_nested_sequence | read_col_unformated after write_array2d | read_col_unformated after write_flattened_sequence | read_col_unformated after write_nested_sequence | read_formatted_as_numpy after write_array2d | read_formatted_as_numpy after write_flattened_sequence | read_formatted_as_numpy after write_nested_sequence | read_unformated after write_array2d | read_unformated after write_flattened_sequence | read_unformated after write_nested_sequence | write_array2d | write_flattened_sequence | write_nested_sequence |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.008753 / 0.011353 (-0.002600) | 0.004877 / 0.011008 (-0.006131) | 0.098320 / 0.038508 (0.059812) | 0.034123 / 0.023109 (0.011014) | 0.289539 / 0.275898 (0.013641) | 0.323584 / 0.323480 (0.000104) | 0.007455 / 0.007986 (-0.000531) | 0.004763 / 0.004328 (0.000434) | 0.074350 / 0.004250 (0.070100) | 0.039018 / 0.037052 (0.001966) | 0.294319 / 0.258489 (0.035830) | 0.348686 / 0.293841 (0.054845) | 0.037814 / 0.128546 (-0.090732) | 0.011808 / 0.075646 (-0.063838) | 0.333808 / 0.419271 (-0.085464) | 0.047758 / 0.043533 (0.004225) | 0.298533 / 0.255139 (0.043394) | 0.320790 / 0.283200 (0.037590) | 0.095909 / 0.141683 (-0.045774) | 1.434422 / 1.452155 (-0.017732) | 1.509703 / 1.492716 (0.016987) |

### Benchmark: benchmark_getitem\_100B.json

| metric | get_batch_of\_1024\_random_rows | get_batch_of\_1024\_rows | get_first_row | get_last_row |
|--------|---|---|---|---|
| new / old (diff) | 0.201728 / 0.018006 (0.183722) | 0.432243 / 0.000490 (0.431753) | 0.002760 / 0.000200 (0.002560) | 0.000080 / 0.000054 (0.000026) |

### Benchmark: benchmark_indices_mapping.json

| metric | select | shard | shuffle | sort | train_test_split |
|--------|---|---|---|---|---|
| new / old (diff) | 0.026090 / 0.037411 (-0.011321) | 0.105914 / 0.014526 (0.091388) | 0.115869 / 0.176557 (-0.060688) | 0.178291 / 0.737135 (-0.558844) | 0.121435 / 0.296338 (-0.174904) |

### Benchmark: benchmark_iterating.json

| metric | read 5000 | read 50000 | read_batch 50000 10 | read_batch 50000 100 | read_batch 50000 1000 | read_formatted numpy 5000 | read_formatted pandas 5000 | read_formatted tensorflow 5000 | read_formatted torch 5000 | read_formatted_batch numpy 5000 10 | read_formatted_batch numpy 5000 1000 | shuffled read 5000 | shuffled read 50000 | shuffled read_batch 50000 10 | shuffled read_batch 50000 100 | shuffled read_batch 50000 1000 | shuffled read_formatted numpy 5000 | shuffled read_formatted_batch numpy 5000 10 | shuffled read_formatted_batch numpy 5000 1000 |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.402304 / 0.215209 (0.187095) | 3.995183 / 2.077655 (1.917529) | 1.794548 / 1.504120 (0.290428) | 1.603034 / 1.541195 (0.061839) | 1.643836 / 1.468490 (0.175346) | 0.694934 / 4.584777 (-3.889843) | 3.695128 / 3.745712 (-0.050584) | 2.018582 / 5.269862 (-3.251279) | 1.294315 / 4.565676 (-3.271362) | 0.085346 / 0.424275 (-0.338929) | 0.012201 / 0.007607 (0.004594) | 0.510057 / 0.226044 (0.284012) | 5.123404 / 2.268929 (2.854476) | 2.319089 / 55.444624 (-53.125535) | 1.930935 / 6.876477 (-4.945542) | 1.939700 / 2.142072 (-0.202372) | 0.848282 / 4.805227 (-3.956945) | 0.165561 / 6.500664 (-6.335103) | 0.062028 / 0.075469 (-0.013441) |

### Benchmark: benchmark_map_filter.json

| metric | filter | map fast-tokenizer batched | map identity | map identity batched | map no-op batched | map no-op batched numpy | map no-op batched pandas | map no-op batched pytorch | map no-op batched tensorflow |
|--------|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 1.220576 / 1.841788 (-0.621212) | 14.413853 / 8.074308 (6.339544) | 14.027156 / 10.191392 (3.835764) | 0.170109 / 0.680424 (-0.510315) | 0.029412 / 0.534201 (-0.504789) | 0.443898 / 0.579283 (-0.135386) | 0.433059 / 0.434364 (-0.001305) | 0.533465 / 0.540337 (-0.006872) | 0.626562 / 1.386936 (-0.760374) |

PyArrow==latest

Show updated benchmarks!

### Benchmark: benchmark_array_xd.json

| metric | read_batch_formatted_as_numpy after write_array2d | read_batch_formatted_as_numpy after write_flattened_sequence | read_batch_formatted_as_numpy after write_nested_sequence | read_batch_unformated after write_array2d | read_batch_unformated after write_flattened_sequence | read_batch_unformated after write_nested_sequence | read_col_formatted_as_numpy after write_array2d | read_col_formatted_as_numpy after write_flattened_sequence | read_col_formatted_as_numpy after write_nested_sequence | read_col_unformated after write_array2d | read_col_unformated after write_flattened_sequence | read_col_unformated after write_nested_sequence | read_formatted_as_numpy after write_array2d | read_formatted_as_numpy after write_flattened_sequence | read_formatted_as_numpy after write_nested_sequence | read_unformated after write_array2d | read_unformated after write_flattened_sequence | read_unformated after write_nested_sequence | write_array2d | write_flattened_sequence | write_nested_sequence |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.007148 / 0.011353 (-0.004205) | 0.005019 / 0.011008 (-0.005989) | 0.073132 / 0.038508 (0.034624) | 0.032763 / 0.023109 (0.009654) | 0.329309 / 0.275898 (0.053411) | 0.361658 / 0.323480 (0.038178) | 0.005683 / 0.007986 (-0.002302) | 0.003793 / 0.004328 (-0.000536) | 0.071858 / 0.004250 (0.067608) | 0.045160 / 0.037052 (0.008107) | 0.335852 / 0.258489 (0.077363) | 0.384274 / 0.293841 (0.090433) | 0.036647 / 0.128546 (-0.091899) | 0.012217 / 0.075646 (-0.063430) | 0.086265 / 0.419271 (-0.333007) | 0.049223 / 0.043533 (0.005690) | 0.331460 / 0.255139 (0.076321) | 0.353175 / 0.283200 (0.069975) | 0.102214 / 0.141683 (-0.039469) | 1.491451 / 1.452155 (0.039296) | 1.553702 / 1.492716 (0.060985) |

### Benchmark: benchmark_getitem\_100B.json

| metric | get_batch_of\_1024\_random_rows | get_batch_of\_1024\_rows | get_first_row | get_last_row |
|--------|---|---|---|---|
| new / old (diff) | 0.222972 / 0.018006 (0.204966) | 0.432862 / 0.000490 (0.432372) | 0.000421 / 0.000200 (0.000221) | 0.000059 / 0.000054 (0.000005) |

### Benchmark: benchmark_indices_mapping.json

| metric | select | shard | shuffle | sort | train_test_split |
|--------|---|---|---|---|---|
| new / old (diff) | 0.028401 / 0.037411 (-0.009010) | 0.109331 / 0.014526 (0.094805) | 0.119246 / 0.176557 (-0.057311) | 0.187997 / 0.737135 (-0.549138) | 0.124212 / 0.296338 (-0.172127) |

### Benchmark: benchmark_iterating.json

| metric | read 5000 | read 50000 | read_batch 50000 10 | read_batch 50000 100 | read_batch 50000 1000 | read_formatted numpy 5000 | read_formatted pandas 5000 | read_formatted tensorflow 5000 | read_formatted torch 5000 | read_formatted_batch numpy 5000 10 | read_formatted_batch numpy 5000 1000 | shuffled read 5000 | shuffled read 50000 | shuffled read_batch 50000 10 | shuffled read_batch 50000 100 | shuffled read_batch 50000 1000 | shuffled read_formatted numpy 5000 | shuffled read_formatted_batch numpy 5000 10 | shuffled read_formatted_batch numpy 5000 1000 |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.427240 / 0.215209 (0.212031) | 4.271619 / 2.077655 (2.193964) | 2.104948 / 1.504120 (0.600828) | 1.910624 / 1.541195 (0.369430) | 1.943812 / 1.468490 (0.475322) | 0.711466 / 4.584777 (-3.873311) | 3.778987 / 3.745712 (0.033275) | 2.976258 / 5.269862 (-2.293604) | 1.807591 / 4.565676 (-2.758086) | 0.088286 / 0.424275 (-0.335989) | 0.012461 / 0.007607 (0.004854) | 0.527554 / 0.226044 (0.301509) | 5.279461 / 2.268929 (3.010532) | 2.517911 / 55.444624 (-52.926713) | 2.176557 / 6.876477 (-4.699920) | 2.205322 / 2.142072 (0.063249) | 0.855012 / 4.805227 (-3.950215) | 0.170007 / 6.500664 (-6.330658) | 0.065273 / 0.075469 (-0.010196) |

### Benchmark: benchmark_map_filter.json

| metric | filter | map fast-tokenizer batched | map identity | map identity batched | map no-op batched | map no-op batched numpy | map no-op batched pandas | map no-op batched pytorch | map no-op batched tensorflow |
|--------|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 1.282785 / 1.841788 (-0.559003) | 14.819500 / 8.074308 (6.745192) | 13.282211 / 10.191392 (3.090819) | 0.161804 / 0.680424 (-0.518620) | 0.017615 / 0.534201 (-0.516586) | 0.420159 / 0.579283 (-0.159124) | 0.441304 / 0.434364 (0.006940) | 0.531704 / 0.540337 (-0.008634) | 0.627477 / 1.386936 (-0.759459) |

![](https://cml.dev/watermark.png#b91070b9c09673e2e148eec458036ab6a62ac042 "CML watermark")

Hmm I think we need to add more tests. Not sure what would happen with :
- decodable features that may end up decoded twice
- formatted datasets

I'd be in favor of reverting this until we checked those


dataset.to_iterable_dataset() loses useful info like dataset features

enhancement

New feature or request

good first issue

Good for newcomers

Hi ! Oh good catch. I think the features should be passed to `IterableDataset.from_generator()` in `to_iterable_dataset()` indeed.

Setting this as a good first issue if someone would like to contribute, otherwise we can take care of it :)

seems like the feature parameter is missing from `return IterableDataset.from_generator(Dataset._iter_shards, gen_kwargs={"shards": shards})`  hence it defaults to None.

### Describe the bug

Hello,
I like the new `to_iterable_dataset` feature but I noticed something that seems to be missing.

When using `to_iterable_dataset` to transform your map style dataset into iterable dataset, you lose valuable metadata like the features.

These metadata are useful if you want to interleave iterable datasets, cast columns etc.

### Steps to reproduce the bug

```python
dataset = load_dataset("lhoestq/demo1")["train"]
print(dataset.features)
# {'id': Value(dtype='string', id=None), 'package_name': Value(dtype='string', id=None), 'review': Value(dtype='string', id=None), 'date': Value(dtype='string', id=None), 'star': Value(dtype='int64', id=None), 'version_id': Value(dtype='int64', id=None)}
dataset = dataset.to_iterable_dataset()
print(dataset.features)
# None
```

### Expected behavior

Keep the relevant information

### Environment info

datasets==2.10.0


Directly reading parquet files in a s3 bucket from the load_dataset method

duplicate

This issue or pull request already exists

enhancement

New feature or request

Hi ! I think is in the scope of this other issue: to https://github.com/huggingface/datasets/issues/5281

### Feature request

Right now, we have to read the get the parquet file to the local storage. So having ability to read given the bucket directly address would be benificial

### Motivation

In a production set up, this feature can help us a lot. So we do not need move training datafiles in between storage.

### Your contribution

I am willing to help if there's anyway.


Add writer_batch_size for ArrowBasedBuilder

Show benchmarks

PyArrow==6.0.0

Show updated benchmarks!

### Benchmark: benchmark_array_xd.json

| metric | read_batch_formatted_as_numpy after write_array2d | read_batch_formatted_as_numpy after write_flattened_sequence | read_batch_formatted_as_numpy after write_nested_sequence | read_batch_unformated after write_array2d | read_batch_unformated after write_flattened_sequence | read_batch_unformated after write_nested_sequence | read_col_formatted_as_numpy after write_array2d | read_col_formatted_as_numpy after write_flattened_sequence | read_col_formatted_as_numpy after write_nested_sequence | read_col_unformated after write_array2d | read_col_unformated after write_flattened_sequence | read_col_unformated after write_nested_sequence | read_formatted_as_numpy after write_array2d | read_formatted_as_numpy after write_flattened_sequence | read_formatted_as_numpy after write_nested_sequence | read_unformated after write_array2d | read_unformated after write_flattened_sequence | read_unformated after write_nested_sequence | write_array2d | write_flattened_sequence | write_nested_sequence |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.008745 / 0.011353 (-0.002608) | 0.004651 / 0.011008 (-0.006357) | 0.099678 / 0.038508 (0.061170) | 0.029441 / 0.023109 (0.006332) | 0.300314 / 0.275898 (0.024416) | 0.342022 / 0.323480 (0.018542) | 0.006965 / 0.007986 (-0.001021) | 0.003382 / 0.004328 (-0.000946) | 0.078195 / 0.004250 (0.073945) | 0.033308 / 0.037052 (-0.003744) | 0.300857 / 0.258489 (0.042368) | 0.356763 / 0.293841 (0.062922) | 0.033919 / 0.128546 (-0.094627) | 0.011436 / 0.075646 (-0.064210) | 0.319581 / 0.419271 (-0.099691) | 0.041303 / 0.043533 (-0.002229) | 0.299387 / 0.255139 (0.044248) | 0.327783 / 0.283200 (0.044583) | 0.087210 / 0.141683 (-0.054473) | 1.498757 / 1.452155 (0.046603) | 1.560417 / 1.492716 (0.067701) |

### Benchmark: benchmark_getitem\_100B.json

| metric | get_batch_of\_1024\_random_rows | get_batch_of\_1024\_rows | get_first_row | get_last_row |
|--------|---|---|---|---|
| new / old (diff) | 0.191806 / 0.018006 (0.173800) | 0.407044 / 0.000490 (0.406554) | 0.005116 / 0.000200 (0.004916) | 0.000073 / 0.000054 (0.000019) |

### Benchmark: benchmark_indices_mapping.json

| metric | select | shard | shuffle | sort | train_test_split |
|--------|---|---|---|---|---|
| new / old (diff) | 0.023760 / 0.037411 (-0.013652) | 0.096844 / 0.014526 (0.082318) | 0.104710 / 0.176557 (-0.071847) | 0.168161 / 0.737135 (-0.568974) | 0.107808 / 0.296338 (-0.188531) |

### Benchmark: benchmark_iterating.json

| metric | read 5000 | read 50000 | read_batch 50000 10 | read_batch 50000 100 | read_batch 50000 1000 | read_formatted numpy 5000 | read_formatted pandas 5000 | read_formatted tensorflow 5000 | read_formatted torch 5000 | read_formatted_batch numpy 5000 10 | read_formatted_batch numpy 5000 1000 | shuffled read 5000 | shuffled read 50000 | shuffled read_batch 50000 10 | shuffled read_batch 50000 100 | shuffled read_batch 50000 1000 | shuffled read_formatted numpy 5000 | shuffled read_formatted_batch numpy 5000 10 | shuffled read_formatted_batch numpy 5000 1000 |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.417707 / 0.215209 (0.202498) | 4.155952 / 2.077655 (2.078297) | 1.864934 / 1.504120 (0.360814) | 1.654925 / 1.541195 (0.113730) | 1.731341 / 1.468490 (0.262851) | 0.692014 / 4.584777 (-3.892763) | 3.407318 / 3.745712 (-0.338394) | 3.394791 / 5.269862 (-1.875071) | 1.650429 / 4.565676 (-2.915247) | 0.082177 / 0.424275 (-0.342098) | 0.012463 / 0.007607 (0.004856) | 0.523681 / 0.226044 (0.297637) | 5.249426 / 2.268929 (2.980498) | 2.327443 / 55.444624 (-53.117181) | 1.982160 / 6.876477 (-4.894317) | 2.019822 / 2.142072 (-0.122250) | 0.804820 / 4.805227 (-4.000408) | 0.148423 / 6.500664 (-6.352241) | 0.064938 / 0.075469 (-0.010531) |

### Benchmark: benchmark_map_filter.json

| metric | filter | map fast-tokenizer batched | map identity | map identity batched | map no-op batched | map no-op batched numpy | map no-op batched pandas | map no-op batched pytorch | map no-op batched tensorflow |
|--------|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 1.225722 / 1.841788 (-0.616066) | 13.774257 / 8.074308 (5.699949) | 14.090298 / 10.191392 (3.898906) | 0.152489 / 0.680424 (-0.527935) | 0.028595 / 0.534201 (-0.505606) | 0.399011 / 0.579283 (-0.180272) | 0.399546 / 0.434364 (-0.034818) | 0.485513 / 0.540337 (-0.054824) | 0.564055 / 1.386936 (-0.822881) |

PyArrow==latest

Show updated benchmarks!

### Benchmark: benchmark_array_xd.json

| metric | read_batch_formatted_as_numpy after write_array2d | read_batch_formatted_as_numpy after write_flattened_sequence | read_batch_formatted_as_numpy after write_nested_sequence | read_batch_unformated after write_array2d | read_batch_unformated after write_flattened_sequence | read_batch_unformated after write_nested_sequence | read_col_formatted_as_numpy after write_array2d | read_col_formatted_as_numpy after write_flattened_sequence | read_col_formatted_as_numpy after write_nested_sequence | read_col_unformated after write_array2d | read_col_unformated after write_flattened_sequence | read_col_unformated after write_nested_sequence | read_formatted_as_numpy after write_array2d | read_formatted_as_numpy after write_flattened_sequence | read_formatted_as_numpy after write_nested_sequence | read_unformated after write_array2d | read_unformated after write_flattened_sequence | read_unformated after write_nested_sequence | write_array2d | write_flattened_sequence | write_nested_sequence |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.006891 / 0.011353 (-0.004462) | 0.004557 / 0.011008 (-0.006451) | 0.077868 / 0.038508 (0.039360) | 0.028767 / 0.023109 (0.005657) | 0.344127 / 0.275898 (0.068229) | 0.377097 / 0.323480 (0.053617) | 0.005119 / 0.007986 (-0.002866) | 0.003547 / 0.004328 (-0.000782) | 0.077047 / 0.004250 (0.072796) | 0.043037 / 0.037052 (0.005984) | 0.341900 / 0.258489 (0.083410) | 0.384570 / 0.293841 (0.090729) | 0.032606 / 0.128546 (-0.095940) | 0.011752 / 0.075646 (-0.063894) | 0.086731 / 0.419271 (-0.332540) | 0.045459 / 0.043533 (0.001926) | 0.339308 / 0.255139 (0.084169) | 0.370498 / 0.283200 (0.087298) | 0.096237 / 0.141683 (-0.045446) | 1.499253 / 1.452155 (0.047098) | 1.583871 / 1.492716 (0.091154) |

### Benchmark: benchmark_getitem\_100B.json

| metric | get_batch_of\_1024\_random_rows | get_batch_of\_1024\_rows | get_first_row | get_last_row |
|--------|---|---|---|---|
| new / old (diff) | 0.245471 / 0.018006 (0.227465) | 0.408750 / 0.000490 (0.408260) | 0.008992 / 0.000200 (0.008792) | 0.000249 / 0.000054 (0.000194) |

### Benchmark: benchmark_indices_mapping.json

| metric | select | shard | shuffle | sort | train_test_split |
|--------|---|---|---|---|---|
| new / old (diff) | 0.025508 / 0.037411 (-0.011903) | 0.102103 / 0.014526 (0.087578) | 0.109247 / 0.176557 (-0.067310) | 0.176369 / 0.737135 (-0.560766) | 0.111241 / 0.296338 (-0.185097) |

### Benchmark: benchmark_iterating.json

| metric | read 5000 | read 50000 | read_batch 50000 10 | read_batch 50000 100 | read_batch 50000 1000 | read_formatted numpy 5000 | read_formatted pandas 5000 | read_formatted tensorflow 5000 | read_formatted torch 5000 | read_formatted_batch numpy 5000 10 | read_formatted_batch numpy 5000 1000 | shuffled read 5000 | shuffled read 50000 | shuffled read_batch 50000 10 | shuffled read_batch 50000 100 | shuffled read_batch 50000 1000 | shuffled read_formatted numpy 5000 | shuffled read_formatted_batch numpy 5000 10 | shuffled read_formatted_batch numpy 5000 1000 |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.437209 / 0.215209 (0.222000) | 4.354386 / 2.077655 (2.276731) | 2.064008 / 1.504120 (0.559888) | 1.855518 / 1.541195 (0.314323) | 1.931647 / 1.468490 (0.463157) | 0.704913 / 4.584777 (-3.879864) | 3.397913 / 3.745712 (-0.347800) | 1.871524 / 5.269862 (-3.398338) | 1.176492 / 4.565676 (-3.389185) | 0.083976 / 0.424275 (-0.340299) | 0.012806 / 0.007607 (0.005199) | 0.539138 / 0.226044 (0.313094) | 5.401493 / 2.268929 (3.132564) | 2.539185 / 55.444624 (-52.905440) | 2.186445 / 6.876477 (-4.690031) | 2.222170 / 2.142072 (0.080097) | 0.815641 / 4.805227 (-3.989586) | 0.153033 / 6.500664 (-6.347631) | 0.069168 / 0.075469 (-0.006301) |

### Benchmark: benchmark_map_filter.json

| metric | filter | map fast-tokenizer batched | map identity | map identity batched | map no-op batched | map no-op batched numpy | map no-op batched pandas | map no-op batched pytorch | map no-op batched tensorflow |
|--------|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 1.283530 / 1.841788 (-0.558258) | 14.075831 / 8.074308 (6.001523) | 13.649137 / 10.191392 (3.457745) | 0.127517 / 0.680424 (-0.552907) | 0.016619 / 0.534201 (-0.517582) | 0.377400 / 0.579283 (-0.201883) | 0.410796 / 0.434364 (-0.023568) | 0.463996 / 0.540337 (-0.076342) | 0.551867 / 1.386936 (-0.835069) |

![](https://cml.dev/watermark.png#1135285d80ff9cd65fc51905f08343b4d7c2fa9c "CML watermark")

Show benchmarks

PyArrow==6.0.0

Show updated benchmarks!

### Benchmark: benchmark_array_xd.json

| metric | read_batch_formatted_as_numpy after write_array2d | read_batch_formatted_as_numpy after write_flattened_sequence | read_batch_formatted_as_numpy after write_nested_sequence | read_batch_unformated after write_array2d | read_batch_unformated after write_flattened_sequence | read_batch_unformated after write_nested_sequence | read_col_formatted_as_numpy after write_array2d | read_col_formatted_as_numpy after write_flattened_sequence | read_col_formatted_as_numpy after write_nested_sequence | read_col_unformated after write_array2d | read_col_unformated after write_flattened_sequence | read_col_unformated after write_nested_sequence | read_formatted_as_numpy after write_array2d | read_formatted_as_numpy after write_flattened_sequence | read_formatted_as_numpy after write_nested_sequence | read_unformated after write_array2d | read_unformated after write_flattened_sequence | read_unformated after write_nested_sequence | write_array2d | write_flattened_sequence | write_nested_sequence |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.009161 / 0.011353 (-0.002192) | 0.004987 / 0.011008 (-0.006022) | 0.098553 / 0.038508 (0.060045) | 0.034326 / 0.023109 (0.011216) | 0.295325 / 0.275898 (0.019427) | 0.326361 / 0.323480 (0.002881) | 0.007827 / 0.007986 (-0.000159) | 0.004933 / 0.004328 (0.000604) | 0.074236 / 0.004250 (0.069986) | 0.040410 / 0.037052 (0.003357) | 0.295644 / 0.258489 (0.037155) | 0.355050 / 0.293841 (0.061209) | 0.038384 / 0.128546 (-0.090162) | 0.011845 / 0.075646 (-0.063801) | 0.340678 / 0.419271 (-0.078594) | 0.047615 / 0.043533 (0.004082) | 0.292429 / 0.255139 (0.037290) | 0.312610 / 0.283200 (0.029410) | 0.100106 / 0.141683 (-0.041577) | 1.446186 / 1.452155 (-0.005969) | 1.534763 / 1.492716 (0.042046) |

### Benchmark: benchmark_getitem\_100B.json

| metric | get_batch_of\_1024\_random_rows | get_batch_of\_1024\_rows | get_first_row | get_last_row |
|--------|---|---|---|---|
| new / old (diff) | 0.213667 / 0.018006 (0.195661) | 0.447310 / 0.000490 (0.446820) | 0.000402 / 0.000200 (0.000202) | 0.000056 / 0.000054 (0.000002) |

### Benchmark: benchmark_indices_mapping.json

| metric | select | shard | shuffle | sort | train_test_split |
|--------|---|---|---|---|---|
| new / old (diff) | 0.027604 / 0.037411 (-0.009807) | 0.112785 / 0.014526 (0.098259) | 0.119450 / 0.176557 (-0.057106) | 0.185728 / 0.737135 (-0.551407) | 0.122860 / 0.296338 (-0.173478) |

### Benchmark: benchmark_iterating.json

| metric | read 5000 | read 50000 | read_batch 50000 10 | read_batch 50000 100 | read_batch 50000 1000 | read_formatted numpy 5000 | read_formatted pandas 5000 | read_formatted tensorflow 5000 | read_formatted torch 5000 | read_formatted_batch numpy 5000 10 | read_formatted_batch numpy 5000 1000 | shuffled read 5000 | shuffled read 50000 | shuffled read_batch 50000 10 | shuffled read_batch 50000 100 | shuffled read_batch 50000 1000 | shuffled read_formatted numpy 5000 | shuffled read_formatted_batch numpy 5000 10 | shuffled read_formatted_batch numpy 5000 1000 |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.399162 / 0.215209 (0.183953) | 3.992701 / 2.077655 (1.915046) | 1.773881 / 1.504120 (0.269761) | 1.589842 / 1.541195 (0.048647) | 1.670065 / 1.468490 (0.201575) | 0.707669 / 4.584777 (-3.877107) | 3.719657 / 3.745712 (-0.026055) | 2.139629 / 5.269862 (-3.130232) | 1.467224 / 4.565676 (-3.098453) | 0.086033 / 0.424275 (-0.338242) | 0.012151 / 0.007607 (0.004544) | 0.519700 / 0.226044 (0.293656) | 5.150254 / 2.268929 (2.881325) | 2.305076 / 55.444624 (-53.139548) | 1.927914 / 6.876477 (-4.948563) | 1.999461 / 2.142072 (-0.142612) | 0.851819 / 4.805227 (-3.953408) | 0.165513 / 6.500664 (-6.335151) | 0.061898 / 0.075469 (-0.013571) |

### Benchmark: benchmark_map_filter.json

| metric | filter | map fast-tokenizer batched | map identity | map identity batched | map no-op batched | map no-op batched numpy | map no-op batched pandas | map no-op batched pytorch | map no-op batched tensorflow |
|--------|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 1.226251 / 1.841788 (-0.615536) | 14.990253 / 8.074308 (6.915945) | 14.658720 / 10.191392 (4.467328) | 0.191665 / 0.680424 (-0.488759) | 0.028768 / 0.534201 (-0.505433) | 0.443907 / 0.579283 (-0.135376) | 0.455183 / 0.434364 (0.020819) | 0.552760 / 0.540337 (0.012422) | 0.653927 / 1.386936 (-0.733009) |

PyArrow==latest

Show updated benchmarks!

### Benchmark: benchmark_array_xd.json

| metric | read_batch_formatted_as_numpy after write_array2d | read_batch_formatted_as_numpy after write_flattened_sequence | read_batch_formatted_as_numpy after write_nested_sequence | read_batch_unformated after write_array2d | read_batch_unformated after write_flattened_sequence | read_batch_unformated after write_nested_sequence | read_col_formatted_as_numpy after write_array2d | read_col_formatted_as_numpy after write_flattened_sequence | read_col_formatted_as_numpy after write_nested_sequence | read_col_unformated after write_array2d | read_col_unformated after write_flattened_sequence | read_col_unformated after write_nested_sequence | read_formatted_as_numpy after write_array2d | read_formatted_as_numpy after write_flattened_sequence | read_formatted_as_numpy after write_nested_sequence | read_unformated after write_array2d | read_unformated after write_flattened_sequence | read_unformated after write_nested_sequence | write_array2d | write_flattened_sequence | write_nested_sequence |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.007677 / 0.011353 (-0.003675) | 0.005340 / 0.011008 (-0.005668) | 0.075644 / 0.038508 (0.037136) | 0.035046 / 0.023109 (0.011937) | 0.341437 / 0.275898 (0.065538) | 0.377782 / 0.323480 (0.054302) | 0.006091 / 0.007986 (-0.001895) | 0.004170 / 0.004328 (-0.000158) | 0.074294 / 0.004250 (0.070044) | 0.049851 / 0.037052 (0.012798) | 0.351691 / 0.258489 (0.093202) | 0.386020 / 0.293841 (0.092179) | 0.036884 / 0.128546 (-0.091662) | 0.012475 / 0.075646 (-0.063172) | 0.087267 / 0.419271 (-0.332005) | 0.058623 / 0.043533 (0.015090) | 0.347186 / 0.255139 (0.092047) | 0.355869 / 0.283200 (0.072669) | 0.112022 / 0.141683 (-0.029661) | 1.451798 / 1.452155 (-0.000357) | 1.553262 / 1.492716 (0.060546) |

### Benchmark: benchmark_getitem\_100B.json

| metric | get_batch_of\_1024\_random_rows | get_batch_of\_1024\_rows | get_first_row | get_last_row |
|--------|---|---|---|---|
| new / old (diff) | 0.233451 / 0.018006 (0.215445) | 0.444384 / 0.000490 (0.443895) | 0.003695 / 0.000200 (0.003495) | 0.000088 / 0.000054 (0.000034) |

### Benchmark: benchmark_indices_mapping.json

| metric | select | shard | shuffle | sort | train_test_split |
|--------|---|---|---|---|---|
| new / old (diff) | 0.029686 / 0.037411 (-0.007725) | 0.113736 / 0.014526 (0.099210) | 0.123998 / 0.176557 (-0.052559) | 0.197847 / 0.737135 (-0.539288) | 0.129936 / 0.296338 (-0.166403) |

### Benchmark: benchmark_iterating.json

| metric | read 5000 | read 50000 | read_batch 50000 10 | read_batch 50000 100 | read_batch 50000 1000 | read_formatted numpy 5000 | read_formatted pandas 5000 | read_formatted tensorflow 5000 | read_formatted torch 5000 | read_formatted_batch numpy 5000 10 | read_formatted_batch numpy 5000 1000 | shuffled read 5000 | shuffled read 50000 | shuffled read_batch 50000 10 | shuffled read_batch 50000 100 | shuffled read_batch 50000 1000 | shuffled read_formatted numpy 5000 | shuffled read_formatted_batch numpy 5000 10 | shuffled read_formatted_batch numpy 5000 1000 |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.421904 / 0.215209 (0.206695) | 4.203533 / 2.077655 (2.125878) | 2.038199 / 1.504120 (0.534079) | 1.832402 / 1.541195 (0.291208) | 1.930765 / 1.468490 (0.462274) | 0.709775 / 4.584777 (-3.875002) | 3.760893 / 3.745712 (0.015181) | 2.091185 / 5.269862 (-3.178677) | 1.342248 / 4.565676 (-3.223428) | 0.087770 / 0.424275 (-0.336505) | 0.012357 / 0.007607 (0.004750) | 0.519605 / 0.226044 (0.293560) | 5.215883 / 2.268929 (2.946954) | 2.510200 / 55.444624 (-52.934425) | 2.192482 / 6.876477 (-4.683995) | 2.290214 / 2.142072 (0.148141) | 0.872067 / 4.805227 (-3.933160) | 0.168491 / 6.500664 (-6.332173) | 0.064707 / 0.075469 (-0.010762) |

### Benchmark: benchmark_map_filter.json

| metric | filter | map fast-tokenizer batched | map identity | map identity batched | map no-op batched | map no-op batched numpy | map no-op batched pandas | map no-op batched pytorch | map no-op batched tensorflow |
|--------|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 1.291956 / 1.841788 (-0.549832) | 15.244530 / 8.074308 (7.170222) | 13.594895 / 10.191392 (3.403503) | 0.172669 / 0.680424 (-0.507755) | 0.017765 / 0.534201 (-0.516436) | 0.426946 / 0.579283 (-0.152337) | 0.442843 / 0.434364 (0.008479) | 0.549683 / 0.540337 (0.009346) | 0.653433 / 1.386936 (-0.733503) |

![](https://cml.dev/watermark.png#b54a6d21795cf6cc50a13ff870648241a60fd2e0 "CML watermark")

Can you review this @mariosasko ? since Albert is off

Show benchmarks

PyArrow==6.0.0

Show updated benchmarks!

### Benchmark: benchmark_array_xd.json

| metric | read_batch_formatted_as_numpy after write_array2d | read_batch_formatted_as_numpy after write_flattened_sequence | read_batch_formatted_as_numpy after write_nested_sequence | read_batch_unformated after write_array2d | read_batch_unformated after write_flattened_sequence | read_batch_unformated after write_nested_sequence | read_col_formatted_as_numpy after write_array2d | read_col_formatted_as_numpy after write_flattened_sequence | read_col_formatted_as_numpy after write_nested_sequence | read_col_unformated after write_array2d | read_col_unformated after write_flattened_sequence | read_col_unformated after write_nested_sequence | read_formatted_as_numpy after write_array2d | read_formatted_as_numpy after write_flattened_sequence | read_formatted_as_numpy after write_nested_sequence | read_unformated after write_array2d | read_unformated after write_flattened_sequence | read_unformated after write_nested_sequence | write_array2d | write_flattened_sequence | write_nested_sequence |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.008396 / 0.011353 (-0.002957) | 0.004556 / 0.011008 (-0.006452) | 0.101343 / 0.038508 (0.062835) | 0.029137 / 0.023109 (0.006027) | 0.298553 / 0.275898 (0.022655) | 0.334050 / 0.323480 (0.010570) | 0.006746 / 0.007986 (-0.001239) | 0.005050 / 0.004328 (0.000721) | 0.076055 / 0.004250 (0.071804) | 0.031988 / 0.037052 (-0.005064) | 0.301324 / 0.258489 (0.042835) | 0.340121 / 0.293841 (0.046280) | 0.033827 / 0.128546 (-0.094720) | 0.011447 / 0.075646 (-0.064200) | 0.321827 / 0.419271 (-0.097445) | 0.040846 / 0.043533 (-0.002687) | 0.296957 / 0.255139 (0.041818) | 0.324178 / 0.283200 (0.040979) | 0.083852 / 0.141683 (-0.057831) | 1.456123 / 1.452155 (0.003968) | 1.538311 / 1.492716 (0.045595) |

### Benchmark: benchmark_getitem\_100B.json

| metric | get_batch_of\_1024\_random_rows | get_batch_of\_1024\_rows | get_first_row | get_last_row |
|--------|---|---|---|---|
| new / old (diff) | 0.208897 / 0.018006 (0.190891) | 0.430560 / 0.000490 (0.430070) | 0.002917 / 0.000200 (0.002717) | 0.000077 / 0.000054 (0.000022) |

### Benchmark: benchmark_indices_mapping.json

| metric | select | shard | shuffle | sort | train_test_split |
|--------|---|---|---|---|---|
| new / old (diff) | 0.024332 / 0.037411 (-0.013080) | 0.101659 / 0.014526 (0.087133) | 0.107636 / 0.176557 (-0.068920) | 0.168805 / 0.737135 (-0.568330) | 0.111404 / 0.296338 (-0.184934) |

### Benchmark: benchmark_iterating.json

| metric | read 5000 | read 50000 | read_batch 50000 10 | read_batch 50000 100 | read_batch 50000 1000 | read_formatted numpy 5000 | read_formatted pandas 5000 | read_formatted tensorflow 5000 | read_formatted torch 5000 | read_formatted_batch numpy 5000 10 | read_formatted_batch numpy 5000 1000 | shuffled read 5000 | shuffled read 50000 | shuffled read_batch 50000 10 | shuffled read_batch 50000 100 | shuffled read_batch 50000 1000 | shuffled read_formatted numpy 5000 | shuffled read_formatted_batch numpy 5000 10 | shuffled read_formatted_batch numpy 5000 1000 |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.412704 / 0.215209 (0.197495) | 4.124852 / 2.077655 (2.047197) | 1.843555 / 1.504120 (0.339435) | 1.641636 / 1.541195 (0.100441) | 1.755783 / 1.468490 (0.287293) | 0.693212 / 4.584777 (-3.891565) | 3.391803 / 3.745712 (-0.353909) | 1.954473 / 5.269862 (-3.315389) | 1.274395 / 4.565676 (-3.291282) | 0.082536 / 0.424275 (-0.341739) | 0.012335 / 0.007607 (0.004728) | 0.523720 / 0.226044 (0.297676) | 5.268339 / 2.268929 (2.999411) | 2.318163 / 55.444624 (-53.126461) | 1.978503 / 6.876477 (-4.897974) | 2.046689 / 2.142072 (-0.095384) | 0.806735 / 4.805227 (-3.998492) | 0.148010 / 6.500664 (-6.352654) | 0.065305 / 0.075469 (-0.010164) |

### Benchmark: benchmark_map_filter.json

| metric | filter | map fast-tokenizer batched | map identity | map identity batched | map no-op batched | map no-op batched numpy | map no-op batched pandas | map no-op batched pytorch | map no-op batched tensorflow |
|--------|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 1.266950 / 1.841788 (-0.574838) | 13.870803 / 8.074308 (5.796495) | 14.272556 / 10.191392 (4.081164) | 0.151703 / 0.680424 (-0.528720) | 0.028991 / 0.534201 (-0.505210) | 0.400831 / 0.579283 (-0.178452) | 0.400891 / 0.434364 (-0.033473) | 0.476225 / 0.540337 (-0.064113) | 0.564925 / 1.386936 (-0.822011) |

PyArrow==latest

Show updated benchmarks!

### Benchmark: benchmark_array_xd.json

| metric | read_batch_formatted_as_numpy after write_array2d | read_batch_formatted_as_numpy after write_flattened_sequence | read_batch_formatted_as_numpy after write_nested_sequence | read_batch_unformated after write_array2d | read_batch_unformated after write_flattened_sequence | read_batch_unformated after write_nested_sequence | read_col_formatted_as_numpy after write_array2d | read_col_formatted_as_numpy after write_flattened_sequence | read_col_formatted_as_numpy after write_nested_sequence | read_col_unformated after write_array2d | read_col_unformated after write_flattened_sequence | read_col_unformated after write_nested_sequence | read_formatted_as_numpy after write_array2d | read_formatted_as_numpy after write_flattened_sequence | read_formatted_as_numpy after write_nested_sequence | read_unformated after write_array2d | read_unformated after write_flattened_sequence | read_unformated after write_nested_sequence | write_array2d | write_flattened_sequence | write_nested_sequence |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.006810 / 0.011353 (-0.004543) | 0.004544 / 0.011008 (-0.006464) | 0.076516 / 0.038508 (0.038008) | 0.027705 / 0.023109 (0.004596) | 0.343215 / 0.275898 (0.067317) | 0.379136 / 0.323480 (0.055656) | 0.005227 / 0.007986 (-0.002758) | 0.003527 / 0.004328 (-0.000801) | 0.074775 / 0.004250 (0.070524) | 0.041700 / 0.037052 (0.004648) | 0.343612 / 0.258489 (0.085123) | 0.385657 / 0.293841 (0.091817) | 0.032082 / 0.128546 (-0.096464) | 0.011567 / 0.075646 (-0.064079) | 0.083814 / 0.419271 (-0.335458) | 0.042173 / 0.043533 (-0.001360) | 0.340261 / 0.255139 (0.085122) | 0.364778 / 0.283200 (0.081578) | 0.093401 / 0.141683 (-0.048282) | 1.513475 / 1.452155 (0.061320) | 1.599393 / 1.492716 (0.106677) |

### Benchmark: benchmark_getitem\_100B.json

| metric | get_batch_of\_1024\_random_rows | get_batch_of\_1024\_rows | get_first_row | get_last_row |
|--------|---|---|---|---|
| new / old (diff) | 0.237117 / 0.018006 (0.219111) | 0.424241 / 0.000490 (0.423751) | 0.002900 / 0.000200 (0.002700) | 0.000076 / 0.000054 (0.000021) |

### Benchmark: benchmark_indices_mapping.json

| metric | select | shard | shuffle | sort | train_test_split |
|--------|---|---|---|---|---|
| new / old (diff) | 0.031122 / 0.037411 (-0.006289) | 0.107530 / 0.014526 (0.093004) | 0.117777 / 0.176557 (-0.058780) | 0.188300 / 0.737135 (-0.548836) | 0.119989 / 0.296338 (-0.176349) |

### Benchmark: benchmark_iterating.json

| metric | read 5000 | read 50000 | read_batch 50000 10 | read_batch 50000 100 | read_batch 50000 1000 | read_formatted numpy 5000 | read_formatted pandas 5000 | read_formatted tensorflow 5000 | read_formatted torch 5000 | read_formatted_batch numpy 5000 10 | read_formatted_batch numpy 5000 1000 | shuffled read 5000 | shuffled read 50000 | shuffled read_batch 50000 10 | shuffled read_batch 50000 100 | shuffled read_batch 50000 1000 | shuffled read_formatted numpy 5000 | shuffled read_formatted_batch numpy 5000 10 | shuffled read_formatted_batch numpy 5000 1000 |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.438563 / 0.215209 (0.223354) | 4.404969 / 2.077655 (2.327315) | 2.260182 / 1.504120 (0.756062) | 2.035472 / 1.541195 (0.494277) | 2.045685 / 1.468490 (0.577195) | 0.706758 / 4.584777 (-3.878019) | 3.434843 / 3.745712 (-0.310869) | 1.909533 / 5.269862 (-3.360328) | 1.175374 / 4.565676 (-3.390303) | 0.084831 / 0.424275 (-0.339444) | 0.012441 / 0.007607 (0.004833) | 0.551818 / 0.226044 (0.325774) | 5.509005 / 2.268929 (3.240077) | 2.576545 / 55.444624 (-52.868080) | 2.226204 / 6.876477 (-4.650272) | 2.276544 / 2.142072 (0.134471) | 0.818069 / 4.805227 (-3.987158) | 0.152797 / 6.500664 (-6.347867) | 0.067896 / 0.075469 (-0.007573) |

### Benchmark: benchmark_map_filter.json

| metric | filter | map fast-tokenizer batched | map identity | map identity batched | map no-op batched | map no-op batched numpy | map no-op batched pandas | map no-op batched pytorch | map no-op batched tensorflow |
|--------|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 1.276859 / 1.841788 (-0.564929) | 14.312914 / 8.074308 (6.238606) | 13.406602 / 10.191392 (3.215210) | 0.157466 / 0.680424 (-0.522958) | 0.016709 / 0.534201 (-0.517492) | 0.390951 / 0.579283 (-0.188333) | 0.395525 / 0.434364 (-0.038839) | 0.484486 / 0.540337 (-0.055852) | 0.576125 / 1.386936 (-0.810811) |

![](https://cml.dev/watermark.png#b951e1b6cdd927604599f1aa5dadfb8ee8e62e05 "CML watermark")

Show benchmarks

PyArrow==8.0.0

Show updated benchmarks!

### Benchmark: benchmark_array_xd.json

| metric | read_batch_formatted_as_numpy after write_array2d | read_batch_formatted_as_numpy after write_flattened_sequence | read_batch_formatted_as_numpy after write_nested_sequence | read_batch_unformated after write_array2d | read_batch_unformated after write_flattened_sequence | read_batch_unformated after write_nested_sequence | read_col_formatted_as_numpy after write_array2d | read_col_formatted_as_numpy after write_flattened_sequence | read_col_formatted_as_numpy after write_nested_sequence | read_col_unformated after write_array2d | read_col_unformated after write_flattened_sequence | read_col_unformated after write_nested_sequence | read_formatted_as_numpy after write_array2d | read_formatted_as_numpy after write_flattened_sequence | read_formatted_as_numpy after write_nested_sequence | read_unformated after write_array2d | read_unformated after write_flattened_sequence | read_unformated after write_nested_sequence | write_array2d | write_flattened_sequence | write_nested_sequence |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.007316 / 0.011353 (-0.004037) | 0.005041 / 0.011008 (-0.005968) | 0.100477 / 0.038508 (0.061969) | 0.034068 / 0.023109 (0.010959) | 0.351156 / 0.275898 (0.075258) | 0.373892 / 0.323480 (0.050412) | 0.005748 / 0.007986 (-0.002237) | 0.003959 / 0.004328 (-0.000370) | 0.075540 / 0.004250 (0.071290) | 0.045282 / 0.037052 (0.008230) | 0.362364 / 0.258489 (0.103874) | 0.376461 / 0.293841 (0.082620) | 0.036724 / 0.128546 (-0.091822) | 0.012008 / 0.075646 (-0.063638) | 0.333802 / 0.419271 (-0.085470) | 0.050107 / 0.043533 (0.006574) | 0.348003 / 0.255139 (0.092864) | 0.367187 / 0.283200 (0.083988) | 0.103171 / 0.141683 (-0.038511) | 1.448281 / 1.452155 (-0.003874) | 1.516231 / 1.492716 (0.023514) |

### Benchmark: benchmark_getitem\_100B.json

| metric | get_batch_of\_1024\_random_rows | get_batch_of\_1024\_rows | get_first_row | get_last_row |
|--------|---|---|---|---|
| new / old (diff) | 0.203651 / 0.018006 (0.185645) | 0.438103 / 0.000490 (0.437613) | 0.004165 / 0.000200 (0.003966) | 0.000085 / 0.000054 (0.000030) |

### Benchmark: benchmark_indices_mapping.json

| metric | select | shard | shuffle | sort | train_test_split |
|--------|---|---|---|---|---|
| new / old (diff) | 0.027068 / 0.037411 (-0.010343) | 0.111728 / 0.014526 (0.097202) | 0.116963 / 0.176557 (-0.059594) | 0.172652 / 0.737135 (-0.564483) | 0.124257 / 0.296338 (-0.172082) |

### Benchmark: benchmark_iterating.json

| metric | read 5000 | read 50000 | read_batch 50000 10 | read_batch 50000 100 | read_batch 50000 1000 | read_formatted numpy 5000 | read_formatted pandas 5000 | read_formatted tensorflow 5000 | read_formatted torch 5000 | read_formatted_batch numpy 5000 10 | read_formatted_batch numpy 5000 1000 | shuffled read 5000 | shuffled read 50000 | shuffled read_batch 50000 10 | shuffled read_batch 50000 100 | shuffled read_batch 50000 1000 | shuffled read_formatted numpy 5000 | shuffled read_formatted_batch numpy 5000 10 | shuffled read_formatted_batch numpy 5000 1000 |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.407937 / 0.215209 (0.192728) | 4.066008 / 2.077655 (1.988353) | 1.895000 / 1.504120 (0.390880) | 1.698422 / 1.541195 (0.157227) | 1.872446 / 1.468490 (0.403956) | 0.688888 / 4.584777 (-3.895889) | 3.743635 / 3.745712 (-0.002077) | 2.161507 / 5.269862 (-3.108354) | 1.485218 / 4.565676 (-3.080458) | 0.085959 / 0.424275 (-0.338316) | 0.012554 / 0.007607 (0.004947) | 0.510953 / 0.226044 (0.284909) | 5.103241 / 2.268929 (2.834312) | 2.439670 / 55.444624 (-53.004955) | 2.057089 / 6.876477 (-4.819387) | 2.240137 / 2.142072 (0.098065) | 0.847750 / 4.805227 (-3.957477) | 0.172952 / 6.500664 (-6.327712) | 0.066023 / 0.075469 (-0.009446) |

### Benchmark: benchmark_map_filter.json

| metric | filter | map fast-tokenizer batched | map identity | map identity batched | map no-op batched | map no-op batched numpy | map no-op batched pandas | map no-op batched pytorch | map no-op batched tensorflow |
|--------|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 1.190677 / 1.841788 (-0.651110) | 14.593162 / 8.074308 (6.518854) | 14.254983 / 10.191392 (4.063591) | 0.155811 / 0.680424 (-0.524613) | 0.017698 / 0.534201 (-0.516503) | 0.420455 / 0.579283 (-0.158828) | 0.412146 / 0.434364 (-0.022218) | 0.493113 / 0.540337 (-0.047225) | 0.582097 / 1.386936 (-0.804839) |

PyArrow==latest

Show updated benchmarks!

### Benchmark: benchmark_array_xd.json

| metric | read_batch_formatted_as_numpy after write_array2d | read_batch_formatted_as_numpy after write_flattened_sequence | read_batch_formatted_as_numpy after write_nested_sequence | read_batch_unformated after write_array2d | read_batch_unformated after write_flattened_sequence | read_batch_unformated after write_nested_sequence | read_col_formatted_as_numpy after write_array2d | read_col_formatted_as_numpy after write_flattened_sequence | read_col_formatted_as_numpy after write_nested_sequence | read_col_unformated after write_array2d | read_col_unformated after write_flattened_sequence | read_col_unformated after write_nested_sequence | read_formatted_as_numpy after write_array2d | read_formatted_as_numpy after write_flattened_sequence | read_formatted_as_numpy after write_nested_sequence | read_unformated after write_array2d | read_unformated after write_flattened_sequence | read_unformated after write_nested_sequence | write_array2d | write_flattened_sequence | write_nested_sequence |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.007319 / 0.011353 (-0.004033) | 0.005102 / 0.011008 (-0.005906) | 0.073760 / 0.038508 (0.035252) | 0.033496 / 0.023109 (0.010387) | 0.338778 / 0.275898 (0.062880) | 0.371870 / 0.323480 (0.048391) | 0.005804 / 0.007986 (-0.002182) | 0.004142 / 0.004328 (-0.000186) | 0.073203 / 0.004250 (0.068953) | 0.046568 / 0.037052 (0.009516) | 0.343544 / 0.258489 (0.085055) | 0.381188 / 0.293841 (0.087347) | 0.036391 / 0.128546 (-0.092155) | 0.012046 / 0.075646 (-0.063600) | 0.086007 / 0.419271 (-0.333265) | 0.048706 / 0.043533 (0.005173) | 0.330836 / 0.255139 (0.075697) | 0.355328 / 0.283200 (0.072128) | 0.100104 / 0.141683 (-0.041579) | 1.434237 / 1.452155 (-0.017917) | 1.549380 / 1.492716 (0.056663) |

### Benchmark: benchmark_getitem\_100B.json

| metric | get_batch_of\_1024\_random_rows | get_batch_of\_1024\_rows | get_first_row | get_last_row |
|--------|---|---|---|---|
| new / old (diff) | 0.231099 / 0.018006 (0.213093) | 0.450650 / 0.000490 (0.450160) | 0.000404 / 0.000200 (0.000204) | 0.000059 / 0.000054 (0.000004) |

### Benchmark: benchmark_indices_mapping.json

| metric | select | shard | shuffle | sort | train_test_split |
|--------|---|---|---|---|---|
| new / old (diff) | 0.030534 / 0.037411 (-0.006877) | 0.119005 / 0.014526 (0.104479) | 0.125362 / 0.176557 (-0.051195) | 0.176823 / 0.737135 (-0.560313) | 0.132044 / 0.296338 (-0.164295) |

### Benchmark: benchmark_iterating.json

| metric | read 5000 | read 50000 | read_batch 50000 10 | read_batch 50000 100 | read_batch 50000 1000 | read_formatted numpy 5000 | read_formatted pandas 5000 | read_formatted tensorflow 5000 | read_formatted torch 5000 | read_formatted_batch numpy 5000 10 | read_formatted_batch numpy 5000 1000 | shuffled read 5000 | shuffled read 50000 | shuffled read_batch 50000 10 | shuffled read_batch 50000 100 | shuffled read_batch 50000 1000 | shuffled read_formatted numpy 5000 | shuffled read_formatted_batch numpy 5000 10 | shuffled read_formatted_batch numpy 5000 1000 |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.431004 / 0.215209 (0.215795) | 4.318969 / 2.077655 (2.241315) | 1.994941 / 1.504120 (0.490821) | 1.791870 / 1.541195 (0.250675) | 1.904134 / 1.468490 (0.435644) | 0.723493 / 4.584777 (-3.861284) | 3.823670 / 3.745712 (0.077958) | 2.118892 / 5.269862 (-3.150969) | 1.375088 / 4.565676 (-3.190588) | 0.088875 / 0.424275 (-0.335400) | 0.013137 / 0.007607 (0.005530) | 0.530523 / 0.226044 (0.304479) | 5.341438 / 2.268929 (3.072509) | 2.459044 / 55.444624 (-52.985580) | 2.150119 / 6.876477 (-4.726357) | 2.228567 / 2.142072 (0.086494) | 0.877549 / 4.805227 (-3.927678) | 0.175040 / 6.500664 (-6.325625) | 0.068188 / 0.075469 (-0.007281) |

### Benchmark: benchmark_map_filter.json

| metric | filter | map fast-tokenizer batched | map identity | map identity batched | map no-op batched | map no-op batched numpy | map no-op batched pandas | map no-op batched pytorch | map no-op batched tensorflow |
|--------|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 1.273780 / 1.841788 (-0.568008) | 15.206331 / 8.074308 (7.132023) | 14.963058 / 10.191392 (4.771666) | 0.184543 / 0.680424 (-0.495881) | 0.017612 / 0.534201 (-0.516589) | 0.426248 / 0.579283 (-0.153035) | 0.437889 / 0.434364 (0.003525) | 0.508979 / 0.540337 (-0.031359) | 0.602040 / 1.386936 (-0.784896) |

![](https://cml.dev/watermark.png#c5ca1d86949ec3a5fdaec03b80500fb822bcfab4 "CML watermark")

This way we can control the size of the record_batches/row_groups of arrow/parquet files.

This can be useful for `datasets-server` to keep control of the row groups size which can affect random access performance for audio/image/video datasets

Right now having 1,000 examples per row group cause some image datasets to be pretty slow for random access (e.g. 4seconds for `beans` to get 20 rows)


Set dev version

The docs for this PR live [here](https://moon-ci-docs.huggingface.co/docs/datasets/pr_5564). All of your documentation changes will be reflected on that endpoint.

Show benchmarks

PyArrow==6.0.0

Show updated benchmarks!

### Benchmark: benchmark_array_xd.json

| metric | read_batch_formatted_as_numpy after write_array2d | read_batch_formatted_as_numpy after write_flattened_sequence | read_batch_formatted_as_numpy after write_nested_sequence | read_batch_unformated after write_array2d | read_batch_unformated after write_flattened_sequence | read_batch_unformated after write_nested_sequence | read_col_formatted_as_numpy after write_array2d | read_col_formatted_as_numpy after write_flattened_sequence | read_col_formatted_as_numpy after write_nested_sequence | read_col_unformated after write_array2d | read_col_unformated after write_flattened_sequence | read_col_unformated after write_nested_sequence | read_formatted_as_numpy after write_array2d | read_formatted_as_numpy after write_flattened_sequence | read_formatted_as_numpy after write_nested_sequence | read_unformated after write_array2d | read_unformated after write_flattened_sequence | read_unformated after write_nested_sequence | write_array2d | write_flattened_sequence | write_nested_sequence |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.008810 / 0.011353 (-0.002543) | 0.004583 / 0.011008 (-0.006425) | 0.100787 / 0.038508 (0.062279) | 0.030170 / 0.023109 (0.007061) | 0.301749 / 0.275898 (0.025851) | 0.386958 / 0.323480 (0.063478) | 0.007211 / 0.007986 (-0.000775) | 0.004939 / 0.004328 (0.000611) | 0.078046 / 0.004250 (0.073796) | 0.035672 / 0.037052 (-0.001380) | 0.314403 / 0.258489 (0.055914) | 0.348547 / 0.293841 (0.054706) | 0.034242 / 0.128546 (-0.094304) | 0.011599 / 0.075646 (-0.064047) | 0.321936 / 0.419271 (-0.097336) | 0.043214 / 0.043533 (-0.000319) | 0.298782 / 0.255139 (0.043643) | 0.334513 / 0.283200 (0.051313) | 0.091630 / 0.141683 (-0.050053) | 1.518194 / 1.452155 (0.066039) | 1.553665 / 1.492716 (0.060949) |

### Benchmark: benchmark_getitem\_100B.json

| metric | get_batch_of\_1024\_random_rows | get_batch_of\_1024\_rows | get_first_row | get_last_row |
|--------|---|---|---|---|
| new / old (diff) | 0.196322 / 0.018006 (0.178316) | 0.427280 / 0.000490 (0.426790) | 0.001933 / 0.000200 (0.001733) | 0.000069 / 0.000054 (0.000015) |

### Benchmark: benchmark_indices_mapping.json

| metric | select | shard | shuffle | sort | train_test_split |
|--------|---|---|---|---|---|
| new / old (diff) | 0.023190 / 0.037411 (-0.014221) | 0.097387 / 0.014526 (0.082862) | 0.104532 / 0.176557 (-0.072024) | 0.166670 / 0.737135 (-0.570465) | 0.108787 / 0.296338 (-0.187552) |

### Benchmark: benchmark_iterating.json

| metric | read 5000 | read 50000 | read_batch 50000 10 | read_batch 50000 100 | read_batch 50000 1000 | read_formatted numpy 5000 | read_formatted pandas 5000 | read_formatted tensorflow 5000 | read_formatted torch 5000 | read_formatted_batch numpy 5000 10 | read_formatted_batch numpy 5000 1000 | shuffled read 5000 | shuffled read 50000 | shuffled read_batch 50000 10 | shuffled read_batch 50000 100 | shuffled read_batch 50000 1000 | shuffled read_formatted numpy 5000 | shuffled read_formatted_batch numpy 5000 10 | shuffled read_formatted_batch numpy 5000 1000 |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.415776 / 0.215209 (0.200567) | 4.135899 / 2.077655 (2.058244) | 1.857600 / 1.504120 (0.353480) | 1.654099 / 1.541195 (0.112904) | 1.729102 / 1.468490 (0.260612) | 0.695946 / 4.584777 (-3.888831) | 3.352776 / 3.745712 (-0.392936) | 2.754443 / 5.269862 (-2.515418) | 1.517181 / 4.565676 (-3.048495) | 0.082782 / 0.424275 (-0.341493) | 0.012431 / 0.007607 (0.004824) | 0.526593 / 0.226044 (0.300548) | 5.263051 / 2.268929 (2.994123) | 2.290713 / 55.444624 (-53.153911) | 1.953017 / 6.876477 (-4.923460) | 1.998419 / 2.142072 (-0.143653) | 0.817055 / 4.805227 (-3.988173) | 0.148213 / 6.500664 (-6.352451) | 0.065527 / 0.075469 (-0.009942) |

### Benchmark: benchmark_map_filter.json

| metric | filter | map fast-tokenizer batched | map identity | map identity batched | map no-op batched | map no-op batched numpy | map no-op batched pandas | map no-op batched pytorch | map no-op batched tensorflow |
|--------|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 1.254275 / 1.841788 (-0.587513) | 13.618962 / 8.074308 (5.544654) | 14.057134 / 10.191392 (3.865742) | 0.137180 / 0.680424 (-0.543244) | 0.028460 / 0.534201 (-0.505741) | 0.393836 / 0.579283 (-0.185447) | 0.406665 / 0.434364 (-0.027699) | 0.476812 / 0.540337 (-0.063526) | 0.561047 / 1.386936 (-0.825889) |

PyArrow==latest

Show updated benchmarks!

### Benchmark: benchmark_array_xd.json

| metric | read_batch_formatted_as_numpy after write_array2d | read_batch_formatted_as_numpy after write_flattened_sequence | read_batch_formatted_as_numpy after write_nested_sequence | read_batch_unformated after write_array2d | read_batch_unformated after write_flattened_sequence | read_batch_unformated after write_nested_sequence | read_col_formatted_as_numpy after write_array2d | read_col_formatted_as_numpy after write_flattened_sequence | read_col_formatted_as_numpy after write_nested_sequence | read_col_unformated after write_array2d | read_col_unformated after write_flattened_sequence | read_col_unformated after write_nested_sequence | read_formatted_as_numpy after write_array2d | read_formatted_as_numpy after write_flattened_sequence | read_formatted_as_numpy after write_nested_sequence | read_unformated after write_array2d | read_unformated after write_flattened_sequence | read_unformated after write_nested_sequence | write_array2d | write_flattened_sequence | write_nested_sequence |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.006483 / 0.011353 (-0.004870) | 0.004525 / 0.011008 (-0.006483) | 0.075696 / 0.038508 (0.037188) | 0.027306 / 0.023109 (0.004197) | 0.359141 / 0.275898 (0.083243) | 0.394595 / 0.323480 (0.071115) | 0.004907 / 0.007986 (-0.003079) | 0.003403 / 0.004328 (-0.000925) | 0.074473 / 0.004250 (0.070223) | 0.037801 / 0.037052 (0.000749) | 0.359350 / 0.258489 (0.100861) | 0.411902 / 0.293841 (0.118061) | 0.032280 / 0.128546 (-0.096267) | 0.011728 / 0.075646 (-0.063918) | 0.085692 / 0.419271 (-0.333580) | 0.047779 / 0.043533 (0.004246) | 0.348820 / 0.255139 (0.093681) | 0.389396 / 0.283200 (0.106197) | 0.094923 / 0.141683 (-0.046760) | 1.507137 / 1.452155 (0.054982) | 1.556873 / 1.492716 (0.064157) |

### Benchmark: benchmark_getitem\_100B.json

| metric | get_batch_of\_1024\_random_rows | get_batch_of\_1024\_rows | get_first_row | get_last_row |
|--------|---|---|---|---|
| new / old (diff) | 0.197510 / 0.018006 (0.179504) | 0.413885 / 0.000490 (0.413395) | 0.002527 / 0.000200 (0.002327) | 0.000073 / 0.000054 (0.000019) |

### Benchmark: benchmark_indices_mapping.json

| metric | select | shard | shuffle | sort | train_test_split |
|--------|---|---|---|---|---|
| new / old (diff) | 0.024571 / 0.037411 (-0.012840) | 0.099845 / 0.014526 (0.085319) | 0.108130 / 0.176557 (-0.068426) | 0.176153 / 0.737135 (-0.560982) | 0.111907 / 0.296338 (-0.184432) |

### Benchmark: benchmark_iterating.json

| metric | read 5000 | read 50000 | read_batch 50000 10 | read_batch 50000 100 | read_batch 50000 1000 | read_formatted numpy 5000 | read_formatted pandas 5000 | read_formatted tensorflow 5000 | read_formatted torch 5000 | read_formatted_batch numpy 5000 10 | read_formatted_batch numpy 5000 1000 | shuffled read 5000 | shuffled read 50000 | shuffled read_batch 50000 10 | shuffled read_batch 50000 100 | shuffled read_batch 50000 1000 | shuffled read_formatted numpy 5000 | shuffled read_formatted_batch numpy 5000 10 | shuffled read_formatted_batch numpy 5000 1000 |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.436393 / 0.215209 (0.221184) | 4.343296 / 2.077655 (2.265642) | 2.056062 / 1.504120 (0.551942) | 1.855372 / 1.541195 (0.314177) | 1.946429 / 1.468490 (0.477939) | 0.701862 / 4.584777 (-3.882915) | 3.337115 / 3.745712 (-0.408597) | 2.755416 / 5.269862 (-2.514446) | 1.335596 / 4.565676 (-3.230081) | 0.083938 / 0.424275 (-0.340337) | 0.012914 / 0.007607 (0.005307) | 0.530272 / 0.226044 (0.304228) | 5.307739 / 2.268929 (3.038810) | 2.506435 / 55.444624 (-52.938189) | 2.170830 / 6.876477 (-4.705646) | 2.224641 / 2.142072 (0.082568) | 0.804416 / 4.805227 (-4.000811) | 0.151594 / 6.500664 (-6.349070) | 0.067221 / 0.075469 (-0.008248) |

### Benchmark: benchmark_map_filter.json

| metric | filter | map fast-tokenizer batched | map identity | map identity batched | map no-op batched | map no-op batched numpy | map no-op batched pandas | map no-op batched pytorch | map no-op batched tensorflow |
|--------|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 1.257063 / 1.841788 (-0.584725) | 14.054346 / 8.074308 (5.980038) | 13.490649 / 10.191392 (3.299257) | 0.139320 / 0.680424 (-0.541104) | 0.016501 / 0.534201 (-0.517700) | 0.382655 / 0.579283 (-0.196629) | 0.383305 / 0.434364 (-0.051059) | 0.465091 / 0.540337 (-0.075247) | 0.552552 / 1.386936 (-0.834384) |

![](https://cml.dev/watermark.png#c480083958126c40bb7bdba8e1eeb3945a8fe6ea "CML watermark")

Show benchmarks

PyArrow==6.0.0

Show updated benchmarks!

### Benchmark: benchmark_array_xd.json

| metric | read_batch_formatted_as_numpy after write_array2d | read_batch_formatted_as_numpy after write_flattened_sequence | read_batch_formatted_as_numpy after write_nested_sequence | read_batch_unformated after write_array2d | read_batch_unformated after write_flattened_sequence | read_batch_unformated after write_nested_sequence | read_col_formatted_as_numpy after write_array2d | read_col_formatted_as_numpy after write_flattened_sequence | read_col_formatted_as_numpy after write_nested_sequence | read_col_unformated after write_array2d | read_col_unformated after write_flattened_sequence | read_col_unformated after write_nested_sequence | read_formatted_as_numpy after write_array2d | read_formatted_as_numpy after write_flattened_sequence | read_formatted_as_numpy after write_nested_sequence | read_unformated after write_array2d | read_unformated after write_flattened_sequence | read_unformated after write_nested_sequence | write_array2d | write_flattened_sequence | write_nested_sequence |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.011278 / 0.011353 (-0.000075) | 0.007351 / 0.011008 (-0.003657) | 0.131145 / 0.038508 (0.092637) | 0.041585 / 0.023109 (0.018476) | 0.410230 / 0.275898 (0.134332) | 0.464069 / 0.323480 (0.140589) | 0.010228 / 0.007986 (0.002242) | 0.005324 / 0.004328 (0.000996) | 0.102680 / 0.004250 (0.098430) | 0.041644 / 0.037052 (0.004592) | 0.439127 / 0.258489 (0.180638) | 0.467828 / 0.293841 (0.173987) | 0.054373 / 0.128546 (-0.074173) | 0.019495 / 0.075646 (-0.056152) | 0.432425 / 0.419271 (0.013153) | 0.056863 / 0.043533 (0.013331) | 0.405883 / 0.255139 (0.150744) | 0.452786 / 0.283200 (0.169586) | 0.109888 / 0.141683 (-0.031795) | 1.797015 / 1.452155 (0.344860) | 1.985937 / 1.492716 (0.493221) |

### Benchmark: benchmark_getitem\_100B.json

| metric | get_batch_of\_1024\_random_rows | get_batch_of\_1024\_rows | get_first_row | get_last_row |
|--------|---|---|---|---|
| new / old (diff) | 0.275121 / 0.018006 (0.257115) | 0.587585 / 0.000490 (0.587095) | 0.005557 / 0.000200 (0.005357) | 0.000118 / 0.000054 (0.000064) |

### Benchmark: benchmark_indices_mapping.json

| metric | select | shard | shuffle | sort | train_test_split |
|--------|---|---|---|---|---|
| new / old (diff) | 0.032968 / 0.037411 (-0.004443) | 0.135886 / 0.014526 (0.121360) | 0.154000 / 0.176557 (-0.022557) | 0.233345 / 0.737135 (-0.503790) | 0.144125 / 0.296338 (-0.152214) |

### Benchmark: benchmark_iterating.json

| metric | read 5000 | read 50000 | read_batch 50000 10 | read_batch 50000 100 | read_batch 50000 1000 | read_formatted numpy 5000 | read_formatted pandas 5000 | read_formatted tensorflow 5000 | read_formatted torch 5000 | read_formatted_batch numpy 5000 10 | read_formatted_batch numpy 5000 1000 | shuffled read 5000 | shuffled read 50000 | shuffled read_batch 50000 10 | shuffled read_batch 50000 100 | shuffled read_batch 50000 1000 | shuffled read_formatted numpy 5000 | shuffled read_formatted_batch numpy 5000 10 | shuffled read_formatted_batch numpy 5000 1000 |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.613056 / 0.215209 (0.397847) | 6.206135 / 2.077655 (4.128480) | 2.686989 / 1.504120 (1.182869) | 2.389946 / 1.541195 (0.848751) | 2.437506 / 1.468490 (0.969016) | 1.255900 / 4.584777 (-3.328877) | 5.654803 / 3.745712 (1.909091) | 5.467693 / 5.269862 (0.197832) | 2.872397 / 4.565676 (-1.693279) | 0.145658 / 0.424275 (-0.278617) | 0.016883 / 0.007607 (0.009276) | 0.793820 / 0.226044 (0.567775) | 7.961881 / 2.268929 (5.692952) | 3.617422 / 55.444624 (-51.827203) | 2.795185 / 6.876477 (-4.081292) | 2.881726 / 2.142072 (0.739653) | 1.434543 / 4.805227 (-3.370685) | 0.252206 / 6.500664 (-6.248458) | 0.094694 / 0.075469 (0.019225) |

### Benchmark: benchmark_map_filter.json

| metric | filter | map fast-tokenizer batched | map identity | map identity batched | map no-op batched | map no-op batched numpy | map no-op batched pandas | map no-op batched pytorch | map no-op batched tensorflow |
|--------|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 1.552401 / 1.841788 (-0.289386) | 18.436068 / 8.074308 (10.361760) | 22.539049 / 10.191392 (12.347657) | 0.269471 / 0.680424 (-0.410953) | 0.053242 / 0.534201 (-0.480959) | 0.568325 / 0.579283 (-0.010958) | 0.660339 / 0.434364 (0.225975) | 0.689507 / 0.540337 (0.149169) | 0.836785 / 1.386936 (-0.550151) |

PyArrow==latest

Show updated benchmarks!

### Benchmark: benchmark_array_xd.json

| metric | read_batch_formatted_as_numpy after write_array2d | read_batch_formatted_as_numpy after write_flattened_sequence | read_batch_formatted_as_numpy after write_nested_sequence | read_batch_unformated after write_array2d | read_batch_unformated after write_flattened_sequence | read_batch_unformated after write_nested_sequence | read_col_formatted_as_numpy after write_array2d | read_col_formatted_as_numpy after write_flattened_sequence | read_col_formatted_as_numpy after write_nested_sequence | read_col_unformated after write_array2d | read_col_unformated after write_flattened_sequence | read_col_unformated after write_nested_sequence | read_formatted_as_numpy after write_array2d | read_formatted_as_numpy after write_flattened_sequence | read_formatted_as_numpy after write_nested_sequence | read_unformated after write_array2d | read_unformated after write_flattened_sequence | read_unformated after write_nested_sequence | write_array2d | write_flattened_sequence | write_nested_sequence |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.009853 / 0.011353 (-0.001500) | 0.009752 / 0.011008 (-0.001256) | 0.095422 / 0.038508 (0.056914) | 0.037760 / 0.023109 (0.014651) | 0.450898 / 0.275898 (0.175000) | 0.501671 / 0.323480 (0.178191) | 0.006748 / 0.007986 (-0.001237) | 0.005054 / 0.004328 (0.000725) | 0.099382 / 0.004250 (0.095131) | 0.058078 / 0.037052 (0.021026) | 0.447606 / 0.258489 (0.189116) | 0.503887 / 0.293841 (0.210046) | 0.054579 / 0.128546 (-0.073967) | 0.026150 / 0.075646 (-0.049496) | 0.113042 / 0.419271 (-0.306230) | 0.061049 / 0.043533 (0.017516) | 0.437831 / 0.255139 (0.182692) | 0.480830 / 0.283200 (0.197630) | 0.121199 / 0.141683 (-0.020484) | 1.795409 / 1.452155 (0.343254) | 1.911207 / 1.492716 (0.418491) |

### Benchmark: benchmark_getitem\_100B.json

| metric | get_batch_of\_1024\_random_rows | get_batch_of\_1024\_rows | get_first_row | get_last_row |
|--------|---|---|---|---|
| new / old (diff) | 0.311774 / 0.018006 (0.293768) | 0.602027 / 0.000490 (0.601537) | 0.000651 / 0.000200 (0.000451) | 0.000136 / 0.000054 (0.000081) |

### Benchmark: benchmark_indices_mapping.json

| metric | select | shard | shuffle | sort | train_test_split |
|--------|---|---|---|---|---|
| new / old (diff) | 0.035185 / 0.037411 (-0.002227) | 0.149574 / 0.014526 (0.135048) | 0.153672 / 0.176557 (-0.022884) | 0.241720 / 0.737135 (-0.495416) | 0.153543 / 0.296338 (-0.142795) |

### Benchmark: benchmark_iterating.json

| metric | read 5000 | read 50000 | read_batch 50000 10 | read_batch 50000 100 | read_batch 50000 1000 | read_formatted numpy 5000 | read_formatted pandas 5000 | read_formatted tensorflow 5000 | read_formatted torch 5000 | read_formatted_batch numpy 5000 10 | read_formatted_batch numpy 5000 1000 | shuffled read 5000 | shuffled read 50000 | shuffled read_batch 50000 10 | shuffled read_batch 50000 100 | shuffled read_batch 50000 1000 | shuffled read_formatted numpy 5000 | shuffled read_formatted_batch numpy 5000 10 | shuffled read_formatted_batch numpy 5000 1000 |
|--------|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 0.678508 / 0.215209 (0.463299) | 6.535313 / 2.077655 (4.457658) | 2.840175 / 1.504120 (1.336055) | 2.458141 / 1.541195 (0.916947) | 2.551369 / 1.468490 (1.082879) | 1.339117 / 4.584777 (-3.245660) | 5.844429 / 3.745712 (2.098717) | 3.221100 / 5.269862 (-2.048762) | 2.114844 / 4.565676 (-2.450833) | 0.149263 / 0.424275 (-0.275012) | 0.016101 / 0.007607 (0.008494) | 0.830650 / 0.226044 (0.604605) | 8.096655 / 2.268929 (5.827727) | 3.445947 / 55.444624 (-51.998677) | 2.826874 / 6.876477 (-4.049603) | 2.812765 / 2.142072 (0.670693) | 1.453789 / 4.805227 (-3.351438) | 0.263911 / 6.500664 (-6.236753) | 0.082609 / 0.075469 (0.007139) |

### Benchmark: benchmark_map_filter.json

| metric | filter | map fast-tokenizer batched | map identity | map identity batched | map no-op batched | map no-op batched numpy | map no-op batched pandas | map no-op batched pytorch | map no-op batched tensorflow |
|--------|---|---|---|---|---|---|---|---|---|
| new / old (diff) | 1.651624 / 1.841788 (-0.190163) | 18.703020 / 8.074308 (10.628712) | 21.360445 / 10.191392 (11.169053) | 0.249718 / 0.680424 (-0.430706) | 0.028373 / 0.534201 (-0.505828) | 0.576237 / 0.579283 (-0.003046) | 0.620574 / 0.434364 (0.186210) | 0.684155 / 0.540337 (0.143817) | 0.758950 / 1.386936 (-0.627986) |

![](https://cml.dev/watermark.png#f51ef325602bb297a18a75680575cbe9b940b1d9 "CML watermark")