import{s as Be,n as Ge,o as ke}from"../chunks/scheduler.9cda857a.js";import{S as qe,i as Ye,e as p,s as l,c as r,h as Ce,a as i,d as t,b as n,f as _e,g as o,j as c,k as ne,l as We,m as s,n as m,t as h,o as d,p as u}from"../chunks/index.003612b0.js";import{C as Re}from"../chunks/CopyLLMTxtMenu.44d08a39.js";import{C as y}from"../chunks/CodeBlock.a90632f1.js";import{H as ra,E as Ve}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.e5cfda8c.js";function Fe(pe){let w,ha,oa,da,J,ua,f,ca,T,ie=`<a href="https://github.com/apache/arrow" rel="nofollow">Arrow</a> is a columnar format and a toolbox for fast data interchange and in-memory analytics.
Since PyArrow supports <a href="https://filesystem-spec.readthedocs.io" rel="nofollow">fsspec</a> to read and write remote data, you can use the Hugging Face paths (<a href="/docs/huggingface_hub/guides/hf_file_system#integrations"><code>hf://</code></a>) to read and write data on the Hub.
It is especially useful for <a href="https://parquet.apache.org/" rel="nofollow">Parquet</a> data, since Parquet is the most common file format on Hugging Face.
Indeed, Parquet is particularly efficient thanks to its structure, typing, metadata and compression.`,ya,g,wa,j,re="You can load data from local files or from remote storage like Hugging Face Datasets. PyArrow supports many formats including CSV, JSON and more importantly Parquet:",Ma,b,Ja,U,oe='To load a file from Hugging Face, the path needs to start with <code>hf://</code>. For example, the path to the <a href="https://huggingface.co/datasets/stanfordnlp/imdb" rel="nofollow">stanfordnlp/imdb</a> dataset repository is <code>hf://datasets/stanfordnlp/imdb</code>. The dataset on Hugging Face contains multiple Parquet files. The Parquet file format is designed to make reading and writing data frames efficient, and to make sharing data across data analysis languages easy. Here is how to load the file <code>plain_text/train-00000-of-00001.parquet</code> as a pyarrow Table (it requires <code>pyarrow&gt;=21.0</code>):',fa,I,Ta,Z,me="If you don’t want to load the full Parquet data, you can get the Parquet metadata or load row group by row group instead:",ga,X,ja,$,he='For more information on the Hugging Face paths and how they are implemented, please refer to the <a href="/docs/huggingface_hub/guides/hf_file_system">the client library’s documentation on the HfFileSystem</a>.',ba,_,Ua,B,de="You can save a pyarrow Table using <code>pyarrow.parquet.write_table</code> to a local file or to Hugging Face directly.",Ia,G,ue='To save the Table on Hugging Face, you first need to <a href="/docs/huggingface_hub/quick-start#login">Login with your Hugging Face account</a>, for example using:',Za,k,Xa,q,ce='Then you can <a href="/docs/huggingface_hub/quick-start#create-a-repository">create a dataset repository</a>, for example using:',$a,Y,_a,C,ye='Finally, you can use <a href="/docs/huggingface_hub/guides/hf_file_system#integrations">Hugging Face paths</a> in PyArrow:',Ba,W,Ga,R,we="We use <code>use_content_defined_chunking=True</code> to enable faster uploads and downloads from Hugging Face thanks to Xet deduplication (it requires <code>pyarrow&gt;=21.0</code>).",ka,M,Me=`<p>Content defined chunking (CDC) makes the Parquet writer chunk the data pages in a way that makes duplicate data chunked and compressed identically.
Without CDC, the pages are arbitrarily chunked and therefore duplicate data are impossible to detect because of compression.
Thanks to CDC, Parquet uploads and downloads from Hugging Face are faster, since duplicate data are uploaded or downloaded only once.</p>`,qa,V,Je='Find more information about Xet <a href="https://huggingface.co/join/xet" rel="nofollow">here</a>.',Ya,F,Ca,x,fe="You can load a folder with a metadata file containing a field for the names or paths to the images, structured like this:",Wa,v,Ra,H,Te="You can iterate on the images paths like this:",Va,E,Fa,S,ge='Since the dataset is in a <a href="https://huggingface.co/docs/hub/en/datasets-image#additional-columns" rel="nofollow">supported structure</a> (a <code>metadata.parquet</code> file with a <code>file_name</code> field), you can save this dataset to Hugging Face and the Dataset Viewer shows both the metadata and images.',xa,Q,va,N,Ha,A,je="PyArrow has a binary type which allows to have the images bytes in Arrow tables. Therefore it enables saving the dataset as one single Parquet file containing both the images (bytes and path) and the samples metadata:",Ea,z,Sa,L,be="Setting the Image type in the Arrow schema metadata allows other libraries and the Hugging Face Dataset Viewer to know that “image” contains images and not just binary data.",Qa,P,Na,K,Ue="You can load a folder with a metadata file containing a field for the names or paths to the audios, structured like this:",Aa,D,za,O,Ie="You can iterate on the audios paths like this:",La,aa,Pa,ea,Ze='Since the dataset is in a <a href="https://huggingface.co/docs/hub/en/datasets-audio#additional-columns" rel="nofollow">supported structure</a> (a <code>metadata.parquet</code> file with a <code>file_name</code> field), you can save it to Hugging Face, and the Hub Dataset Viewer shows both the metadata and audio.',Ka,ta,Da,sa,Oa,la,Xe="PyArrow has a binary type which allows for having audio bytes in Arrow tables. Therefore, it enables saving the dataset as one single Parquet file containing both the audio (bytes and path) and the samples metadata:",ae,na,ee,pa,$e="Setting the Audio type in the Arrow schema metadata enables other libraries and the Hugging Face Dataset Viewer to recognise that “audio” contains audio data, not just binary data.",te,ia,se,ma,le;return J=new Re({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),f=new ra({props:{title:"PyArrow",local:"pyarrow",headingTag:"h1"}}),g=new ra({props:{title:"Load a Table",local:"load-a-table",headingTag:"h2"}}),b=new y({props:{code:"aW1wb3J0JTIwcHlhcnJvdy5wYXJxdWV0JTIwYXMlMjBwcSUwQXRhYmxlJTIwJTNEJTIwcHEucmVhZF90YWJsZSglMjJwYXRoJTJGdG8lMkZkYXRhLnBhcnF1ZXQlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> pyarrow.parquet <span class="hljs-keyword">as</span> pq
<span class="hljs-meta">&gt;&gt;&gt; </span>table = pq.read_table(<span class="hljs-string">&quot;path/to/data.parquet&quot;</span>)`,wrap:!1}}),I=new y({props:{code:"aW1wb3J0JTIwcHlhcnJvdy5wYXJxdWV0JTIwYXMlMjBwcSUwQXRhYmxlJTIwJTNEJTIwcHEucmVhZF90YWJsZSglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGc3RhbmZvcmRubHAlMkZpbWRiJTJGcGxhaW5fdGV4dCUyRnRyYWluLTAwMDAwLW9mLTAwMDAxLnBhcnF1ZXQlMjIpJTBBdGFibGU=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> pyarrow.parquet <span class="hljs-keyword">as</span> pq
<span class="hljs-meta">&gt;&gt;&gt; </span>table = pq.read_table(<span class="hljs-string">&quot;hf://datasets/stanfordnlp/imdb/plain_text/train-00000-of-00001.parquet&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>table
pyarrow.Table
text: string
label: int64
----
text: [[<span class="hljs-string">&quot;I rented I AM CURIOUS-YELLOW from my video store because of all the controversy that surrounded it (... 1542 chars omitted)&quot;</span>, ...],...,[..., <span class="hljs-string">&quot;The story centers around Barry McKenzie who must go to England if he wishes to claim his inheritan (... 221 chars omitted)&quot;</span>]]
label: [[<span class="hljs-number">0</span>,<span class="hljs-number">0</span>,<span class="hljs-number">0</span>,<span class="hljs-number">0</span>,<span class="hljs-number">0</span>,...,<span class="hljs-number">0</span>,<span class="hljs-number">0</span>,<span class="hljs-number">0</span>,<span class="hljs-number">0</span>,<span class="hljs-number">0</span>],...,[<span class="hljs-number">1</span>,<span class="hljs-number">1</span>,<span class="hljs-number">1</span>,<span class="hljs-number">1</span>,<span class="hljs-number">1</span>,...,<span class="hljs-number">1</span>,<span class="hljs-number">1</span>,<span class="hljs-number">1</span>,<span class="hljs-number">1</span>,<span class="hljs-number">1</span>]]`,wrap:!1}}),X=new y({props:{code:"aW1wb3J0JTIwcHlhcnJvdy5wYXJxdWV0JTIwYXMlMjBwcSUwQXBmJTIwJTNEJTIwcHEuUGFycXVldEZpbGUoJTIyaGYlM0ElMkYlMkZkYXRhc2V0cyUyRnN0YW5mb3JkbmxwJTJGaW1kYiUyRnBsYWluX3RleHQlMkZ0cmFpbi0wMDAwMC1vZi0wMDAwMS5wYXJxdWV0JTIyKSUwQXBmLm1ldGFkYXRhJTBBZm9yJTIwaSUyMGluJTIwcGYubnVtX3Jvd19ncm91cHMlM0ElMEElMjAlMjAlMjAlMjB0YWJsZSUyMCUzRCUyMHBmLnJlYWRfcm93X2dyb3VwKGkpJTBBJTIwJTIwJTIwJTIwLi4u",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> pyarrow.parquet <span class="hljs-keyword">as</span> pq
<span class="hljs-meta">&gt;&gt;&gt; </span>pf = pq.ParquetFile(<span class="hljs-string">&quot;hf://datasets/stanfordnlp/imdb/plain_text/train-00000-of-00001.parquet&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>pf.metadata
&lt;pyarrow._parquet.FileMetaData <span class="hljs-built_in">object</span> at <span class="hljs-number">0x1171b4090</span>&gt;
  created_by: parquet-cpp-arrow version <span class="hljs-number">12.0</span><span class="hljs-number">.0</span>
  num_columns: <span class="hljs-number">2</span>
  num_rows: <span class="hljs-number">25000</span>
  num_row_groups: <span class="hljs-number">25</span>
  format_version: <span class="hljs-number">2.6</span>
  serialized_size: <span class="hljs-number">62036</span>
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> pf.num_row_groups:
<span class="hljs-meta">... </span>    table = pf.read_row_group(i)
<span class="hljs-meta">... </span>    ...`,wrap:!1}}),_=new ra({props:{title:"Save a Table",local:"save-a-table",headingTag:"h2"}}),k=new y({props:{code:"aGYlMjBhdXRoJTIwbG9naW4=",highlighted:'<span class="hljs-attribute">hf auth login</span>',wrap:!1}}),Y=new y({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMEhmQXBpJTBBJTBBSGZBcGkoKS5jcmVhdGVfcmVwbyhyZXBvX2lkJTNEJTIydXNlcm5hbWUlMkZteV9kYXRhc2V0JTIyJTJDJTIwcmVwb190eXBlJTNEJTIyZGF0YXNldCUyMik=",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfApi

HfApi().create_repo(repo_id=<span class="hljs-string">&quot;username/my_dataset&quot;</span>, repo_type=<span class="hljs-string">&quot;dataset&quot;</span>)`,wrap:!1}}),W=new y({props:{code:"aW1wb3J0JTIwcHlhcnJvdy5wYXJxdWV0JTIwYXMlMjBwcSUwQSUwQXBxLndyaXRlX3RhYmxlKHRhYmxlJTJDJTIwJTIyaGYlM0ElMkYlMkZkYXRhc2V0cyUyRnVzZXJuYW1lJTJGbXlfZGF0YXNldCUyRmltZGIucGFycXVldCUyMiUyQyUyMHVzZV9jb250ZW50X2RlZmluZWRfY2h1bmtpbmclM0RUcnVlKSUwQSUwQSUyMyUyMG9yJTIwd3JpdGUlMjBpbiUyMHNlcGFyYXRlJTIwZmlsZXMlMjBpZiUyMHRoZSUyMGRhdGFzZXQlMjBoYXMlMjB0cmFpbiUyRnZhbGlkYXRpb24lMkZ0ZXN0JTIwc3BsaXRzJTBBcHEud3JpdGVfdGFibGUodGFibGVfdHJhaW4lMkMlMjAlMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZteV9kYXRhc2V0JTJGdHJhaW4ucGFycXVldCUyMiUyQyUyMHVzZV9jb250ZW50X2RlZmluZWRfY2h1bmtpbmclM0RUcnVlKSUwQXBxLndyaXRlX3RhYmxlKHRhYmxlX3ZhbGlkJTJDJTIwJTIyaGYlM0ElMkYlMkZkYXRhc2V0cyUyRnVzZXJuYW1lJTJGbXlfZGF0YXNldCUyRnZhbGlkYXRpb24ucGFycXVldCUyMiUyQyUyMHVzZV9jb250ZW50X2RlZmluZWRfY2h1bmtpbmclM0RUcnVlKSUwQXBxLndyaXRlX3RhYmxlKHRhYmxlX3Rlc3QlMjAlMkMlMjAlMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZteV9kYXRhc2V0JTJGdGVzdC5wYXJxdWV0JTIyJTJDJTIwdXNlX2NvbnRlbnRfZGVmaW5lZF9jaHVua2luZyUzRFRydWUp",highlighted:`<span class="hljs-keyword">import</span> pyarrow.parquet <span class="hljs-keyword">as</span> pq

pq.write_table(table, <span class="hljs-string">&quot;hf://datasets/username/my_dataset/imdb.parquet&quot;</span>, use_content_defined_chunking=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># or write in separate files if the dataset has train/validation/test splits</span>
pq.write_table(table_train, <span class="hljs-string">&quot;hf://datasets/username/my_dataset/train.parquet&quot;</span>, use_content_defined_chunking=<span class="hljs-literal">True</span>)
pq.write_table(table_valid, <span class="hljs-string">&quot;hf://datasets/username/my_dataset/validation.parquet&quot;</span>, use_content_defined_chunking=<span class="hljs-literal">True</span>)
pq.write_table(table_test , <span class="hljs-string">&quot;hf://datasets/username/my_dataset/test.parquet&quot;</span>, use_content_defined_chunking=<span class="hljs-literal">True</span>)`,wrap:!1}}),F=new ra({props:{title:"Use Images",local:"use-images",headingTag:"h2"}}),v=new y({props:{code:"JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwLi4u",highlighted:`Example <span class="hljs-number">1</span>:            Example <span class="hljs-number">2</span>:
folder/               folder/
├── metadata<span class="hljs-selector-class">.parquet</span>  ├── metadata<span class="hljs-selector-class">.parquet</span>
├── img000<span class="hljs-selector-class">.png</span>        └── images
├── img001<span class="hljs-selector-class">.png</span>            ├── img000<span class="hljs-selector-class">.png</span>
...                       ...
└── imgNNN<span class="hljs-selector-class">.png</span>            └── imgNNN.png`,wrap:!1}}),E=new y({props:{code:"ZnJvbSUyMHBhdGhsaWIlMjBpbXBvcnQlMjBQYXRoJTBBaW1wb3J0JTIwcHlhcnJvdyUyMGFzJTIwcHElMEElMEFmb2xkZXJfcGF0aCUyMCUzRCUyMFBhdGgoJTIycGF0aCUyRnRvJTJGZm9sZGVyJTIyKSUwQXRhYmxlJTIwJTNEJTIwcHEucmVhZF90YWJsZShmb2xkZXJfcGF0aCUyMCUyQiUyMCUyMm1ldGFkYXRhLnBhcnF1ZXQlMjIpJTBBZm9yJTIwZmlsZV9uYW1lJTIwaW4lMjB0YWJsZSU1QiUyMmZpbGVfbmFtZSUyMiU1RC50b19weWxpc3QoKSUzQSUwQSUyMCUyMCUyMCUyMGltYWdlX3BhdGglMjAlM0QlMjBmb2xkZXJfcGF0aCUyMCUyRiUyMGZpbGVfbmFtZSUwQSUyMCUyMCUyMCUyMC4uLg==",highlighted:`<span class="hljs-keyword">from</span> pathlib <span class="hljs-keyword">import</span> Path
<span class="hljs-keyword">import</span> pyarrow <span class="hljs-keyword">as</span> pq

folder_path = Path(<span class="hljs-string">&quot;path/to/folder&quot;</span>)
table = pq.read_table(folder_path + <span class="hljs-string">&quot;metadata.parquet&quot;</span>)
<span class="hljs-keyword">for</span> file_name <span class="hljs-keyword">in</span> table[<span class="hljs-string">&quot;file_name&quot;</span>].to_pylist():
    image_path = folder_path / file_name
    ...`,wrap:!1}}),Q=new y({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMEhmQXBpJTBBYXBpJTIwJTNEJTIwSGZBcGkoKSUwQSUwQWFwaS51cGxvYWRfZm9sZGVyKCUwQSUyMCUyMCUyMCUyMGZvbGRlcl9wYXRoJTNEZm9sZGVyX3BhdGglMkMlMEElMjAlMjAlMjAlMjByZXBvX2lkJTNEJTIydXNlcm5hbWUlMkZteV9pbWFnZV9kYXRhc2V0JTIyJTJDJTBBJTIwJTIwJTIwJTIwcmVwb190eXBlJTNEJTIyZGF0YXNldCUyMiUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfApi
api = HfApi()

api.upload_folder(
    folder_path=folder_path,
    repo_id=<span class="hljs-string">&quot;username/my_image_dataset&quot;</span>,
    repo_type=<span class="hljs-string">&quot;dataset&quot;</span>,
)`,wrap:!1}}),N=new ra({props:{title:"Embed Images inside Parquet",local:"embed-images-inside-parquet",headingTag:"h3"}}),z=new y({props:{code:"aW1wb3J0JTIwcHlhcnJvdyUyMGFzJTIwcGElMEFpbXBvcnQlMjBweWFycm93LnBhcnF1ZXQlMjBhcyUyMHBxJTBBJTBBJTIzJTIwRW1iZWQlMjB0aGUlMjBpbWFnZSUyMGJ5dGVzJTIwaW4lMjBBcnJvdyUwQWltYWdlX2FycmF5JTIwJTNEJTIwcGEuYXJyYXkoJTVCJTBBJTIwJTIwJTIwJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyYnl0ZXMlMjIlM0ElMjAoZm9sZGVyX3BhdGglMjAlMkYlMjBmaWxlX25hbWUpLnJlYWRfYnl0ZXMoKSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnBhdGglMjIlM0ElMjBmaWxlX25hbWUlMkMlMEElMjAlMjAlMjAlMjAlN0QlMEElMjAlMjAlMjAlMjBmb3IlMjBmaWxlX25hbWUlMjBpbiUyMHRhYmxlJTVCJTIyZmlsZV9uYW1lJTIyJTVELnRvX3B5bGlzdCgpJTBBJTVEKSUwQXRhYmxlLmFwcGVuZF9jb2x1bW4oJTIyaW1hZ2UlMjIlMkMlMjBpbWFnZV9hcnJheSklMEElMEElMjMlMjAoT3B0aW9uYWwpJTIwU2V0JTIwdGhlJTIwSEYlMjBJbWFnZSUyMHR5cGUlMjBmb3IlMjB0aGUlMjBEYXRhc2V0JTIwVmlld2VyJTIwYW5kJTIwdGhlJTIwJTYwZGF0YXNldHMlNjAlMjBsaWJyYXJ5JTBBZmVhdHVyZXMlMjAlM0QlMjAlN0IlMjJpbWFnZSUyMiUzQSUyMCU3QiUyMl90eXBlJTIyJTNBJTIwJTIySW1hZ2UlMjIlN0QlN0QlMjAlMjAlMjMlMjBvciUyMHVzaW5nJTIwZGF0YXNldHMuRmVhdHVyZXMoLi4uKS50b19kaWN0KCklMEFzY2hlbWFfbWV0YWRhdGElMjAlM0QlMjAlN0IlMjJodWdnaW5nZmFjZSUyMiUzQSUyMCU3QiUyMmRhdGFzZXRfaW5mbyUyMiUzQSUyMCU3QiUyMmZlYXR1cmVzJTIyJTNBJTIwZmVhdHVyZXMlN0QlN0QlN0QlMEF0YWJsZSUyMCUzRCUyMHRhYmxlLnJlcGxhY2Vfc2NoZW1hX21ldGFkYXRhKHNjaGVtYV9tZXRhZGF0YSklMEElMEElMjMlMjBTYXZlJTIwdG8lMjBQYXJxdWV0JTBBJTIzJTIwKE9wdGlvbmFsKSUyMHdpdGglMjB1c2VfY29udGVudF9kZWZpbmVkX2NodW5raW5nJTIwZm9yJTIwZmFzdGVyJTIwdXBsb2FkcyUyMGFuZCUyMGRvd25sb2FkcyUwQXBxLndyaXRlX3RhYmxlKHRhYmxlJTJDJTIwJTIyZGF0YS5wYXJxdWV0JTIyJTJDJTIwdXNlX2NvbnRlbnRfZGVmaW5lZF9jaHVua2luZyUzRFRydWUp",highlighted:`<span class="hljs-keyword">import</span> pyarrow <span class="hljs-keyword">as</span> pa
<span class="hljs-keyword">import</span> pyarrow.parquet <span class="hljs-keyword">as</span> pq

<span class="hljs-comment"># Embed the image bytes in Arrow</span>
image_array = pa.array([
    {
        <span class="hljs-string">&quot;bytes&quot;</span>: (folder_path / file_name).read_bytes(),
        <span class="hljs-string">&quot;path&quot;</span>: file_name,
    }
    <span class="hljs-keyword">for</span> file_name <span class="hljs-keyword">in</span> table[<span class="hljs-string">&quot;file_name&quot;</span>].to_pylist()
])
table.append_column(<span class="hljs-string">&quot;image&quot;</span>, image_array)

<span class="hljs-comment"># (Optional) Set the HF Image type for the Dataset Viewer and the \`datasets\` library</span>
features = {<span class="hljs-string">&quot;image&quot;</span>: {<span class="hljs-string">&quot;_type&quot;</span>: <span class="hljs-string">&quot;Image&quot;</span>}}  <span class="hljs-comment"># or using datasets.Features(...).to_dict()</span>
schema_metadata = {<span class="hljs-string">&quot;huggingface&quot;</span>: {<span class="hljs-string">&quot;dataset_info&quot;</span>: {<span class="hljs-string">&quot;features&quot;</span>: features}}}
table = table.replace_schema_metadata(schema_metadata)

<span class="hljs-comment"># Save to Parquet</span>
<span class="hljs-comment"># (Optional) with use_content_defined_chunking for faster uploads and downloads</span>
pq.write_table(table, <span class="hljs-string">&quot;data.parquet&quot;</span>, use_content_defined_chunking=<span class="hljs-literal">True</span>)`,wrap:!1}}),P=new ra({props:{title:"Use Audios",local:"use-audios",headingTag:"h2"}}),D=new y({props:{code:"JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwLi4u",highlighted:`Example <span class="hljs-number">1</span>:            Example <span class="hljs-number">2</span>:
folder/               folder/
├── metadata<span class="hljs-selector-class">.parquet</span>  ├── metadata<span class="hljs-selector-class">.parquet</span>
├── rec000<span class="hljs-selector-class">.wav</span>        └── audios
├── rec001<span class="hljs-selector-class">.wav</span>            ├── rec000<span class="hljs-selector-class">.wav</span>
...                       ...
└── recNNN<span class="hljs-selector-class">.wav</span>            └── recNNN.wav`,wrap:!1}}),aa=new y({props:{code:"ZnJvbSUyMHBhdGhsaWIlMjBpbXBvcnQlMjBQYXRoJTBBaW1wb3J0JTIwcHlhcnJvdyUyMGFzJTIwcHElMEElMEFmb2xkZXJfcGF0aCUyMCUzRCUyMFBhdGgoJTIycGF0aCUyRnRvJTJGZm9sZGVyJTIyKSUwQXRhYmxlJTIwJTNEJTIwcHEucmVhZF90YWJsZShmb2xkZXJfcGF0aCUyMCUyQiUyMCUyMm1ldGFkYXRhLnBhcnF1ZXQlMjIpJTBBZm9yJTIwZmlsZV9uYW1lJTIwaW4lMjB0YWJsZSU1QiUyMmZpbGVfbmFtZSUyMiU1RC50b19weWxpc3QoKSUzQSUwQSUyMCUyMCUyMCUyMGF1ZGlvX3BhdGglMjAlM0QlMjBmb2xkZXJfcGF0aCUyMCUyRiUyMGZpbGVfbmFtZSUwQSUyMCUyMCUyMCUyMC4uLg==",highlighted:`<span class="hljs-keyword">from</span> pathlib <span class="hljs-keyword">import</span> Path
<span class="hljs-keyword">import</span> pyarrow <span class="hljs-keyword">as</span> pq

folder_path = Path(<span class="hljs-string">&quot;path/to/folder&quot;</span>)
table = pq.read_table(folder_path + <span class="hljs-string">&quot;metadata.parquet&quot;</span>)
<span class="hljs-keyword">for</span> file_name <span class="hljs-keyword">in</span> table[<span class="hljs-string">&quot;file_name&quot;</span>].to_pylist():
    audio_path = folder_path / file_name
    ...`,wrap:!1}}),ta=new y({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMEhmQXBpJTBBYXBpJTIwJTNEJTIwSGZBcGkoKSUwQSUwQWFwaS51cGxvYWRfZm9sZGVyKCUwQSUyMCUyMCUyMCUyMGZvbGRlcl9wYXRoJTNEZm9sZGVyX3BhdGglMkMlMEElMjAlMjAlMjAlMjByZXBvX2lkJTNEJTIydXNlcm5hbWUlMkZteV9hdWRpb19kYXRhc2V0JTIyJTJDJTBBJTIwJTIwJTIwJTIwcmVwb190eXBlJTNEJTIyZGF0YXNldCUyMiUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfApi
api = HfApi()

api.upload_folder(
    folder_path=folder_path,
    repo_id=<span class="hljs-string">&quot;username/my_audio_dataset&quot;</span>,
    repo_type=<span class="hljs-string">&quot;dataset&quot;</span>,
)`,wrap:!1}}),sa=new ra({props:{title:"Embed Audio inside Parquet",local:"embed-audio-inside-parquet",headingTag:"h3"}}),na=new y({props:{code:"aW1wb3J0JTIwcHlhcnJvdyUyMGFzJTIwcGElMEFpbXBvcnQlMjBweWFycm93LnBhcnF1ZXQlMjBhcyUyMHBxJTBBJTBBJTIzJTIwRW1iZWQlMjB0aGUlMjBhdWRpbyUyMGJ5dGVzJTIwaW4lMjBBcnJvdyUwQWF1ZGlvX2FycmF5JTIwJTNEJTIwcGEuYXJyYXkoJTVCJTBBJTIwJTIwJTIwJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyYnl0ZXMlMjIlM0ElMjAoZm9sZGVyX3BhdGglMjAlMkYlMjBmaWxlX25hbWUpLnJlYWRfYnl0ZXMoKSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnBhdGglMjIlM0ElMjBmaWxlX25hbWUlMkMlMEElMjAlMjAlMjAlMjAlN0QlMEElMjAlMjAlMjAlMjBmb3IlMjBmaWxlX25hbWUlMjBpbiUyMHRhYmxlJTVCJTIyZmlsZV9uYW1lJTIyJTVELnRvX3B5bGlzdCgpJTBBJTVEKSUwQXRhYmxlLmFwcGVuZF9jb2x1bW4oJTIyYXVkaW8lMjIlMkMlMjBhdWRpb19hcnJheSklMEElMEElMjMlMjAoT3B0aW9uYWwpJTIwU2V0JTIwdGhlJTIwSEYlMjBBdWRpbyUyMHR5cGUlMjBmb3IlMjB0aGUlMjBEYXRhc2V0JTIwVmlld2VyJTIwYW5kJTIwdGhlJTIwJTYwZGF0YXNldHMlNjAlMjBsaWJyYXJ5JTBBZmVhdHVyZXMlMjAlM0QlMjAlN0IlMjJhdWRpbyUyMiUzQSUyMCU3QiUyMl90eXBlJTIyJTNBJTIwJTIyQXVkaW8lMjIlN0QlN0QlMjAlMjAlMjMlMjBvciUyMHVzaW5nJTIwZGF0YXNldHMuRmVhdHVyZXMoLi4uKS50b19kaWN0KCklMEFzY2hlbWFfbWV0YWRhdGElMjAlM0QlMjAlN0IlMjJodWdnaW5nZmFjZSUyMiUzQSUyMCU3QiUyMmRhdGFzZXRfaW5mbyUyMiUzQSUyMCU3QiUyMmZlYXR1cmVzJTIyJTNBJTIwZmVhdHVyZXMlN0QlN0QlN0QlMEF0YWJsZSUyMCUzRCUyMHRhYmxlLnJlcGxhY2Vfc2NoZW1hX21ldGFkYXRhKHNjaGVtYV9tZXRhZGF0YSklMEElMEElMjMlMjBTYXZlJTIwdG8lMjBQYXJxdWV0JTBBJTIzJTIwKE9wdGlvbmFsKSUyMHdpdGglMjB1c2VfY29udGVudF9kZWZpbmVkX2NodW5raW5nJTIwZm9yJTIwZmFzdGVyJTIwdXBsb2FkcyUyMGFuZCUyMGRvd25sb2FkcyUwQXBxLndyaXRlX3RhYmxlKHRhYmxlJTJDJTIwJTIyZGF0YS5wYXJxdWV0JTIyJTJDJTIwdXNlX2NvbnRlbnRfZGVmaW5lZF9jaHVua2luZyUzRFRydWUp",highlighted:`<span class="hljs-keyword">import</span> pyarrow <span class="hljs-keyword">as</span> pa
<span class="hljs-keyword">import</span> pyarrow.parquet <span class="hljs-keyword">as</span> pq

<span class="hljs-comment"># Embed the audio bytes in Arrow</span>
audio_array = pa.array([
    {
        <span class="hljs-string">&quot;bytes&quot;</span>: (folder_path / file_name).read_bytes(),
        <span class="hljs-string">&quot;path&quot;</span>: file_name,
    }
    <span class="hljs-keyword">for</span> file_name <span class="hljs-keyword">in</span> table[<span class="hljs-string">&quot;file_name&quot;</span>].to_pylist()
])
table.append_column(<span class="hljs-string">&quot;audio&quot;</span>, audio_array)

<span class="hljs-comment"># (Optional) Set the HF Audio type for the Dataset Viewer and the \`datasets\` library</span>
features = {<span class="hljs-string">&quot;audio&quot;</span>: {<span class="hljs-string">&quot;_type&quot;</span>: <span class="hljs-string">&quot;Audio&quot;</span>}}  <span class="hljs-comment"># or using datasets.Features(...).to_dict()</span>
schema_metadata = {<span class="hljs-string">&quot;huggingface&quot;</span>: {<span class="hljs-string">&quot;dataset_info&quot;</span>: {<span class="hljs-string">&quot;features&quot;</span>: features}}}
table = table.replace_schema_metadata(schema_metadata)

<span class="hljs-comment"># Save to Parquet</span>
<span class="hljs-comment"># (Optional) with use_content_defined_chunking for faster uploads and downloads</span>
pq.write_table(table, <span class="hljs-string">&quot;data.parquet&quot;</span>, use_content_defined_chunking=<span class="hljs-literal">True</span>)`,wrap:!1}}),ia=new Ve({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/datasets-pyarrow.md"}}),{c(){w=p("meta"),ha=l(),oa=p("p"),da=l(),r(J.$$.fragment),ua=l(),r(f.$$.fragment),ca=l(),T=p("p"),T.innerHTML=ie,ya=l(),r(g.$$.fragment),wa=l(),j=p("p"),j.textContent=re,Ma=l(),r(b.$$.fragment),Ja=l(),U=p("p"),U.innerHTML=oe,fa=l(),r(I.$$.fragment),Ta=l(),Z=p("p"),Z.textContent=me,ga=l(),r(X.$$.fragment),ja=l(),$=p("p"),$.innerHTML=he,ba=l(),r(_.$$.fragment),Ua=l(),B=p("p"),B.innerHTML=de,Ia=l(),G=p("p"),G.innerHTML=ue,Za=l(),r(k.$$.fragment),Xa=l(),q=p("p"),q.innerHTML=ce,$a=l(),r(Y.$$.fragment),_a=l(),C=p("p"),C.innerHTML=ye,Ba=l(),r(W.$$.fragment),Ga=l(),R=p("p"),R.innerHTML=we,ka=l(),M=p("blockquote"),M.innerHTML=Me,qa=l(),V=p("p"),V.innerHTML=Je,Ya=l(),r(F.$$.fragment),Ca=l(),x=p("p"),x.textContent=fe,Wa=l(),r(v.$$.fragment),Ra=l(),H=p("p"),H.textContent=Te,Va=l(),r(E.$$.fragment),Fa=l(),S=p("p"),S.innerHTML=ge,xa=l(),r(Q.$$.fragment),va=l(),r(N.$$.fragment),Ha=l(),A=p("p"),A.textContent=je,Ea=l(),r(z.$$.fragment),Sa=l(),L=p("p"),L.textContent=be,Qa=l(),r(P.$$.fragment),Na=l(),K=p("p"),K.textContent=Ue,Aa=l(),r(D.$$.fragment),za=l(),O=p("p"),O.textContent=Ie,La=l(),r(aa.$$.fragment),Pa=l(),ea=p("p"),ea.innerHTML=Ze,Ka=l(),r(ta.$$.fragment),Da=l(),r(sa.$$.fragment),Oa=l(),la=p("p"),la.textContent=Xe,ae=l(),r(na.$$.fragment),ee=l(),pa=p("p"),pa.textContent=$e,te=l(),r(ia.$$.fragment),se=l(),ma=p("p"),this.h()},l(a){const e=Ce("svelte-u9bgzb",document.head);w=i(e,"META",{name:!0,content:!0}),e.forEach(t),ha=n(a),oa=i(a,"P",{}),_e(oa).forEach(t),da=n(a),o(J.$$.fragment,a),ua=n(a),o(f.$$.fragment,a),ca=n(a),T=i(a,"P",{"data-svelte-h":!0}),c(T)!=="svelte-56vgn"&&(T.innerHTML=ie),ya=n(a),o(g.$$.fragment,a),wa=n(a),j=i(a,"P",{"data-svelte-h":!0}),c(j)!=="svelte-1263bgm"&&(j.textContent=re),Ma=n(a),o(b.$$.fragment,a),Ja=n(a),U=i(a,"P",{"data-svelte-h":!0}),c(U)!=="svelte-1othel7"&&(U.innerHTML=oe),fa=n(a),o(I.$$.fragment,a),Ta=n(a),Z=i(a,"P",{"data-svelte-h":!0}),c(Z)!=="svelte-u7zvq0"&&(Z.textContent=me),ga=n(a),o(X.$$.fragment,a),ja=n(a),$=i(a,"P",{"data-svelte-h":!0}),c($)!=="svelte-smuvdf"&&($.innerHTML=he),ba=n(a),o(_.$$.fragment,a),Ua=n(a),B=i(a,"P",{"data-svelte-h":!0}),c(B)!=="svelte-15a6az3"&&(B.innerHTML=de),Ia=n(a),G=i(a,"P",{"data-svelte-h":!0}),c(G)!=="svelte-1c59jr5"&&(G.innerHTML=ue),Za=n(a),o(k.$$.fragment,a),Xa=n(a),q=i(a,"P",{"data-svelte-h":!0}),c(q)!=="svelte-1cj3kqp"&&(q.innerHTML=ce),$a=n(a),o(Y.$$.fragment,a),_a=n(a),C=i(a,"P",{"data-svelte-h":!0}),c(C)!=="svelte-110fpi"&&(C.innerHTML=ye),Ba=n(a),o(W.$$.fragment,a),Ga=n(a),R=i(a,"P",{"data-svelte-h":!0}),c(R)!=="svelte-cz486o"&&(R.innerHTML=we),ka=n(a),M=i(a,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),c(M)!=="svelte-1jam3g9"&&(M.innerHTML=Me),qa=n(a),V=i(a,"P",{"data-svelte-h":!0}),c(V)!=="svelte-10e2td"&&(V.innerHTML=Je),Ya=n(a),o(F.$$.fragment,a),Ca=n(a),x=i(a,"P",{"data-svelte-h":!0}),c(x)!=="svelte-i1x932"&&(x.textContent=fe),Wa=n(a),o(v.$$.fragment,a),Ra=n(a),H=i(a,"P",{"data-svelte-h":!0}),c(H)!=="svelte-1wo91tc"&&(H.textContent=Te),Va=n(a),o(E.$$.fragment,a),Fa=n(a),S=i(a,"P",{"data-svelte-h":!0}),c(S)!=="svelte-17xhcwl"&&(S.innerHTML=ge),xa=n(a),o(Q.$$.fragment,a),va=n(a),o(N.$$.fragment,a),Ha=n(a),A=i(a,"P",{"data-svelte-h":!0}),c(A)!=="svelte-1ypsa1z"&&(A.textContent=je),Ea=n(a),o(z.$$.fragment,a),Sa=n(a),L=i(a,"P",{"data-svelte-h":!0}),c(L)!=="svelte-snsv43"&&(L.textContent=be),Qa=n(a),o(P.$$.fragment,a),Na=n(a),K=i(a,"P",{"data-svelte-h":!0}),c(K)!=="svelte-14ssghx"&&(K.textContent=Ue),Aa=n(a),o(D.$$.fragment,a),za=n(a),O=i(a,"P",{"data-svelte-h":!0}),c(O)!=="svelte-9uthgh"&&(O.textContent=Ie),La=n(a),o(aa.$$.fragment,a),Pa=n(a),ea=i(a,"P",{"data-svelte-h":!0}),c(ea)!=="svelte-gmhdi8"&&(ea.innerHTML=Ze),Ka=n(a),o(ta.$$.fragment,a),Da=n(a),o(sa.$$.fragment,a),Oa=n(a),la=i(a,"P",{"data-svelte-h":!0}),c(la)!=="svelte-1w1yrqx"&&(la.textContent=Xe),ae=n(a),o(na.$$.fragment,a),ee=n(a),pa=i(a,"P",{"data-svelte-h":!0}),c(pa)!=="svelte-i7mbpg"&&(pa.textContent=$e),te=n(a),o(ia.$$.fragment,a),se=n(a),ma=i(a,"P",{}),_e(ma).forEach(t),this.h()},h(){ne(w,"name","hf:doc:metadata"),ne(w,"content",xe),ne(M,"class","tip")},m(a,e){We(document.head,w),s(a,ha,e),s(a,oa,e),s(a,da,e),m(J,a,e),s(a,ua,e),m(f,a,e),s(a,ca,e),s(a,T,e),s(a,ya,e),m(g,a,e),s(a,wa,e),s(a,j,e),s(a,Ma,e),m(b,a,e),s(a,Ja,e),s(a,U,e),s(a,fa,e),m(I,a,e),s(a,Ta,e),s(a,Z,e),s(a,ga,e),m(X,a,e),s(a,ja,e),s(a,$,e),s(a,ba,e),m(_,a,e),s(a,Ua,e),s(a,B,e),s(a,Ia,e),s(a,G,e),s(a,Za,e),m(k,a,e),s(a,Xa,e),s(a,q,e),s(a,$a,e),m(Y,a,e),s(a,_a,e),s(a,C,e),s(a,Ba,e),m(W,a,e),s(a,Ga,e),s(a,R,e),s(a,ka,e),s(a,M,e),s(a,qa,e),s(a,V,e),s(a,Ya,e),m(F,a,e),s(a,Ca,e),s(a,x,e),s(a,Wa,e),m(v,a,e),s(a,Ra,e),s(a,H,e),s(a,Va,e),m(E,a,e),s(a,Fa,e),s(a,S,e),s(a,xa,e),m(Q,a,e),s(a,va,e),m(N,a,e),s(a,Ha,e),s(a,A,e),s(a,Ea,e),m(z,a,e),s(a,Sa,e),s(a,L,e),s(a,Qa,e),m(P,a,e),s(a,Na,e),s(a,K,e),s(a,Aa,e),m(D,a,e),s(a,za,e),s(a,O,e),s(a,La,e),m(aa,a,e),s(a,Pa,e),s(a,ea,e),s(a,Ka,e),m(ta,a,e),s(a,Da,e),m(sa,a,e),s(a,Oa,e),s(a,la,e),s(a,ae,e),m(na,a,e),s(a,ee,e),s(a,pa,e),s(a,te,e),m(ia,a,e),s(a,se,e),s(a,ma,e),le=!0},p:Ge,i(a){le||(h(J.$$.fragment,a),h(f.$$.fragment,a),h(g.$$.fragment,a),h(b.$$.fragment,a),h(I.$$.fragment,a),h(X.$$.fragment,a),h(_.$$.fragment,a),h(k.$$.fragment,a),h(Y.$$.fragment,a),h(W.$$.fragment,a),h(F.$$.fragment,a),h(v.$$.fragment,a),h(E.$$.fragment,a),h(Q.$$.fragment,a),h(N.$$.fragment,a),h(z.$$.fragment,a),h(P.$$.fragment,a),h(D.$$.fragment,a),h(aa.$$.fragment,a),h(ta.$$.fragment,a),h(sa.$$.fragment,a),h(na.$$.fragment,a),h(ia.$$.fragment,a),le=!0)},o(a){d(J.$$.fragment,a),d(f.$$.fragment,a),d(g.$$.fragment,a),d(b.$$.fragment,a),d(I.$$.fragment,a),d(X.$$.fragment,a),d(_.$$.fragment,a),d(k.$$.fragment,a),d(Y.$$.fragment,a),d(W.$$.fragment,a),d(F.$$.fragment,a),d(v.$$.fragment,a),d(E.$$.fragment,a),d(Q.$$.fragment,a),d(N.$$.fragment,a),d(z.$$.fragment,a),d(P.$$.fragment,a),d(D.$$.fragment,a),d(aa.$$.fragment,a),d(ta.$$.fragment,a),d(sa.$$.fragment,a),d(na.$$.fragment,a),d(ia.$$.fragment,a),le=!1},d(a){a&&(t(ha),t(oa),t(da),t(ua),t(ca),t(T),t(ya),t(wa),t(j),t(Ma),t(Ja),t(U),t(fa),t(Ta),t(Z),t(ga),t(ja),t($),t(ba),t(Ua),t(B),t(Ia),t(G),t(Za),t(Xa),t(q),t($a),t(_a),t(C),t(Ba),t(Ga),t(R),t(ka),t(M),t(qa),t(V),t(Ya),t(Ca),t(x),t(Wa),t(Ra),t(H),t(Va),t(Fa),t(S),t(xa),t(va),t(Ha),t(A),t(Ea),t(Sa),t(L),t(Qa),t(Na),t(K),t(Aa),t(za),t(O),t(La),t(Pa),t(ea),t(Ka),t(Da),t(Oa),t(la),t(ae),t(ee),t(pa),t(te),t(se),t(ma)),t(w),u(J,a),u(f,a),u(g,a),u(b,a),u(I,a),u(X,a),u(_,a),u(k,a),u(Y,a),u(W,a),u(F,a),u(v,a),u(E,a),u(Q,a),u(N,a),u(z,a),u(P,a),u(D,a),u(aa,a),u(ta,a),u(sa,a),u(na,a),u(ia,a)}}}const xe='{"title":"PyArrow","local":"pyarrow","sections":[{"title":"Load a Table","local":"load-a-table","sections":[],"depth":2},{"title":"Save a Table","local":"save-a-table","sections":[],"depth":2},{"title":"Use Images","local":"use-images","sections":[{"title":"Embed Images inside Parquet","local":"embed-images-inside-parquet","sections":[],"depth":3}],"depth":2},{"title":"Use Audios","local":"use-audios","sections":[{"title":"Embed Audio inside Parquet","local":"embed-audio-inside-parquet","sections":[],"depth":3}],"depth":2}],"depth":1}';function ve(pe){return ke(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ae extends qe{constructor(w){super(),Ye(this,w,ve,Fe,Be,{})}}export{Ae as component};
