import{s as Kt,n as Ot,o as te}from"../chunks/scheduler.9cda857a.js";import{S as ee,i as ae,e as i,s as l,c as p,h as se,a as m,d as a,b as n,f as qt,g as f,j as d,k as N,l as le,m as s,n as o,t as r,o as u,p as h}from"../chunks/index.003612b0.js";import{C as ne}from"../chunks/CopyLLMTxtMenu.44d08a39.js";import{C as _t}from"../chunks/CodeBlock.a90632f1.js";import{H as B,E as ie}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.e5cfda8c.js";function me(Jt){let g,E,V,q,M,K,v,O,$,Wt=`<a href="https://github.com/webdataset/webdataset" rel="nofollow">WebDataset</a> is a library for writing I/O pipelines for large datasets.
Its sequential I/O and sharding features make it especially useful for streaming large-scale datasets to a DataLoader.`,tt,C,et,y,Gt=`A WebDataset file is a TAR archive containing a series of data files.
All successive data files with the same prefix are considered to be part of the same example (e.g., an image/audio file and its label or metadata):`,at,c,Lt='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-format.png"/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-format-dark.png"/>',st,x,Ht="Labels and metadata can be in a <code>.json</code> file, in a <code>.txt</code> (for a caption, a description), or in a <code>.cls</code> (for a class index).",lt,k,Zt=`A large scale WebDataset is made of many files called shards, where each shard is a TAR archive.
Each shard is often ~1GB but the full dataset can be multiple terabytes!`,nt,T,it,j,Rt="WebDataset is designed for multimodal datasets, i.e. for image, audio and/or video datasets.",mt,U,Dt="Indeed, since media files tend to be quite big, WebDataset’s sequential I/O enables large reads and buffering, resulting in the best data loading speed.",dt,_,Xt="Here is a non-exhaustive list of supported data formats:",pt,J,zt="<li>image: jpeg, png, tiff</li> <li>audio: mp3, m4a, wav, flac</li> <li>video: mp4, mov, avi</li> <li>other: npy, npz</li>",ft,W,At='The full list evolves over time and depends on the implementation. For example, you can find which formats the <code>webdataset</code> package supports in the source code <a href="https://github.com/webdataset/webdataset/blob/main/src/webdataset/autodecode.py" rel="nofollow">here</a>.',ot,G,rt,L,It=`Streaming TAR archives is fast because it reads contiguous chunks of data.
It can be orders of magnitude faster than reading separate data files one by one.`,ut,H,Pt="WebDataset streaming offers high-speed performance both when reading from disk and from cloud storage, which makes it an ideal format to feed to a DataLoader:",ht,b,Qt='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-stream.png"/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-stream-dark.png"/>',gt,Z,Ft='For example here is how to stream the <a href="https://huggingface.co/datasets/timm/imagenet-12k-wds" rel="nofollow">timm/imagenet-12k-wds</a> dataset directly from Hugging Face:',ct,R,St='First you need to <a href="/docs/huggingface_hub/quick-start#login">Login with your Hugging Face account</a>, for example using:',bt,D,wt,X,Vt="And then you can stream the dataset with WebDataset:",Mt,z,vt,A,$t,I,Yt=`Generally, datasets in WebDataset formats are already shuffled and ready to feed to a DataLoader.
But you can still reshuffle the data with WebDataset’s approximate shuffling.`,Ct,P,Nt="In addition to shuffling the list of shards, WebDataset uses a buffer to shuffle a dataset without any cost to speed:",yt,w,Bt='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-shuffle.png"/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-shuffle-dark.png"/>',xt,Q,Et="To shuffle a list of sharded files and randomly sample from the shuffle buffer:",kt,F,Tt,S,jt,Y,Ut;return M=new ne({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),v=new B({props:{title:"WebDataset",local:"webdataset",headingTag:"h1"}}),C=new B({props:{title:"The WebDataset format",local:"the-webdataset-format",headingTag:"h2"}}),T=new B({props:{title:"Multimodal support",local:"multimodal-support",headingTag:"h2"}}),G=new B({props:{title:"Streaming",local:"streaming",headingTag:"h2"}}),D=new _t({props:{code:"aGYlMjBhdXRoJTIwbG9naW4=",highlighted:'<span class="hljs-attribute">hf auth login</span>',wrap:!1}}),z=new _t({props:{code:"aW1wb3J0JTIwd2ViZGF0YXNldCUyMGFzJTIwd2RzJTBBZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMGdldF90b2tlbiUwQWZyb20lMjB0b3JjaC51dGlscy5kYXRhJTIwaW1wb3J0JTIwRGF0YUxvYWRlciUwQSUwQWhmX3Rva2VuJTIwJTNEJTIwZ2V0X3Rva2VuKCklMEF1cmwlMjAlM0QlMjAlMjJodHRwcyUzQSUyRiUyRmh1Z2dpbmdmYWNlLmNvJTJGZGF0YXNldHMlMkZ0aW1tJTJGaW1hZ2VuZXQtMTJrLXdkcyUyRnJlc29sdmUlMkZtYWluJTJGaW1hZ2VuZXQxMmstdHJhaW4tJTdCJTdCMDAwMC4uMTAyMyU3RCU3RC50YXIlMjIlMEF1cmwlMjAlM0QlMjBmJTIycGlwZSUzQWN1cmwlMjAtcyUyMC1MJTIwJTdCdXJsJTdEJTIwLUglMjAnQXV0aG9yaXphdGlvbiUzQUJlYXJlciUyMCU3QmhmX3Rva2VuJTdEJyUyMiUwQWRhdGFzZXQlMjAlM0QlMjB3ZHMuV2ViRGF0YXNldCh1cmwpLmRlY29kZSgpJTBBZGF0YWxvYWRlciUyMCUzRCUyMERhdGFMb2FkZXIoZGF0YXNldCUyQyUyMGJhdGNoX3NpemUlM0Q2NCUyQyUyMG51bV93b3JrZXJzJTNENCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> webdataset <span class="hljs-keyword">as</span> wds
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> get_token
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader

<span class="hljs-meta">&gt;&gt;&gt; </span>hf_token = get_token()
<span class="hljs-meta">&gt;&gt;&gt; </span>url = <span class="hljs-string">&quot;https://huggingface.co/datasets/timm/imagenet-12k-wds/resolve/main/imagenet12k-train-{{0000..1023}}.tar&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>url = <span class="hljs-string">f&quot;pipe:curl -s -L <span class="hljs-subst">{url}</span> -H &#x27;Authorization:Bearer <span class="hljs-subst">{hf_token}</span>&#x27;&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = wds.WebDataset(url).decode()
<span class="hljs-meta">&gt;&gt;&gt; </span>dataloader = DataLoader(dataset, batch_size=<span class="hljs-number">64</span>, num_workers=<span class="hljs-number">4</span>)`,wrap:!1}}),A=new B({props:{title:"Shuffle",local:"shuffle",headingTag:"h2"}}),F=new _t({props:{code:"YnVmZmVyX3NpemUlMjAlM0QlMjAxMDAwJTBBZGF0YXNldCUyMCUzRCUyMCglMEElMjAlMjAlMjAlMjB3ZHMuV2ViRGF0YXNldCh1cmwlMkMlMjBzaGFyZHNodWZmbGUlM0RUcnVlKSUwQSUyMCUyMCUyMCUyMC5zaHVmZmxlKGJ1ZmZlcl9zaXplKSUwQSUyMCUyMCUyMCUyMC5kZWNvZGUoKSUwQSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>buffer_size = <span class="hljs-number">1000</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = (
<span class="hljs-meta">... </span>    wds.WebDataset(url, shardshuffle=<span class="hljs-literal">True</span>)
<span class="hljs-meta">... </span>    .shuffle(buffer_size)
<span class="hljs-meta">... </span>    .decode()
<span class="hljs-meta">... </span>)`,wrap:!1}}),S=new ie({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/datasets-webdataset.md"}}),{c(){g=i("meta"),E=l(),V=i("p"),q=l(),p(M.$$.fragment),K=l(),p(v.$$.fragment),O=l(),$=i("p"),$.innerHTML=Wt,tt=l(),p(C.$$.fragment),et=l(),y=i("p"),y.textContent=Gt,at=l(),c=i("div"),c.innerHTML=Lt,st=l(),x=i("p"),x.innerHTML=Ht,lt=l(),k=i("p"),k.textContent=Zt,nt=l(),p(T.$$.fragment),it=l(),j=i("p"),j.textContent=Rt,mt=l(),U=i("p"),U.textContent=Dt,dt=l(),_=i("p"),_.textContent=Xt,pt=l(),J=i("ul"),J.innerHTML=zt,ft=l(),W=i("p"),W.innerHTML=At,ot=l(),p(G.$$.fragment),rt=l(),L=i("p"),L.textContent=It,ut=l(),H=i("p"),H.textContent=Pt,ht=l(),b=i("div"),b.innerHTML=Qt,gt=l(),Z=i("p"),Z.innerHTML=Ft,ct=l(),R=i("p"),R.innerHTML=St,bt=l(),p(D.$$.fragment),wt=l(),X=i("p"),X.textContent=Vt,Mt=l(),p(z.$$.fragment),vt=l(),p(A.$$.fragment),$t=l(),I=i("p"),I.textContent=Yt,Ct=l(),P=i("p"),P.textContent=Nt,yt=l(),w=i("div"),w.innerHTML=Bt,xt=l(),Q=i("p"),Q.textContent=Et,kt=l(),p(F.$$.fragment),Tt=l(),p(S.$$.fragment),jt=l(),Y=i("p"),this.h()},l(t){const e=se("svelte-u9bgzb",document.head);g=m(e,"META",{name:!0,content:!0}),e.forEach(a),E=n(t),V=m(t,"P",{}),qt(V).forEach(a),q=n(t),f(M.$$.fragment,t),K=n(t),f(v.$$.fragment,t),O=n(t),$=m(t,"P",{"data-svelte-h":!0}),d($)!=="svelte-1x0ilw7"&&($.innerHTML=Wt),tt=n(t),f(C.$$.fragment,t),et=n(t),y=m(t,"P",{"data-svelte-h":!0}),d(y)!=="svelte-s28fzx"&&(y.textContent=Gt),at=n(t),c=m(t,"DIV",{class:!0,"data-svelte-h":!0}),d(c)!=="svelte-1bdrcp9"&&(c.innerHTML=Lt),st=n(t),x=m(t,"P",{"data-svelte-h":!0}),d(x)!=="svelte-7sb7k1"&&(x.innerHTML=Ht),lt=n(t),k=m(t,"P",{"data-svelte-h":!0}),d(k)!=="svelte-1pmozez"&&(k.textContent=Zt),nt=n(t),f(T.$$.fragment,t),it=n(t),j=m(t,"P",{"data-svelte-h":!0}),d(j)!=="svelte-11egvlw"&&(j.textContent=Rt),mt=n(t),U=m(t,"P",{"data-svelte-h":!0}),d(U)!=="svelte-td3uuk"&&(U.textContent=Dt),dt=n(t),_=m(t,"P",{"data-svelte-h":!0}),d(_)!=="svelte-lwvb2g"&&(_.textContent=Xt),pt=n(t),J=m(t,"UL",{"data-svelte-h":!0}),d(J)!=="svelte-p3flt6"&&(J.innerHTML=zt),ft=n(t),W=m(t,"P",{"data-svelte-h":!0}),d(W)!=="svelte-1npudm"&&(W.innerHTML=At),ot=n(t),f(G.$$.fragment,t),rt=n(t),L=m(t,"P",{"data-svelte-h":!0}),d(L)!=="svelte-d0lfwq"&&(L.textContent=It),ut=n(t),H=m(t,"P",{"data-svelte-h":!0}),d(H)!=="svelte-1ski7pk"&&(H.textContent=Pt),ht=n(t),b=m(t,"DIV",{class:!0,"data-svelte-h":!0}),d(b)!=="svelte-16x5sv3"&&(b.innerHTML=Qt),gt=n(t),Z=m(t,"P",{"data-svelte-h":!0}),d(Z)!=="svelte-kd5fni"&&(Z.innerHTML=Ft),ct=n(t),R=m(t,"P",{"data-svelte-h":!0}),d(R)!=="svelte-1jdzygp"&&(R.innerHTML=St),bt=n(t),f(D.$$.fragment,t),wt=n(t),X=m(t,"P",{"data-svelte-h":!0}),d(X)!=="svelte-3qvfc"&&(X.textContent=Vt),Mt=n(t),f(z.$$.fragment,t),vt=n(t),f(A.$$.fragment,t),$t=n(t),I=m(t,"P",{"data-svelte-h":!0}),d(I)!=="svelte-1qsq6cj"&&(I.textContent=Yt),Ct=n(t),P=m(t,"P",{"data-svelte-h":!0}),d(P)!=="svelte-e5b3kw"&&(P.textContent=Nt),yt=n(t),w=m(t,"DIV",{class:!0,"data-svelte-h":!0}),d(w)!=="svelte-15c8cpf"&&(w.innerHTML=Bt),xt=n(t),Q=m(t,"P",{"data-svelte-h":!0}),d(Q)!=="svelte-1qykgdn"&&(Q.textContent=Et),kt=n(t),f(F.$$.fragment,t),Tt=n(t),f(S.$$.fragment,t),jt=n(t),Y=m(t,"P",{}),qt(Y).forEach(a),this.h()},h(){N(g,"name","hf:doc:metadata"),N(g,"content",de),N(c,"class","flex justify-center"),N(b,"class","flex justify-center"),N(w,"class","flex justify-center")},m(t,e){le(document.head,g),s(t,E,e),s(t,V,e),s(t,q,e),o(M,t,e),s(t,K,e),o(v,t,e),s(t,O,e),s(t,$,e),s(t,tt,e),o(C,t,e),s(t,et,e),s(t,y,e),s(t,at,e),s(t,c,e),s(t,st,e),s(t,x,e),s(t,lt,e),s(t,k,e),s(t,nt,e),o(T,t,e),s(t,it,e),s(t,j,e),s(t,mt,e),s(t,U,e),s(t,dt,e),s(t,_,e),s(t,pt,e),s(t,J,e),s(t,ft,e),s(t,W,e),s(t,ot,e),o(G,t,e),s(t,rt,e),s(t,L,e),s(t,ut,e),s(t,H,e),s(t,ht,e),s(t,b,e),s(t,gt,e),s(t,Z,e),s(t,ct,e),s(t,R,e),s(t,bt,e),o(D,t,e),s(t,wt,e),s(t,X,e),s(t,Mt,e),o(z,t,e),s(t,vt,e),o(A,t,e),s(t,$t,e),s(t,I,e),s(t,Ct,e),s(t,P,e),s(t,yt,e),s(t,w,e),s(t,xt,e),s(t,Q,e),s(t,kt,e),o(F,t,e),s(t,Tt,e),o(S,t,e),s(t,jt,e),s(t,Y,e),Ut=!0},p:Ot,i(t){Ut||(r(M.$$.fragment,t),r(v.$$.fragment,t),r(C.$$.fragment,t),r(T.$$.fragment,t),r(G.$$.fragment,t),r(D.$$.fragment,t),r(z.$$.fragment,t),r(A.$$.fragment,t),r(F.$$.fragment,t),r(S.$$.fragment,t),Ut=!0)},o(t){u(M.$$.fragment,t),u(v.$$.fragment,t),u(C.$$.fragment,t),u(T.$$.fragment,t),u(G.$$.fragment,t),u(D.$$.fragment,t),u(z.$$.fragment,t),u(A.$$.fragment,t),u(F.$$.fragment,t),u(S.$$.fragment,t),Ut=!1},d(t){t&&(a(E),a(V),a(q),a(K),a(O),a($),a(tt),a(et),a(y),a(at),a(c),a(st),a(x),a(lt),a(k),a(nt),a(it),a(j),a(mt),a(U),a(dt),a(_),a(pt),a(J),a(ft),a(W),a(ot),a(rt),a(L),a(ut),a(H),a(ht),a(b),a(gt),a(Z),a(ct),a(R),a(bt),a(wt),a(X),a(Mt),a(vt),a($t),a(I),a(Ct),a(P),a(yt),a(w),a(xt),a(Q),a(kt),a(Tt),a(jt),a(Y)),a(g),h(M,t),h(v,t),h(C,t),h(T,t),h(G,t),h(D,t),h(z,t),h(A,t),h(F,t),h(S,t)}}}const de='{"title":"WebDataset","local":"webdataset","sections":[{"title":"The WebDataset format","local":"the-webdataset-format","sections":[],"depth":2},{"title":"Multimodal support","local":"multimodal-support","sections":[],"depth":2},{"title":"Streaming","local":"streaming","sections":[],"depth":2},{"title":"Shuffle","local":"shuffle","sections":[],"depth":2}],"depth":1}';function pe(Jt){return te(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ge extends ee{constructor(g){super(),ae(this,g,pe,me,Kt,{})}}export{ge as component};
