import{s as Zt,n as Wt,o as Gt}from"../chunks/scheduler.d6170356.js";import{S as Ht,i as Lt,g as i,s as l,r as W,A as Rt,h as d,f as a,c as n,j as _t,u as G,x as m,k as Q,y as Xt,a as s,v as H,d as L,t as R,w as X}from"../chunks/index.fcd4cc08.js";import{C as ht}from"../chunks/CodeBlock.7b16bdef.js";import{H as ot}from"../chunks/Heading.8660bd0e.js";function At(ct){let f,z,A,I,h,V,c,ut=`<a href="https://github.com/webdataset/webdataset" rel="nofollow">WebDataset</a> is a library for writing I/O pipelines for large datasets.
Its sequential I/O and sharding features make it especially useful for streaming large-scale datasets to a DataLoader.`,Y,u,F,g,gt=`A WebDataset file is a TAR archive containing a series of data files.
All successive data files with the same prefix are considered to be part of the same example (e.g., an image/audio file and its label or metadata):`,N,r,bt='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-format.png"/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-format-dark.png"/>',S,b,wt="Labels and metadata can be in a <code>.json</code> file, in a <code>.txt</code> (for a caption, a description), or in a <code>.cls</code> (for a class index).",P,w,Mt=`A large scale WebDataset is made of many files called shards, where each shard is a TAR archive.
Each shard is often ~1GB but the full dataset can be multiple terabytes!`,B,M,E,y,yt=`Streaming TAR archives is fast because it reads contiguous chunks of data.
It can be orders of magnitude faster than reading separate data files one by one.`,q,C,Ct="WebDataset streaming offers high-speed performance both when reading from disk and from cloud storage, which makes it an ideal format to feed to a DataLoader:",K,p,vt='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-stream.png"/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-stream-dark.png"/>',O,v,kt='For example here is how to stream the <a href="https://huggingface.co/datasets/timm/imagenet-12k-wds" rel="nofollow">timm/imagenet-12k-wds</a> dataset directly from Hugging Face:',tt,k,$t='First you need to <a href="../huggingface_hub/quick-start#login">Login with your Hugging Face account</a>, for example using:',et,$,at,x,xt="And then you can stream the dataset with WebDataset:",st,j,lt,U,nt,T,jt=`Generally, datasets in WebDataset formats are already shuffled and ready to feed to a DataLoader.
But you can still reshuffle the data with WebDataset’s approximate shuffling.`,it,J,Ut="In addition to shuffling the list of shards, WebDataset uses a buffer to shuffle a dataset without any cost to speed:",dt,o,Tt='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-shuffle.png"/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-webdataset-shuffle-dark.png"/>',mt,_,Jt="To shuffle a list of sharded files and randomly sample from the shuffle buffer:",ft,Z,rt,D,pt;return h=new ot({props:{title:"WebDataset",local:"webdataset",headingTag:"h1"}}),u=new ot({props:{title:"The WebDataset format",local:"the-webdataset-format",headingTag:"h2"}}),M=new ot({props:{title:"Streaming",local:"streaming",headingTag:"h2"}}),$=new ht({props:{code:"aHVnZ2luZ2ZhY2UtY2xpJTIwbG9naW4=",highlighted:'huggingface-<span class="hljs-keyword">cli</span> login',wrap:!1}}),j=new ht({props:{code:"aW1wb3J0JTIwd2ViZGF0YXNldCUyMGFzJTIwd2RzJTBBZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMGdldF90b2tlbiUwQWZyb20lMjB0b3JjaC51dGlscy5kYXRhJTIwaW1wb3J0JTIwRGF0YUxvYWRlciUwQSUwQWhmX3Rva2VuJTIwJTNEJTIwZ2V0X3Rva2VuKCklMEF1cmwlMjAlM0QlMjAlMjJodHRwcyUzQSUyRiUyRmh1Z2dpbmdmYWNlLmNvJTJGZGF0YXNldHMlMkZ0aW1tJTJGaW1hZ2VuZXQtMTJrLXdkcyUyRnJlc29sdmUlMkZtYWluJTJGaW1hZ2VuZXQxMmstdHJhaW4tJTdCJTdCMDAwMC4uMTAyMyU3RCU3RC50YXIlMjIlMEF1cmwlMjAlM0QlMjBmJTIycGlwZSUzQWN1cmwlMjAtcyUyMC1MJTIwJTdCdXJsJTdEJTIwLUglMjAnQXV0aG9yaXphdGlvbiUzQUJlYXJlciUyMCU3QmhmX3Rva2VuJTdEJyUyMiUwQWRhdGFzZXQlMjAlM0QlMjB3ZHMuV2ViRGF0YXNldCh1cmwpLmRlY29kZSgpJTBBZGF0YWxvYWRlciUyMCUzRCUyMERhdGFMb2FkZXIoZGF0YXNldCUyQyUyMGJhdGNoX3NpemUlM0Q2NCUyQyUyMG51bV93b3JrZXJzJTNENCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> webdataset <span class="hljs-keyword">as</span> wds
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> get_token
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> torch.utils.data <span class="hljs-keyword">import</span> DataLoader

<span class="hljs-meta">&gt;&gt;&gt; </span>hf_token = get_token()
<span class="hljs-meta">&gt;&gt;&gt; </span>url = <span class="hljs-string">&quot;https://huggingface.co/datasets/timm/imagenet-12k-wds/resolve/main/imagenet12k-train-{{0000..1023}}.tar&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>url = <span class="hljs-string">f&quot;pipe:curl -s -L <span class="hljs-subst">{url}</span> -H &#x27;Authorization:Bearer <span class="hljs-subst">{hf_token}</span>&#x27;&quot;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = wds.WebDataset(url).decode()
<span class="hljs-meta">&gt;&gt;&gt; </span>dataloader = DataLoader(dataset, batch_size=<span class="hljs-number">64</span>, num_workers=<span class="hljs-number">4</span>)`,wrap:!1}}),U=new ot({props:{title:"Shuffle",local:"shuffle",headingTag:"h2"}}),Z=new ht({props:{code:"YnVmZmVyX3NpemUlMjAlM0QlMjAxMDAwJTBBZGF0YXNldCUyMCUzRCUyMCglMEElMjAlMjAlMjAlMjB3ZHMuV2ViRGF0YXNldCh1cmwlMkMlMjBzaGFyZHNodWZmbGUlM0RUcnVlKSUwQSUyMCUyMCUyMCUyMC5zaHVmZmxlKGJ1ZmZlcl9zaXplKSUwQSUyMCUyMCUyMCUyMC5kZWNvZGUoKSUwQSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>buffer_size = <span class="hljs-number">1000</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>dataset = (
<span class="hljs-meta">... </span>    wds.WebDataset(url, shardshuffle=<span class="hljs-literal">True</span>)
<span class="hljs-meta">... </span>    .shuffle(buffer_size)
<span class="hljs-meta">... </span>    .decode()
<span class="hljs-meta">... </span>)`,wrap:!1}}),{c(){f=i("meta"),z=l(),A=i("p"),I=l(),W(h.$$.fragment),V=l(),c=i("p"),c.innerHTML=ut,Y=l(),W(u.$$.fragment),F=l(),g=i("p"),g.textContent=gt,N=l(),r=i("div"),r.innerHTML=bt,S=l(),b=i("p"),b.innerHTML=wt,P=l(),w=i("p"),w.textContent=Mt,B=l(),W(M.$$.fragment),E=l(),y=i("p"),y.textContent=yt,q=l(),C=i("p"),C.textContent=Ct,K=l(),p=i("div"),p.innerHTML=vt,O=l(),v=i("p"),v.innerHTML=kt,tt=l(),k=i("p"),k.innerHTML=$t,et=l(),W($.$$.fragment),at=l(),x=i("p"),x.textContent=xt,st=l(),W(j.$$.fragment),lt=l(),W(U.$$.fragment),nt=l(),T=i("p"),T.textContent=jt,it=l(),J=i("p"),J.textContent=Ut,dt=l(),o=i("div"),o.innerHTML=Tt,mt=l(),_=i("p"),_.textContent=Jt,ft=l(),W(Z.$$.fragment),rt=l(),D=i("p"),this.h()},l(t){const e=Rt("svelte-u9bgzb",document.head);f=d(e,"META",{name:!0,content:!0}),e.forEach(a),z=n(t),A=d(t,"P",{}),_t(A).forEach(a),I=n(t),G(h.$$.fragment,t),V=n(t),c=d(t,"P",{"data-svelte-h":!0}),m(c)!=="svelte-1x0ilw7"&&(c.innerHTML=ut),Y=n(t),G(u.$$.fragment,t),F=n(t),g=d(t,"P",{"data-svelte-h":!0}),m(g)!=="svelte-s28fzx"&&(g.textContent=gt),N=n(t),r=d(t,"DIV",{class:!0,"data-svelte-h":!0}),m(r)!=="svelte-1bdrcp9"&&(r.innerHTML=bt),S=n(t),b=d(t,"P",{"data-svelte-h":!0}),m(b)!=="svelte-7sb7k1"&&(b.innerHTML=wt),P=n(t),w=d(t,"P",{"data-svelte-h":!0}),m(w)!=="svelte-1pmozez"&&(w.textContent=Mt),B=n(t),G(M.$$.fragment,t),E=n(t),y=d(t,"P",{"data-svelte-h":!0}),m(y)!=="svelte-d0lfwq"&&(y.textContent=yt),q=n(t),C=d(t,"P",{"data-svelte-h":!0}),m(C)!=="svelte-1ski7pk"&&(C.textContent=Ct),K=n(t),p=d(t,"DIV",{class:!0,"data-svelte-h":!0}),m(p)!=="svelte-16x5sv3"&&(p.innerHTML=vt),O=n(t),v=d(t,"P",{"data-svelte-h":!0}),m(v)!=="svelte-kd5fni"&&(v.innerHTML=kt),tt=n(t),k=d(t,"P",{"data-svelte-h":!0}),m(k)!=="svelte-1hbzj8b"&&(k.innerHTML=$t),et=n(t),G($.$$.fragment,t),at=n(t),x=d(t,"P",{"data-svelte-h":!0}),m(x)!=="svelte-3qvfc"&&(x.textContent=xt),st=n(t),G(j.$$.fragment,t),lt=n(t),G(U.$$.fragment,t),nt=n(t),T=d(t,"P",{"data-svelte-h":!0}),m(T)!=="svelte-1qsq6cj"&&(T.textContent=jt),it=n(t),J=d(t,"P",{"data-svelte-h":!0}),m(J)!=="svelte-e5b3kw"&&(J.textContent=Ut),dt=n(t),o=d(t,"DIV",{class:!0,"data-svelte-h":!0}),m(o)!=="svelte-15c8cpf"&&(o.innerHTML=Tt),mt=n(t),_=d(t,"P",{"data-svelte-h":!0}),m(_)!=="svelte-1qykgdn"&&(_.textContent=Jt),ft=n(t),G(Z.$$.fragment,t),rt=n(t),D=d(t,"P",{}),_t(D).forEach(a),this.h()},h(){Q(f,"name","hf:doc:metadata"),Q(f,"content",Dt),Q(r,"class","flex justify-center"),Q(p,"class","flex justify-center"),Q(o,"class","flex justify-center")},m(t,e){Xt(document.head,f),s(t,z,e),s(t,A,e),s(t,I,e),H(h,t,e),s(t,V,e),s(t,c,e),s(t,Y,e),H(u,t,e),s(t,F,e),s(t,g,e),s(t,N,e),s(t,r,e),s(t,S,e),s(t,b,e),s(t,P,e),s(t,w,e),s(t,B,e),H(M,t,e),s(t,E,e),s(t,y,e),s(t,q,e),s(t,C,e),s(t,K,e),s(t,p,e),s(t,O,e),s(t,v,e),s(t,tt,e),s(t,k,e),s(t,et,e),H($,t,e),s(t,at,e),s(t,x,e),s(t,st,e),H(j,t,e),s(t,lt,e),H(U,t,e),s(t,nt,e),s(t,T,e),s(t,it,e),s(t,J,e),s(t,dt,e),s(t,o,e),s(t,mt,e),s(t,_,e),s(t,ft,e),H(Z,t,e),s(t,rt,e),s(t,D,e),pt=!0},p:Wt,i(t){pt||(L(h.$$.fragment,t),L(u.$$.fragment,t),L(M.$$.fragment,t),L($.$$.fragment,t),L(j.$$.fragment,t),L(U.$$.fragment,t),L(Z.$$.fragment,t),pt=!0)},o(t){R(h.$$.fragment,t),R(u.$$.fragment,t),R(M.$$.fragment,t),R($.$$.fragment,t),R(j.$$.fragment,t),R(U.$$.fragment,t),R(Z.$$.fragment,t),pt=!1},d(t){t&&(a(z),a(A),a(I),a(V),a(c),a(Y),a(F),a(g),a(N),a(r),a(S),a(b),a(P),a(w),a(B),a(E),a(y),a(q),a(C),a(K),a(p),a(O),a(v),a(tt),a(k),a(et),a(at),a(x),a(st),a(lt),a(nt),a(T),a(it),a(J),a(dt),a(o),a(mt),a(_),a(ft),a(rt),a(D)),a(f),X(h,t),X(u,t),X(M,t),X($,t),X(j,t),X(U,t),X(Z,t)}}}const Dt='{"title":"WebDataset","local":"webdataset","sections":[{"title":"The WebDataset format","local":"the-webdataset-format","sections":[],"depth":2},{"title":"Streaming","local":"streaming","sections":[],"depth":2},{"title":"Shuffle","local":"shuffle","sections":[],"depth":2}],"depth":1}';function Qt(ct){return Gt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ft extends Ht{constructor(f){super(),Lt(this,f,Qt,At,Zt,{})}}export{Ft as component};
