import{s as Re,n as He,o as Ye}from"../chunks/scheduler.9cda857a.js";import{S as qe,i as Xe,e as s,s as i,c as d,h as Se,a as f,d as a,b as l,f as je,g as r,j as u,k as ye,l as Qe,m as n,n as o,t as p,o as m,p as g}from"../chunks/index.003612b0.js";import{C as Be}from"../chunks/CopyLLMTxtMenu.44d08a39.js";import{C as E}from"../chunks/CodeBlock.a90632f1.js";import{H as Q,E as Ne}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.e5cfda8c.js";function Ee(Ze){let c,L,B,P,$,D,w,V,T,Me='<a href="https://daft.ai/" rel="nofollow">Daft</a> is a high-performance data engine providing simple and reliable data processing for any modality and scale. Daft has native support for reading from and writing to Hugging Face datasets.',z,h,_e='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/daft_hf.png"/>',K,J,A,y,be="To get started, pip install <code>daft</code> with the <code>huggingface</code> feature:",O,Z,ee,M,te,_,ve='Daft is able to read datasets directly from the Hugging Face Hub using the <a href="https://docs.daft.ai/en/stable/api/io/#daft.read_huggingface" rel="nofollow"><code>daft.read_huggingface()</code></a> function or via the <code>hf://datasets/</code> protocol.',ae,b,ne,v,Fe='Using <a href="https://docs.daft.ai/en/stable/api/io/#daft.read_huggingface" rel="nofollow"><code>daft.read_huggingface()</code></a>, you can easily load a dataset.',ie,F,le,U,Ue="This will read the entire dataset into a DataFrame.",se,C,fe,W,Ce='Not only can you read entire datasets, but you can also read individual files from a dataset repository. Using a read function that takes in a path (such as <a href="https://docs.daft.ai/en/stable/api/io/#daft.read_parquet" rel="nofollow"><code>daft.read_parquet()</code></a>, <a href="https://docs.daft.ai/en/stable/api/io/#daft.read_csv" rel="nofollow"><code>daft.read_csv()</code></a>, or <a href="https://docs.daft.ai/en/stable/api/io/#daft.read_json" rel="nofollow"><code>daft.read_json()</code></a>), specify a Hugging Face dataset path via the <code>hf://datasets/</code> prefix:',de,k,re,G,oe,I,We='Daft is able to write Parquet files to a Hugging Face dataset repository using <a href="https://docs.daft.ai/en/stable/api/dataframe/#daft.DataFrame.write_deltalake" rel="nofollow"><code>daft.DataFrame.write_huggingface</code></a>. Daft supports <a href="https://huggingface.co/blog/parquet-cdc" rel="nofollow">Content-Defined Chunking</a> and <a href="https://huggingface.co/blog/xet-on-the-hub" rel="nofollow">Xet</a> for faster, deduplicated writes.',pe,x,ke="Basic usage:",me,j,ge,R,Ge='See the <a href="https://docs.daft.ai/en/stable/api/dataframe/#daft.DataFrame.write_huggingface" rel="nofollow"><code>DataFrame.write_huggingface</code></a> API page for more info.',ue,H,ce,Y,Ie='The <code>token</code> parameter in <a href="https://docs.daft.ai/en/stable/api/config/#daft.io.HuggingFaceConfig" rel="nofollow"><code>daft.io.HuggingFaceConfig</code></a> can be used to specify a Hugging Face access token for requests that require authentication (e.g. reading private dataset repositories or writing to a dataset repository).',he,q,xe="Example of loading a dataset with a specified token:",$e,X,we,S,Te,N,Je;return $=new Be({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),w=new Q({props:{title:"Daft",local:"daft",headingTag:"h1"}}),J=new Q({props:{title:"Getting Started",local:"getting-started",headingTag:"h2"}}),Z=new E({props:{code:"cGlwJTIwaW5zdGFsbCUyMCdkYWZ0JTVCaHVnZ2luZ2ZhY2UlNUQn",highlighted:'pip install <span class="hljs-string">&#x27;daft[huggingface]&#x27;</span>',wrap:!1}}),M=new Q({props:{title:"Read",local:"read",headingTag:"h2"}}),b=new Q({props:{title:"Reading an Entire Dataset",local:"reading-an-entire-dataset",headingTag:"h3"}}),F=new E({props:{code:"aW1wb3J0JTIwZGFmdCUwQSUwQWRmJTIwJTNEJTIwZGFmdC5yZWFkX2h1Z2dpbmdmYWNlKCUyMnVzZXJuYW1lJTJGZGF0YXNldF9uYW1lJTIyKQ==",highlighted:`<span class="hljs-keyword">import</span> daft

df = daft.read_huggingface(<span class="hljs-string">&quot;username/dataset_name&quot;</span>)`,wrap:!1}}),C=new Q({props:{title:"Reading Specific Files",local:"reading-specific-files",headingTag:"h3"}}),k=new E({props:{code:"aW1wb3J0JTIwZGFmdCUwQSUwQSUyMyUyMHJlYWQlMjBhJTIwc3BlY2lmaWMlMjBQYXJxdWV0JTIwZmlsZSUwQWRmJTIwJTNEJTIwZGFmdC5yZWFkX3BhcnF1ZXQoJTIyaGYlM0ElMkYlMkZkYXRhc2V0cyUyRnVzZXJuYW1lJTJGZGF0YXNldF9uYW1lJTJGZmlsZV9uYW1lLnBhcnF1ZXQlMjIpJTBBJTBBJTIzJTIwb3IlMjBhJTIwY3N2JTIwZmlsZSUwQWRmJTIwJTNEJTIwZGFmdC5yZWFkX2NzdiglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZkYXRhc2V0X25hbWUlMkZmaWxlX25hbWUuY3N2JTIyKSUwQSUwQSUyMyUyMG9yJTIwYSUyMHNldCUyMG9mJTIwUGFycXVldCUyMGZpbGVzJTIwdXNpbmclMjBhJTIwZ2xvYiUyMHBhdHRlcm4lMEFkZiUyMCUzRCUyMGRhZnQucmVhZF9wYXJxdWV0KCUyMmhmJTNBJTJGJTJGZGF0YXNldHMlMkZ1c2VybmFtZSUyRmRhdGFzZXRfbmFtZSUyRioqJTJGKi5wYXJxdWV0JTIyKQ==",highlighted:`<span class="hljs-keyword">import</span> daft

<span class="hljs-comment"># read a specific Parquet file</span>
df = daft.read_parquet(<span class="hljs-string">&quot;hf://datasets/username/dataset_name/file_name.parquet&quot;</span>)

<span class="hljs-comment"># or a csv file</span>
df = daft.read_csv(<span class="hljs-string">&quot;hf://datasets/username/dataset_name/file_name.csv&quot;</span>)

<span class="hljs-comment"># or a set of Parquet files using a glob pattern</span>
df = daft.read_parquet(<span class="hljs-string">&quot;hf://datasets/username/dataset_name/**/*.parquet&quot;</span>)`,wrap:!1}}),G=new Q({props:{title:"Write",local:"write",headingTag:"h2"}}),j=new E({props:{code:"aW1wb3J0JTIwZGFmdCUwQSUwQWRmJTNBJTIwZGFmdC5EYXRhRnJhbWUlMjAlM0QlMjAuLi4lMEElMEFkZi53cml0ZV9odWdnaW5nZmFjZSglMjJ1c2VybmFtZSUyRmRhdGFzZXRfbmFtZSUyMik=",highlighted:`<span class="hljs-keyword">import</span> daft

df: daft.DataFrame = ...

df.write_huggingface(<span class="hljs-string">&quot;username/dataset_name&quot;</span>)`,wrap:!1}}),H=new Q({props:{title:"Authentication",local:"authentication",headingTag:"h2"}}),X=new E({props:{code:"ZnJvbSUyMGRhZnQuaW8lMjBpbXBvcnQlMjBJT0NvbmZpZyUyQyUyMEh1Z2dpbmdGYWNlQ29uZmlnJTBBJTBBaW9fY29uZmlnJTIwJTNEJTIwSU9Db25maWcoaGYlM0RIdWdnaW5nRmFjZUNvbmZpZyh0b2tlbiUzRCUyMnlvdXJfdG9rZW4lMjIpKSUwQWRmJTIwJTNEJTIwZGFmdC5yZWFkX3BhcnF1ZXQoJTIyaGYlM0ElMkYlMkZkYXRhc2V0cyUyRnVzZXJuYW1lJTJGZGF0YXNldF9uYW1lJTIyJTJDJTIwaW9fY29uZmlnJTNEaW9fY29uZmlnKQ==",highlighted:`<span class="hljs-keyword">from</span> daft.io <span class="hljs-keyword">import</span> IOConfig, HuggingFaceConfig

io_config = IOConfig(hf=HuggingFaceConfig(token=<span class="hljs-string">&quot;your_token&quot;</span>))
df = daft.read_parquet(<span class="hljs-string">&quot;hf://datasets/username/dataset_name&quot;</span>, io_config=io_config)`,wrap:!1}}),S=new Ne({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/datasets-daft.md"}}),{c(){c=s("meta"),L=i(),B=s("p"),P=i(),d($.$$.fragment),D=i(),d(w.$$.fragment),V=i(),T=s("p"),T.innerHTML=Me,z=i(),h=s("div"),h.innerHTML=_e,K=i(),d(J.$$.fragment),A=i(),y=s("p"),y.innerHTML=be,O=i(),d(Z.$$.fragment),ee=i(),d(M.$$.fragment),te=i(),_=s("p"),_.innerHTML=ve,ae=i(),d(b.$$.fragment),ne=i(),v=s("p"),v.innerHTML=Fe,ie=i(),d(F.$$.fragment),le=i(),U=s("p"),U.textContent=Ue,se=i(),d(C.$$.fragment),fe=i(),W=s("p"),W.innerHTML=Ce,de=i(),d(k.$$.fragment),re=i(),d(G.$$.fragment),oe=i(),I=s("p"),I.innerHTML=We,pe=i(),x=s("p"),x.textContent=ke,me=i(),d(j.$$.fragment),ge=i(),R=s("p"),R.innerHTML=Ge,ue=i(),d(H.$$.fragment),ce=i(),Y=s("p"),Y.innerHTML=Ie,he=i(),q=s("p"),q.textContent=xe,$e=i(),d(X.$$.fragment),we=i(),d(S.$$.fragment),Te=i(),N=s("p"),this.h()},l(e){const t=Se("svelte-u9bgzb",document.head);c=f(t,"META",{name:!0,content:!0}),t.forEach(a),L=l(e),B=f(e,"P",{}),je(B).forEach(a),P=l(e),r($.$$.fragment,e),D=l(e),r(w.$$.fragment,e),V=l(e),T=f(e,"P",{"data-svelte-h":!0}),u(T)!=="svelte-qmy1ip"&&(T.innerHTML=Me),z=l(e),h=f(e,"DIV",{class:!0,"data-svelte-h":!0}),u(h)!=="svelte-1xvl7v5"&&(h.innerHTML=_e),K=l(e),r(J.$$.fragment,e),A=l(e),y=f(e,"P",{"data-svelte-h":!0}),u(y)!=="svelte-fialq8"&&(y.innerHTML=be),O=l(e),r(Z.$$.fragment,e),ee=l(e),r(M.$$.fragment,e),te=l(e),_=f(e,"P",{"data-svelte-h":!0}),u(_)!=="svelte-17u1dsn"&&(_.innerHTML=ve),ae=l(e),r(b.$$.fragment,e),ne=l(e),v=f(e,"P",{"data-svelte-h":!0}),u(v)!=="svelte-1qmv9cj"&&(v.innerHTML=Fe),ie=l(e),r(F.$$.fragment,e),le=l(e),U=f(e,"P",{"data-svelte-h":!0}),u(U)!=="svelte-1n7yb6"&&(U.textContent=Ue),se=l(e),r(C.$$.fragment,e),fe=l(e),W=f(e,"P",{"data-svelte-h":!0}),u(W)!=="svelte-1ykvjbj"&&(W.innerHTML=Ce),de=l(e),r(k.$$.fragment,e),re=l(e),r(G.$$.fragment,e),oe=l(e),I=f(e,"P",{"data-svelte-h":!0}),u(I)!=="svelte-1eu2lag"&&(I.innerHTML=We),pe=l(e),x=f(e,"P",{"data-svelte-h":!0}),u(x)!=="svelte-1aguo8v"&&(x.textContent=ke),me=l(e),r(j.$$.fragment,e),ge=l(e),R=f(e,"P",{"data-svelte-h":!0}),u(R)!=="svelte-127tqnc"&&(R.innerHTML=Ge),ue=l(e),r(H.$$.fragment,e),ce=l(e),Y=f(e,"P",{"data-svelte-h":!0}),u(Y)!=="svelte-19rkul4"&&(Y.innerHTML=Ie),he=l(e),q=f(e,"P",{"data-svelte-h":!0}),u(q)!=="svelte-ehovs6"&&(q.textContent=xe),$e=l(e),r(X.$$.fragment,e),we=l(e),r(S.$$.fragment,e),Te=l(e),N=f(e,"P",{}),je(N).forEach(a),this.h()},h(){ye(c,"name","hf:doc:metadata"),ye(c,"content",Le),ye(h,"class","flex justify-center")},m(e,t){Qe(document.head,c),n(e,L,t),n(e,B,t),n(e,P,t),o($,e,t),n(e,D,t),o(w,e,t),n(e,V,t),n(e,T,t),n(e,z,t),n(e,h,t),n(e,K,t),o(J,e,t),n(e,A,t),n(e,y,t),n(e,O,t),o(Z,e,t),n(e,ee,t),o(M,e,t),n(e,te,t),n(e,_,t),n(e,ae,t),o(b,e,t),n(e,ne,t),n(e,v,t),n(e,ie,t),o(F,e,t),n(e,le,t),n(e,U,t),n(e,se,t),o(C,e,t),n(e,fe,t),n(e,W,t),n(e,de,t),o(k,e,t),n(e,re,t),o(G,e,t),n(e,oe,t),n(e,I,t),n(e,pe,t),n(e,x,t),n(e,me,t),o(j,e,t),n(e,ge,t),n(e,R,t),n(e,ue,t),o(H,e,t),n(e,ce,t),n(e,Y,t),n(e,he,t),n(e,q,t),n(e,$e,t),o(X,e,t),n(e,we,t),o(S,e,t),n(e,Te,t),n(e,N,t),Je=!0},p:He,i(e){Je||(p($.$$.fragment,e),p(w.$$.fragment,e),p(J.$$.fragment,e),p(Z.$$.fragment,e),p(M.$$.fragment,e),p(b.$$.fragment,e),p(F.$$.fragment,e),p(C.$$.fragment,e),p(k.$$.fragment,e),p(G.$$.fragment,e),p(j.$$.fragment,e),p(H.$$.fragment,e),p(X.$$.fragment,e),p(S.$$.fragment,e),Je=!0)},o(e){m($.$$.fragment,e),m(w.$$.fragment,e),m(J.$$.fragment,e),m(Z.$$.fragment,e),m(M.$$.fragment,e),m(b.$$.fragment,e),m(F.$$.fragment,e),m(C.$$.fragment,e),m(k.$$.fragment,e),m(G.$$.fragment,e),m(j.$$.fragment,e),m(H.$$.fragment,e),m(X.$$.fragment,e),m(S.$$.fragment,e),Je=!1},d(e){e&&(a(L),a(B),a(P),a(D),a(V),a(T),a(z),a(h),a(K),a(A),a(y),a(O),a(ee),a(te),a(_),a(ae),a(ne),a(v),a(ie),a(le),a(U),a(se),a(fe),a(W),a(de),a(re),a(oe),a(I),a(pe),a(x),a(me),a(ge),a(R),a(ue),a(ce),a(Y),a(he),a(q),a($e),a(we),a(Te),a(N)),a(c),g($,e),g(w,e),g(J,e),g(Z,e),g(M,e),g(b,e),g(F,e),g(C,e),g(k,e),g(G,e),g(j,e),g(H,e),g(X,e),g(S,e)}}}const Le='{"title":"Daft","local":"daft","sections":[{"title":"Getting Started","local":"getting-started","sections":[],"depth":2},{"title":"Read","local":"read","sections":[{"title":"Reading an Entire Dataset","local":"reading-an-entire-dataset","sections":[],"depth":3},{"title":"Reading Specific Files","local":"reading-specific-files","sections":[],"depth":3}],"depth":2},{"title":"Write","local":"write","sections":[],"depth":2},{"title":"Authentication","local":"authentication","sections":[],"depth":2}],"depth":1}';function Pe(Ze){return Ye(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Oe extends qe{constructor(c){super(),Xe(this,c,Pe,Ee,Re,{})}}export{Oe as component};
