import{s as vt,n as xt,o as Rt}from"../chunks/scheduler.53228c21.js";import{S as Ct,i as Ht,e as o,s,c as d,h as Yt,a as i,d as a,b as n,f as Bt,g as p,j as r,k as ht,l as Wt,m as l,n as m,t as u,o as M,p as c}from"../chunks/index.100fac89.js";import{C as zt}from"../chunks/CopyLLMTxtMenu.2cdbb51f.js";import{C as q}from"../chunks/CodeBlock.d30a6509.js";import{H as gt,E as Ft}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.3fcbbc76.js";function qt(yt){let f,L,z,E,y,N,T,Q,J,Tt='There are many datasets on the <a href="https://huggingface.co/datasets?task_categories=task_categories:text-to-image&amp;sort=downloads" rel="nofollow">Hub</a> to train a model on, but if you can’t find one you’re interested in or want to use your own, you can create a dataset with the 🤗 <a href="https://huggingface.co/docs/datasets" rel="nofollow">Datasets</a> library. The dataset structure depends on the task you want to train your model on. The most basic dataset structure is a directory of images for tasks like unconditional image generation. Another dataset structure may be a directory of images and a text file containing their corresponding text captions for tasks like text-to-image generation.',A,w,Jt="This guide will show you two ways to create a dataset to finetune on:",S,j,wt="<li>provide a folder of images to the <code>--train_data_dir</code> argument</li> <li>upload a dataset to the Hub and pass the dataset repository id to the <code>--dataset_name</code> argument</li>",V,h,jt='<p>💡 Learn more about how to create an image dataset for training in the <a href="https://huggingface.co/docs/datasets/image_dataset" rel="nofollow">Create an image dataset</a> guide.</p>',P,_,D,b,_t='For unconditional generation, you can provide your own dataset as a folder of images. The training script uses the <a href="https://huggingface.co/docs/datasets/en/image_dataset#imagefolder" rel="nofollow"><code>ImageFolder</code></a> builder from 🤗 Datasets to automatically build a dataset from the folder. Your directory structure should look like:',K,Z,O,$,bt="Pass the path to the dataset directory to the <code>--train_data_dir</code> argument, and then you can start training:",tt,U,et,k,at,g,Zt='<p>💡 For more details and context about creating and uploading a dataset to the Hub, take a look at the <a href="https://huggingface.co/blog/image-search-datasets" rel="nofollow">Image search with 🤗 Datasets</a> post.</p>',lt,I,$t='Start by creating a dataset with the <a href="https://huggingface.co/docs/datasets/image_load#imagefolder" rel="nofollow"><code>ImageFolder</code></a> feature, which creates an <code>image</code> column containing the PIL-encoded images.',st,G,Ut="You can use the <code>data_dir</code> or <code>data_files</code> parameters to specify the location of the dataset. The <code>data_files</code> parameter supports mapping specific files to dataset splits like <code>train</code> or <code>test</code>:",nt,X,ot,B,kt='Then use the <a href="https://huggingface.co/docs/datasets/main/en/package_reference/main_classes#datasets.Dataset.push_to_hub" rel="nofollow">push_to_hub</a> method to upload the dataset to the Hub:',it,v,rt,x,It="Now the dataset is available for training by passing the dataset name to the <code>--dataset_name</code> argument:",dt,R,pt,C,mt,H,Gt="Now that you’ve created a dataset, you can plug it into the <code>train_data_dir</code> (if your dataset is local) or <code>dataset_name</code> (if your dataset is on the Hub) arguments of a training script.",ut,Y,Xt='For your next steps, feel free to try and use your dataset to train a model for <a href="unconditional_training">unconditional generation</a> or <a href="text2image">text-to-image generation</a>!',Mt,W,ct,F,ft;return y=new zt({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),T=new gt({props:{title:"Create a dataset for training",local:"create-a-dataset-for-training",headingTag:"h1"}}),_=new gt({props:{title:"Provide a dataset as a folder",local:"provide-a-dataset-as-a-folder",headingTag:"h2"}}),Z=new q({props:{code:"ZGF0YV9kaXIlMkZ4eHgucG5nJTBBZGF0YV9kaXIlMkZ4eHkucG5nJTBBZGF0YV9kaXIlMkYlNUIuLi4lNUQlMkZ4eHoucG5n",highlighted:`data_dir/xxx.png
data_dir/xxy.png
data_dir/[...]/xxz.png`,wrap:!1}}),U=new q({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMHRyYWluX3VuY29uZGl0aW9uYWwucHklMjAlNUMlMEElMjAlMjAlMjAlMjAtLXRyYWluX2RhdGFfZGlyJTIwJTNDcGF0aC10by10cmFpbi1kaXJlY3RvcnklM0UlMjAlNUMlMEElMjAlMjAlMjAlMjAlM0NvdGhlci1hcmd1bWVudHMlM0U=",highlighted:`accelerate launch train_unconditional.py \\
    --train_data_dir &lt;path-to-train-directory&gt; \\
    &lt;other-arguments&gt;`,wrap:!1}}),k=new gt({props:{title:"Upload your data to the Hub",local:"upload-your-data-to-the-hub",headingTag:"h2"}}),X=new q({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBJTIzJTIwZXhhbXBsZSUyMDElM0ElMjBsb2NhbCUyMGZvbGRlciUwQWRhdGFzZXQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyaW1hZ2Vmb2xkZXIlMjIlMkMlMjBkYXRhX2RpciUzRCUyMnBhdGhfdG9feW91cl9mb2xkZXIlMjIpJTBBJTBBJTIzJTIwZXhhbXBsZSUyMDIlM0ElMjBsb2NhbCUyMGZpbGVzJTIwKHN1cHBvcnRlZCUyMGZvcm1hdHMlMjBhcmUlMjB0YXIlMkMlMjBnemlwJTJDJTIwemlwJTJDJTIweHolMkMlMjByYXIlMkMlMjB6c3RkKSUwQWRhdGFzZXQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTIyaW1hZ2Vmb2xkZXIlMjIlMkMlMjBkYXRhX2ZpbGVzJTNEJTIycGF0aF90b196aXBfZmlsZSUyMiklMEElMEElMjMlMjBleGFtcGxlJTIwMyUzQSUyMHJlbW90ZSUyMGZpbGVzJTIwKHN1cHBvcnRlZCUyMGZvcm1hdHMlMjBhcmUlMjB0YXIlMkMlMjBnemlwJTJDJTIwemlwJTJDJTIweHolMkMlMjByYXIlMkMlMjB6c3RkKSUwQWRhdGFzZXQlMjAlM0QlMjBsb2FkX2RhdGFzZXQoJTBBJTIwJTIwJTIwJTIwJTIyaW1hZ2Vmb2xkZXIlMjIlMkMlMEElMjAlMjAlMjAlMjBkYXRhX2ZpbGVzJTNEJTIyaHR0cHMlM0ElMkYlMkZkb3dubG9hZC5taWNyb3NvZnQuY29tJTJGZG93bmxvYWQlMkYzJTJGRSUyRjElMkYzRTFDM0YyMS1FQ0RCLTQ4NjktODM2OC02REVCQTc3QjkxOUYlMkZrYWdnbGVjYXRzYW5kZG9nc18zMzY3YS56aXAlMjIlMkMlMEEpJTBBJTBBJTIzJTIwZXhhbXBsZSUyMDQlM0ElMjBwcm92aWRpbmclMjBzZXZlcmFsJTIwc3BsaXRzJTBBZGF0YXNldCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMEElMjAlMjAlMjAlMjAlMjJpbWFnZWZvbGRlciUyMiUyQyUyMGRhdGFfZmlsZXMlM0QlN0IlMjJ0cmFpbiUyMiUzQSUyMCU1QiUyMnBhdGglMkZ0byUyRmZpbGUxJTIyJTJDJTIwJTIycGF0aCUyRnRvJTJGZmlsZTIlMjIlNUQlMkMlMjAlMjJ0ZXN0JTIyJTNBJTIwJTVCJTIycGF0aCUyRnRvJTJGZmlsZTMlMjIlMkMlMjAlMjJwYXRoJTJGdG8lMkZmaWxlNCUyMiU1RCU3RCUwQSk=",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-comment"># example 1: local folder</span>
dataset = load_dataset(<span class="hljs-string">&quot;imagefolder&quot;</span>, data_dir=<span class="hljs-string">&quot;path_to_your_folder&quot;</span>)

<span class="hljs-comment"># example 2: local files (supported formats are tar, gzip, zip, xz, rar, zstd)</span>
dataset = load_dataset(<span class="hljs-string">&quot;imagefolder&quot;</span>, data_files=<span class="hljs-string">&quot;path_to_zip_file&quot;</span>)

<span class="hljs-comment"># example 3: remote files (supported formats are tar, gzip, zip, xz, rar, zstd)</span>
dataset = load_dataset(
    <span class="hljs-string">&quot;imagefolder&quot;</span>,
    data_files=<span class="hljs-string">&quot;https://download.microsoft.com/download/3/E/1/3E1C3F21-ECDB-4869-8368-6DEBA77B919F/kagglecatsanddogs_3367a.zip&quot;</span>,
)

<span class="hljs-comment"># example 4: providing several splits</span>
dataset = load_dataset(
    <span class="hljs-string">&quot;imagefolder&quot;</span>, data_files={<span class="hljs-string">&quot;train&quot;</span>: [<span class="hljs-string">&quot;path/to/file1&quot;</span>, <span class="hljs-string">&quot;path/to/file2&quot;</span>], <span class="hljs-string">&quot;test&quot;</span>: [<span class="hljs-string">&quot;path/to/file3&quot;</span>, <span class="hljs-string">&quot;path/to/file4&quot;</span>]}
)`,wrap:!1}}),v=new q({props:{code:"JTIzJTIwYXNzdW1pbmclMjB5b3UlMjBoYXZlJTIwcmFuJTIwdGhlJTIwaGYlMjBhdXRoJTIwbG9naW4lMjBjb21tYW5kJTIwaW4lMjBhJTIwdGVybWluYWwlMEFkYXRhc2V0LnB1c2hfdG9faHViKCUyMm5hbWVfb2ZfeW91cl9kYXRhc2V0JTIyKSUwQSUwQSUyMyUyMGlmJTIweW91JTIwd2FudCUyMHRvJTIwcHVzaCUyMHRvJTIwYSUyMHByaXZhdGUlMjByZXBvJTJDJTIwc2ltcGx5JTIwcGFzcyUyMHByaXZhdGUlM0RUcnVlJTNBJTBBZGF0YXNldC5wdXNoX3RvX2h1YiglMjJuYW1lX29mX3lvdXJfZGF0YXNldCUyMiUyQyUyMHByaXZhdGUlM0RUcnVlKQ==",highlighted:`<span class="hljs-comment"># assuming you have ran the hf auth login command in a terminal</span>
dataset.push_to_hub(<span class="hljs-string">&quot;name_of_your_dataset&quot;</span>)

<span class="hljs-comment"># if you want to push to a private repo, simply pass private=True:</span>
dataset.push_to_hub(<span class="hljs-string">&quot;name_of_your_dataset&quot;</span>, private=<span class="hljs-literal">True</span>)`,wrap:!1}}),R=new q({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMC0tbWl4ZWRfcHJlY2lzaW9uJTNEJTIyZnAxNiUyMiUyMCUyMHRyYWluX3RleHRfdG9faW1hZ2UucHklMjAlNUMlMEElMjAlMjAtLXByZXRyYWluZWRfbW9kZWxfbmFtZV9vcl9wYXRoJTNEJTIyc3RhYmxlLWRpZmZ1c2lvbi12MS01JTJGc3RhYmxlLWRpZmZ1c2lvbi12MS01JTIyJTIwJTVDJTBBJTIwJTIwLS1kYXRhc2V0X25hbWUlM0QlMjJuYW1lX29mX3lvdXJfZGF0YXNldCUyMiUyMCU1QyUwQSUyMCUyMCUzQ290aGVyLWFyZ3VtZW50cyUzRQ==",highlighted:`accelerate launch --mixed_precision=<span class="hljs-string">&quot;fp16&quot;</span>  train_text_to_image.py \\
  --pretrained_model_name_or_path=<span class="hljs-string">&quot;stable-diffusion-v1-5/stable-diffusion-v1-5&quot;</span> \\
  --dataset_name=<span class="hljs-string">&quot;name_of_your_dataset&quot;</span> \\
  &lt;other-arguments&gt;`,wrap:!1}}),C=new gt({props:{title:"Next steps",local:"next-steps",headingTag:"h2"}}),W=new Ft({props:{source:"https://github.com/huggingface/diffusers/blob/main/docs/source/en/training/create_dataset.md"}}),{c(){f=o("meta"),L=s(),z=o("p"),E=s(),d(y.$$.fragment),N=s(),d(T.$$.fragment),Q=s(),J=o("p"),J.innerHTML=Tt,A=s(),w=o("p"),w.textContent=Jt,S=s(),j=o("ul"),j.innerHTML=wt,V=s(),h=o("blockquote"),h.innerHTML=jt,P=s(),d(_.$$.fragment),D=s(),b=o("p"),b.innerHTML=_t,K=s(),d(Z.$$.fragment),O=s(),$=o("p"),$.innerHTML=bt,tt=s(),d(U.$$.fragment),et=s(),d(k.$$.fragment),at=s(),g=o("blockquote"),g.innerHTML=Zt,lt=s(),I=o("p"),I.innerHTML=$t,st=s(),G=o("p"),G.innerHTML=Ut,nt=s(),d(X.$$.fragment),ot=s(),B=o("p"),B.innerHTML=kt,it=s(),d(v.$$.fragment),rt=s(),x=o("p"),x.innerHTML=It,dt=s(),d(R.$$.fragment),pt=s(),d(C.$$.fragment),mt=s(),H=o("p"),H.innerHTML=Gt,ut=s(),Y=o("p"),Y.innerHTML=Xt,Mt=s(),d(W.$$.fragment),ct=s(),F=o("p"),this.h()},l(t){const e=Yt("svelte-u9bgzb",document.head);f=i(e,"META",{name:!0,content:!0}),e.forEach(a),L=n(t),z=i(t,"P",{}),Bt(z).forEach(a),E=n(t),p(y.$$.fragment,t),N=n(t),p(T.$$.fragment,t),Q=n(t),J=i(t,"P",{"data-svelte-h":!0}),r(J)!=="svelte-1kq6fl5"&&(J.innerHTML=Tt),A=n(t),w=i(t,"P",{"data-svelte-h":!0}),r(w)!=="svelte-1ki6nhc"&&(w.textContent=Jt),S=n(t),j=i(t,"UL",{"data-svelte-h":!0}),r(j)!=="svelte-136e0w1"&&(j.innerHTML=wt),V=n(t),h=i(t,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),r(h)!=="svelte-1f197i8"&&(h.innerHTML=jt),P=n(t),p(_.$$.fragment,t),D=n(t),b=i(t,"P",{"data-svelte-h":!0}),r(b)!=="svelte-1tbm8ke"&&(b.innerHTML=_t),K=n(t),p(Z.$$.fragment,t),O=n(t),$=i(t,"P",{"data-svelte-h":!0}),r($)!=="svelte-16r2a6g"&&($.innerHTML=bt),tt=n(t),p(U.$$.fragment,t),et=n(t),p(k.$$.fragment,t),at=n(t),g=i(t,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),r(g)!=="svelte-f9us11"&&(g.innerHTML=Zt),lt=n(t),I=i(t,"P",{"data-svelte-h":!0}),r(I)!=="svelte-117nzzz"&&(I.innerHTML=$t),st=n(t),G=i(t,"P",{"data-svelte-h":!0}),r(G)!=="svelte-10hf74w"&&(G.innerHTML=Ut),nt=n(t),p(X.$$.fragment,t),ot=n(t),B=i(t,"P",{"data-svelte-h":!0}),r(B)!=="svelte-2nvk44"&&(B.innerHTML=kt),it=n(t),p(v.$$.fragment,t),rt=n(t),x=i(t,"P",{"data-svelte-h":!0}),r(x)!=="svelte-19t9o3p"&&(x.innerHTML=It),dt=n(t),p(R.$$.fragment,t),pt=n(t),p(C.$$.fragment,t),mt=n(t),H=i(t,"P",{"data-svelte-h":!0}),r(H)!=="svelte-1ata9m5"&&(H.innerHTML=Gt),ut=n(t),Y=i(t,"P",{"data-svelte-h":!0}),r(Y)!=="svelte-oyixgb"&&(Y.innerHTML=Xt),Mt=n(t),p(W.$$.fragment,t),ct=n(t),F=i(t,"P",{}),Bt(F).forEach(a),this.h()},h(){ht(f,"name","hf:doc:metadata"),ht(f,"content",Lt),ht(h,"class","tip"),ht(g,"class","tip")},m(t,e){Wt(document.head,f),l(t,L,e),l(t,z,e),l(t,E,e),m(y,t,e),l(t,N,e),m(T,t,e),l(t,Q,e),l(t,J,e),l(t,A,e),l(t,w,e),l(t,S,e),l(t,j,e),l(t,V,e),l(t,h,e),l(t,P,e),m(_,t,e),l(t,D,e),l(t,b,e),l(t,K,e),m(Z,t,e),l(t,O,e),l(t,$,e),l(t,tt,e),m(U,t,e),l(t,et,e),m(k,t,e),l(t,at,e),l(t,g,e),l(t,lt,e),l(t,I,e),l(t,st,e),l(t,G,e),l(t,nt,e),m(X,t,e),l(t,ot,e),l(t,B,e),l(t,it,e),m(v,t,e),l(t,rt,e),l(t,x,e),l(t,dt,e),m(R,t,e),l(t,pt,e),m(C,t,e),l(t,mt,e),l(t,H,e),l(t,ut,e),l(t,Y,e),l(t,Mt,e),m(W,t,e),l(t,ct,e),l(t,F,e),ft=!0},p:xt,i(t){ft||(u(y.$$.fragment,t),u(T.$$.fragment,t),u(_.$$.fragment,t),u(Z.$$.fragment,t),u(U.$$.fragment,t),u(k.$$.fragment,t),u(X.$$.fragment,t),u(v.$$.fragment,t),u(R.$$.fragment,t),u(C.$$.fragment,t),u(W.$$.fragment,t),ft=!0)},o(t){M(y.$$.fragment,t),M(T.$$.fragment,t),M(_.$$.fragment,t),M(Z.$$.fragment,t),M(U.$$.fragment,t),M(k.$$.fragment,t),M(X.$$.fragment,t),M(v.$$.fragment,t),M(R.$$.fragment,t),M(C.$$.fragment,t),M(W.$$.fragment,t),ft=!1},d(t){t&&(a(L),a(z),a(E),a(N),a(Q),a(J),a(A),a(w),a(S),a(j),a(V),a(h),a(P),a(D),a(b),a(K),a(O),a($),a(tt),a(et),a(at),a(g),a(lt),a(I),a(st),a(G),a(nt),a(ot),a(B),a(it),a(rt),a(x),a(dt),a(pt),a(mt),a(H),a(ut),a(Y),a(Mt),a(ct),a(F)),a(f),c(y,t),c(T,t),c(_,t),c(Z,t),c(U,t),c(k,t),c(X,t),c(v,t),c(R,t),c(C,t),c(W,t)}}}const Lt='{"title":"Create a dataset for training","local":"create-a-dataset-for-training","sections":[{"title":"Provide a dataset as a folder","local":"provide-a-dataset-as-a-folder","sections":[],"depth":2},{"title":"Upload your data to the Hub","local":"upload-your-data-to-the-hub","sections":[],"depth":2},{"title":"Next steps","local":"next-steps","sections":[],"depth":2}],"depth":1}';function Et(yt){return Rt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Pt extends Ct{constructor(f){super(),Ht(this,f,Et,qt,vt,{})}}export{Pt as component};
