import{s as Ue,o as Ie,n as Le}from"../chunks/scheduler.d9d004e7.js";import{S as Ce,i as Ee,e as n,s,c as E,h as He,a as l,d as t,b as r,f as h,g as H,j as d,k as je,l as i,m as o,n as Z,t as X,o as B,p as A}from"../chunks/index.4a2b774c.js";import{T as Ze}from"../chunks/Tip.cf0287e4.js";import{C as Xe,H as Be,E as Ae}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.2d77c8e6.js";import{C as he}from"../chunks/CodeBlock.8806415f.js";function Ge(D){let u,M='You can set the training argument <code>--distribution_strategy fast_ddp</code> for simpler and usually faster distributed training management. More information <a href="../usage_guides/accelerate_training#fast-ddp">here</a>.';return{c(){u=n("p"),u.innerHTML=M},l(p){u=l(p,"P",{"data-svelte-h":!0}),d(u)!=="svelte-qxsr5k"&&(u.innerHTML=M)},m(p,G){o(p,u,G)},p:Le,d(p){p&&t(u)}}}function ke(D){let u,M,p,G,w,Y,_,Q,b,Me="As models get bigger, parallelism has emerged as a strategy for training larger models on limited hardware and accelerating training speed by several orders of magnitude.",q,y,we=`All the <a href="/examples">PyTorch examples</a> and the <code>GaudiTrainer</code> script work out of the box with distributed training.
There are two ways of launching them:`,O,f,$,k,_e='Using the <a href="https://github.com/huggingface/optimum-habana/blob/main/examples/gaudi_spawn.py" rel="nofollow">gaudi_spawn.py</a> script:',oe,T,m,z,be="Use MPI for distributed training:",ue,J,pe,W,ye=`where <code>--argX</code> is an argument of the script to run in a distributed way.
Examples are given for question answering <a href="https://github.com/huggingface/optimum-habana/blob/main/examples/question-answering/README.md#multi-card-training" rel="nofollow">here</a> and text classification <a href="/examples/text-classification#multi-card-training">here</a>.`,de,c,P,$e="Use DeepSpeed for distributed training:",me,v,ce,S,Te=`where <code>--argX</code> is an argument of the script to run in a distributed way.
Examples are given for question answering <a href="https://github.com/huggingface/optimum-habana/blob/main/examples/question-answering/README.md#using-deepspeed" rel="nofollow">here</a> and text classification <a href="/examples/text-classification#using-deepspeed">here</a>.`,fe,x,F,Je="Using the <code>DistributedRunner</code> directly in code:",ge,j,K,g,ee,U,ve="To go further, we invite you to read our guides about:",te,I,xe='<li><a href="../usage_guides/accelerate_training">Accelerating training</a></li> <li><a href="../usage_guides/pretraining">Pretraining</a></li> <li><a href="../usage_guides/deepspeed">DeepSpeed</a> to train bigger models</li> <li><a href="../usage_guides/multi_node_training">Multi-node training</a> to speed up even more your distributed runs</li>',ae,L,ie,R,ne;return w=new Xe({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),_=new Be({props:{title:"Distributed training with Optimum for Intel Gaudi",local:"distributed-training-with-optimum-for-intel-gaudi",headingTag:"h1"}}),J=new he({props:{code:"cHl0aG9uJTIwZ2F1ZGlfc3Bhd24ucHklMjAlNUMlMEElMjAlMjAlMjAlMjAtLXdvcmxkX3NpemUlMjBudW1iZXJfb2ZfaHB1X3lvdV9oYXZlJTIwLS11c2VfbXBpJTIwJTVDJTBBJTIwJTIwJTIwJTIwcGF0aF90b19zY3JpcHQucHklMjAtLWFyZ3MxJTIwLS1hcmdzMiUyMC4uLiUyMC0tYXJnc04=",highlighted:`python gaudi_spawn.py \\
    --world_size number_of_hpu_you_have --use_mpi \\
    path_to_script.py --args1 --args2 ... --argsN`,wrap:!1}}),v=new he({props:{code:"cHl0aG9uJTIwZ2F1ZGlfc3Bhd24ucHklMjAlNUMlMEElMjAlMjAlMjAlMjAtLXdvcmxkX3NpemUlMjBudW1iZXJfb2ZfaHB1X3lvdV9oYXZlJTIwLS11c2VfZGVlcHNwZWVkJTIwJTVDJTBBJTIwJTIwJTIwJTIwcGF0aF90b19zY3JpcHQucHklMjAtLWFyZ3MxJTIwLS1hcmdzMiUyMC4uLiUyMC0tYXJnc04=",highlighted:`python gaudi_spawn.py \\
    --world_size number_of_hpu_you_have --use_deepspeed \\
    path_to_script.py --args1 --args2 ... --argsN`,wrap:!1}}),j=new he({props:{code:"ZnJvbSUyMG9wdGltdW0uaGFiYW5hLmRpc3RyaWJ1dGVkJTIwaW1wb3J0JTIwRGlzdHJpYnV0ZWRSdW5uZXIlMEFmcm9tJTIwb3B0aW11bS51dGlscyUyMGltcG9ydCUyMGxvZ2dpbmclMEElMEF3b3JsZF9zaXplJTNEOCUyMCUyMyUyME51bWJlciUyMG9mJTIwSFBVcyUyMHRvJTIwdXNlJTIwKDElMjBvciUyMDgpJTBBJTBBJTIzJTIwZGVmaW5lJTIwZGlzdHJpYnV0ZWQlMjBydW5uZXIlMEFkaXN0cmlidXRlZF9ydW5uZXIlMjAlM0QlMjBEaXN0cmlidXRlZFJ1bm5lciglMEElMjAlMjAlMjAlMjBjb21tYW5kX2xpc3QlM0QlNUIlMjJzY3JpcHRzJTJGdHJhaW4ucHklMjAtLWFyZ3MxJTIwLS1hcmdzMiUyMC4uLiUyMC0tYXJnc04lMjIlNUQlMkMlMEElMjAlMjAlMjAlMjB3b3JsZF9zaXplJTNEd29ybGRfc2l6ZSUyQyUwQSUyMCUyMCUyMCUyMHVzZV9tcGklM0RUcnVlJTJDJTBBKSUwQSUwQSUyMyUyMHN0YXJ0JTIwam9iJTBBcmV0X2NvZGUlMjAlM0QlMjBkaXN0cmlidXRlZF9ydW5uZXIucnVuKCk=",highlighted:`<span class="hljs-keyword">from</span> optimum.habana.distributed <span class="hljs-keyword">import</span> DistributedRunner
<span class="hljs-keyword">from</span> optimum.utils <span class="hljs-keyword">import</span> logging

world_size=<span class="hljs-number">8</span> <span class="hljs-comment"># Number of HPUs to use (1 or 8)</span>

<span class="hljs-comment"># define distributed runner</span>
distributed_runner = DistributedRunner(
    command_list=[<span class="hljs-string">&quot;scripts/train.py --args1 --args2 ... --argsN&quot;</span>],
    world_size=world_size,
    use_mpi=<span class="hljs-literal">True</span>,
)

<span class="hljs-comment"># start job</span>
ret_code = distributed_runner.run()`,wrap:!1}}),g=new Ze({props:{$$slots:{default:[Ge]},$$scope:{ctx:D}}}),L=new Ae({props:{source:"https://github.com/huggingface/optimum-habana/blob/main/docs/source/tutorials/distributed.mdx"}}),{c(){u=n("meta"),M=s(),p=n("p"),G=s(),E(w.$$.fragment),Y=s(),E(_.$$.fragment),Q=s(),b=n("p"),b.textContent=Me,q=s(),y=n("p"),y.innerHTML=we,O=s(),f=n("ol"),$=n("li"),k=n("p"),k.innerHTML=_e,oe=s(),T=n("ul"),m=n("li"),z=n("p"),z.textContent=be,ue=s(),E(J.$$.fragment),pe=s(),W=n("p"),W.innerHTML=ye,de=s(),c=n("li"),P=n("p"),P.textContent=$e,me=s(),E(v.$$.fragment),ce=s(),S=n("p"),S.innerHTML=Te,fe=s(),x=n("li"),F=n("p"),F.innerHTML=Je,ge=s(),E(j.$$.fragment),K=s(),E(g.$$.fragment),ee=s(),U=n("p"),U.textContent=ve,te=s(),I=n("ul"),I.innerHTML=xe,ae=s(),E(L.$$.fragment),ie=s(),R=n("p"),this.h()},l(e){const a=He("svelte-u9bgzb",document.head);u=l(a,"META",{name:!0,content:!0}),a.forEach(t),M=r(e),p=l(e,"P",{}),h(p).forEach(t),G=r(e),H(w.$$.fragment,e),Y=r(e),H(_.$$.fragment,e),Q=r(e),b=l(e,"P",{"data-svelte-h":!0}),d(b)!=="svelte-89pyl9"&&(b.textContent=Me),q=r(e),y=l(e,"P",{"data-svelte-h":!0}),d(y)!=="svelte-7zd0cx"&&(y.innerHTML=we),O=r(e),f=l(e,"OL",{});var C=h(f);$=l(C,"LI",{});var le=h($);k=l(le,"P",{"data-svelte-h":!0}),d(k)!=="svelte-12ujwdq"&&(k.innerHTML=_e),oe=r(le),T=l(le,"UL",{});var se=h(T);m=l(se,"LI",{});var N=h(m);z=l(N,"P",{"data-svelte-h":!0}),d(z)!=="svelte-azmcdt"&&(z.textContent=be),ue=r(N),H(J.$$.fragment,N),pe=r(N),W=l(N,"P",{"data-svelte-h":!0}),d(W)!=="svelte-1v8vaeh"&&(W.innerHTML=ye),N.forEach(t),de=r(se),c=l(se,"LI",{});var V=h(c);P=l(V,"P",{"data-svelte-h":!0}),d(P)!=="svelte-bign48"&&(P.textContent=$e),me=r(V),H(v.$$.fragment,V),ce=r(V),S=l(V,"P",{"data-svelte-h":!0}),d(S)!=="svelte-1vlvd2z"&&(S.innerHTML=Te),V.forEach(t),se.forEach(t),le.forEach(t),fe=r(C),x=l(C,"LI",{});var re=h(x);F=l(re,"P",{"data-svelte-h":!0}),d(F)!=="svelte-1ugz6yl"&&(F.innerHTML=Je),ge=r(re),H(j.$$.fragment,re),re.forEach(t),C.forEach(t),K=r(e),H(g.$$.fragment,e),ee=r(e),U=l(e,"P",{"data-svelte-h":!0}),d(U)!=="svelte-qtbvas"&&(U.textContent=ve),te=r(e),I=l(e,"UL",{"data-svelte-h":!0}),d(I)!=="svelte-1eggs5o"&&(I.innerHTML=xe),ae=r(e),H(L.$$.fragment,e),ie=r(e),R=l(e,"P",{}),h(R).forEach(t),this.h()},h(){je(u,"name","hf:doc:metadata"),je(u,"content",ze)},m(e,a){i(document.head,u),o(e,M,a),o(e,p,a),o(e,G,a),Z(w,e,a),o(e,Y,a),Z(_,e,a),o(e,Q,a),o(e,b,a),o(e,q,a),o(e,y,a),o(e,O,a),o(e,f,a),i(f,$),i($,k),i($,oe),i($,T),i(T,m),i(m,z),i(m,ue),Z(J,m,null),i(m,pe),i(m,W),i(T,de),i(T,c),i(c,P),i(c,me),Z(v,c,null),i(c,ce),i(c,S),i(f,fe),i(f,x),i(x,F),i(x,ge),Z(j,x,null),o(e,K,a),Z(g,e,a),o(e,ee,a),o(e,U,a),o(e,te,a),o(e,I,a),o(e,ae,a),Z(L,e,a),o(e,ie,a),o(e,R,a),ne=!0},p(e,[a]){const C={};a&2&&(C.$$scope={dirty:a,ctx:e}),g.$set(C)},i(e){ne||(X(w.$$.fragment,e),X(_.$$.fragment,e),X(J.$$.fragment,e),X(v.$$.fragment,e),X(j.$$.fragment,e),X(g.$$.fragment,e),X(L.$$.fragment,e),ne=!0)},o(e){B(w.$$.fragment,e),B(_.$$.fragment,e),B(J.$$.fragment,e),B(v.$$.fragment,e),B(j.$$.fragment,e),B(g.$$.fragment,e),B(L.$$.fragment,e),ne=!1},d(e){e&&(t(M),t(p),t(G),t(Y),t(Q),t(b),t(q),t(y),t(O),t(f),t(K),t(ee),t(U),t(te),t(I),t(ae),t(ie),t(R)),t(u),A(w,e),A(_,e),A(J),A(v),A(j),A(g,e),A(L,e)}}}const ze='{"title":"Distributed training with Optimum for Intel Gaudi","local":"distributed-training-with-optimum-for-intel-gaudi","sections":[],"depth":1}';function We(D){return Ie(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Re extends Ce{constructor(u){super(),Ee(this,u,We,ke,Ue,{})}}export{Re as component};
