import{s as Ke,n as et,o as tt}from"../chunks/scheduler.d9d004e7.js";import{S as lt,i as nt,e as s,s as i,c as p,h as it,a as o,d as l,b as a,f as qe,g as r,j as m,k as Ze,l as at,m as n,n as u,t as d,o as f,p as h}from"../chunks/index.4a2b774c.js";import{C as st,H as M,E as ot}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.02c268bd.js";import{C as O}from"../chunks/CodeBlock.67ad8ed0.js";function mt(xe){let c,q,N,K,T,ee,_,te,g,Re="Using several Gaudi servers to perform multi-node training can be done easily. This guide shows how to:",le,y,Se="<li>set up several Gaudi instances</li> <li>set up your computing environment</li> <li>launch a multi-node run</li>",ne,J,ie,$,We="Two types of configurations are possible:",ae,U,Ae="<li>scale-out using Gaudi NICs or Host NICs (on-premises)</li> <li>scale-out using Intel® Tiber™ AI Cloud instances</li>",se,I,oe,b,Ge='To set up your servers on premises, check out the <a href="https://docs.habana.ai/en/latest/Installation_Guide/Bare_Metal_Fresh_OS.html" rel="nofollow">installation</a> and <a href="https://docs.habana.ai/en/latest/PyTorch/PyTorch_Scaling_Guide/index.html" rel="nofollow">distributed training</a> pages of Intel® Gaudi® AI Accelerator’s documentation.',me,v,pe,C,He='Follow the steps on <a href="https://docs.habana.ai/en/latest/Intel_DevCloud_Quick_Start/Intel_DevCloud_Quick_Start.html#creating-an-account-and-getting-an-instance" rel="nofollow">creating an account and getting an instance</a> pages of Intel® Gaudi® AI Accelerator’s documentation.',re,X,ue,j,Ve='Once your Intel Gaudi instances are ready, follow the steps for <a href="https://docs.habana.ai/en/latest/Intel_DevCloud_Quick_Start/Intel_DevCloud_Quick_Start.html#setting-up-a-multi-server-environment" rel="nofollow">setting up a multi-server environment</a> pages of Intel® Gaudi® AI Accelerator’s documentation.',de,L,ke="Finally, there are two possible ways to run your training script on several nodes:",fe,B,Fe='<li>With the <a href="https://github.com/huggingface/optimum-habana/blob/main/examples/gaudi_spawn.py" rel="nofollow"><code>gaudi_spawn.py</code></a> script, you can run the following command:</li>',he,Z,ce,x,Ee="where <code>--argX</code> is an argument of the script to run.",we,w,Ye="<li>With the <code>DistributedRunner</code>, you can add this code snippet to a script:</li>",Me,R,Te,S,_e,W,Qe='If you need to set environment variables for all nodes, you can specify them in a <a href="https://www.deepspeed.ai/getting-started/#multi-node-environment-variables" rel="nofollow"><code>.deepspeed_env</code></a> file which should be located in the local path you are executing from or in your home directory. The format is the following:',ge,A,ye,G,Je,H,ze='<li>It is strongly recommended to use gradient checkpointing for multi-node runs to get the highest speedups. You can enable it with <code>--gradient_checkpointing</code> in <a href="/examples">these examples</a> or with <code>gradient_checkpointing=True</code> in your <code>GaudiTrainingArguments</code>.</li> <li>Larger batch sizes should lead to higher speedups.</li> <li>Multi-node inference is not recommended and can provide inconsistent results.</li> <li>On Intel Tiber AI Cloud instances, run your Docker containers with the <code>--privileged</code> flag so that EFA devices are visible.</li>',$e,V,Ue,k,De=`In this example, we fine-tune a pre-trained GPT2-XL model on the <a href="https://huggingface.co/datasets/wikitext" rel="nofollow">WikiText dataset</a>.
We are going to use the <a href="/examples/language-modeling#gpt-2gpt-and-causal-language-modeling">causal language modeling example which is given in the Github repository</a>.`,Ie,F,Ne="The first step consists in training the model on several nodes with this command:",be,E,ve,Y,Pe="Evaluation is not performed in the same command because we do not recommend performing multi-node inference at the moment.",Ce,Q,Oe=`Once the model is trained, we can evaluate it with the following command.
The argument <code>--model_name_or_path</code> should be equal to the argument <code>--output_dir</code> of the previous command.`,Xe,z,je,D,Le,P,Be;return T=new st({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),_=new M({props:{title:"Multi-node Training",local:"multi-node-training",headingTag:"h1"}}),J=new M({props:{title:"Setting up several Gaudi instances",local:"setting-up-several-gaudi-instances",headingTag:"h2"}}),I=new M({props:{title:"On premises",local:"on-premises",headingTag:"h3"}}),v=new M({props:{title:"Intel Tiber AI Cloud instances",local:"intel-tiber-ai-cloud-instances",headingTag:"h3"}}),X=new M({props:{title:"Launching a Multi-node Run",local:"launching-a-multi-node-run",headingTag:"h2"}}),Z=new O({props:{code:"cHl0aG9uJTIwZ2F1ZGlfc3Bhd24ucHklMjAlNUMlMEElMjAlMjAlMjAlMjAtLWhvc3RmaWxlJTIwcGF0aF90b19teV9ob3N0ZmlsZSUyMC0tdXNlX2RlZXBzcGVlZCUyMCU1QyUwQSUyMCUyMCUyMCUyMHBhdGhfdG9fbXlfc2NyaXB0LnB5JTIwLS1hcmdzMSUyMC0tYXJnczIlMjAuLi4lMjAtLWFyZ3NOJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1kZWVwc3BlZWQlMjBwYXRoX3RvX215X2RlZXBzcGVlZF9jb25maWc=",highlighted:`python gaudi_spawn.py \\
    --hostfile path_to_my_hostfile --use_deepspeed \\
    path_to_my_script.py --args1 --args2 ... --argsN \\
    --deepspeed path_to_my_deepspeed_config`,wrap:!1}}),R=new O({props:{code:"ZnJvbSUyMG9wdGltdW0uaGFiYW5hLmRpc3RyaWJ1dGVkJTIwaW1wb3J0JTIwRGlzdHJpYnV0ZWRSdW5uZXIlMEElMEFkaXN0cmlidXRlZF9ydW5uZXIlMjAlM0QlMjBEaXN0cmlidXRlZFJ1bm5lciglMEElMjAlMjAlMjAlMjBjb21tYW5kX2xpc3QlM0QlNUIlMjJwYXRoX3RvX215X3NjcmlwdC5weSUyMC0tYXJnczElMjAtLWFyZ3MyJTIwLi4uJTIwLS1hcmdzTiUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMGhvc3RmaWxlJTNEcGF0aF90b19teV9ob3N0ZmlsZSUyQyUwQSUyMCUyMCUyMCUyMHVzZV9kZWVwc3BlZWQlM0RUcnVlJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> optimum.habana.distributed <span class="hljs-keyword">import</span> DistributedRunner

distributed_runner = DistributedRunner(
    command_list=[<span class="hljs-string">&quot;path_to_my_script.py --args1 --args2 ... --argsN&quot;</span>],
    hostfile=path_to_my_hostfile,
    use_deepspeed=<span class="hljs-literal">True</span>,
)`,wrap:!1}}),S=new M({props:{title:"Environment Variables",local:"environment-variables",headingTag:"h2"}}),A=new O({props:{code:"ZW52X3ZhcmlhYmxlXzFfbmFtZSUzRHZhbHVlJTBBZW52X3ZhcmlhYmxlXzJfbmFtZSUzRHZhbHVlJTBBLi4u",highlighted:`<span class="hljs-attribute">env_variable_1_name</span>=value
<span class="hljs-attribute">env_variable_2_name</span>=value
<span class="hljs-built_in">..</span>.`,wrap:!1}}),G=new M({props:{title:"Recommendations",local:"recommendations",headingTag:"h2"}}),V=new M({props:{title:"Example",local:"example",headingTag:"h2"}}),E=new O({props:{code:"UFRfSFBVX0xBWllfTU9ERSUzRDElMjBweXRob24lMjAuLiUyRmdhdWRpX3NwYXduLnB5JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1ob3N0ZmlsZSUyMHBhdGhfdG9faG9zdGZpbGUlMjAtLXVzZV9kZWVwc3BlZWQlMjBydW5fY2xtLnB5JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tb2RlbF9uYW1lX29yX3BhdGglMjBncHQyLXhsJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1nYXVkaV9jb25maWdfbmFtZSUyMEhhYmFuYSUyRmdwdDIlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWRhdGFzZXRfbmFtZSUyMHdpa2l0ZXh0JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1kYXRhc2V0X2NvbmZpZ19uYW1lJTIwd2lraXRleHQtMi1yYXctdjElMjAlNUMlMEElMjAlMjAlMjAlMjAtLWRvX3RyYWluJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1vdXRwdXRfZGlyJTIwJTJGdG1wJTJGZ3B0Ml94bF9tdWx0aV9ub2RlJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1sZWFybmluZ19yYXRlJTIwNGUtMDQlMjAlNUMlMEElMjAlMjAlMjAlMjAtLXBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUyMDE2JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1ncmFkaWVudF9jaGVja3BvaW50aW5nJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1udW1fdHJhaW5fZXBvY2hzJTIwMSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tdXNlX2hhYmFuYSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tdXNlX2xhenlfbW9kZSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tdGhyb3VnaHB1dF93YXJtdXBfc3RlcHMlMjAzJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1kZWVwc3BlZWQlMjBwYXRoX3RvX2RlZXBzcGVlZF9jb25maWc=",highlighted:`PT_HPU_LAZY_MODE=1 python ../gaudi_spawn.py \\
    --hostfile path_to_hostfile --use_deepspeed run_clm.py \\
    --model_name_or_path gpt2-xl \\
    --gaudi_config_name Habana/gpt2 \\
    --dataset_name wikitext \\
    --dataset_config_name wikitext-2-raw-v1 \\
    --do_train \\
    --output_dir /tmp/gpt2_xl_multi_node \\
    --learning_rate 4e-04 \\
    --per_device_train_batch_size 16 \\
    --gradient_checkpointing \\
    --num_train_epochs 1 \\
    --use_habana \\
    --use_lazy_mode \\
    --throughput_warmup_steps 3 \\
    --deepspeed path_to_deepspeed_config`,wrap:!1}}),z=new O({props:{code:"UFRfSFBVX0xBWllfTU9ERSUzRDElMjBweXRob24lMjBydW5fY2xtLnB5JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tb2RlbF9uYW1lX29yX3BhdGglMjAlMkZ0bXAlMkZncHQyX3hsX211bHRpX25vZGUlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWdhdWRpX2NvbmZpZ19uYW1lJTIwSGFiYW5hJTJGZ3B0MiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tZGF0YXNldF9uYW1lJTIwd2lraXRleHQlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWRhdGFzZXRfY29uZmlnX25hbWUlMjB3aWtpdGV4dC0yLXJhdy12MSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tZG9fZXZhbCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tb3V0cHV0X2RpciUyMCUyRnRtcCUyRmdwdDJfeGxfbXVsdGlfbm9kZSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tcGVyX2RldmljZV9ldmFsX2JhdGNoX3NpemUlMjA4JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS11c2VfaGFiYW5hJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS11c2VfbGF6eV9tb2Rl",highlighted:`PT_HPU_LAZY_MODE=1 python run_clm.py \\
    --model_name_or_path /tmp/gpt2_xl_multi_node \\
    --gaudi_config_name Habana/gpt2 \\
    --dataset_name wikitext \\
    --dataset_config_name wikitext-2-raw-v1 \\
    --do_eval \\
    --output_dir /tmp/gpt2_xl_multi_node \\
    --per_device_eval_batch_size 8 \\
    --use_habana \\
    --use_lazy_mode`,wrap:!1}}),D=new ot({props:{source:"https://github.com/huggingface/optimum-habana/blob/main/docs/source/usage_guides/multi_node_training.mdx"}}),{c(){c=s("meta"),q=i(),N=s("p"),K=i(),p(T.$$.fragment),ee=i(),p(_.$$.fragment),te=i(),g=s("p"),g.textContent=Re,le=i(),y=s("ul"),y.innerHTML=Se,ne=i(),p(J.$$.fragment),ie=i(),$=s("p"),$.textContent=We,ae=i(),U=s("ul"),U.innerHTML=Ae,se=i(),p(I.$$.fragment),oe=i(),b=s("p"),b.innerHTML=Ge,me=i(),p(v.$$.fragment),pe=i(),C=s("p"),C.innerHTML=He,re=i(),p(X.$$.fragment),ue=i(),j=s("p"),j.innerHTML=Ve,de=i(),L=s("p"),L.textContent=ke,fe=i(),B=s("ol"),B.innerHTML=Fe,he=i(),p(Z.$$.fragment),ce=i(),x=s("p"),x.innerHTML=Ee,we=i(),w=s("ol"),w.innerHTML=Ye,Me=i(),p(R.$$.fragment),Te=i(),p(S.$$.fragment),_e=i(),W=s("p"),W.innerHTML=Qe,ge=i(),p(A.$$.fragment),ye=i(),p(G.$$.fragment),Je=i(),H=s("ul"),H.innerHTML=ze,$e=i(),p(V.$$.fragment),Ue=i(),k=s("p"),k.innerHTML=De,Ie=i(),F=s("p"),F.textContent=Ne,be=i(),p(E.$$.fragment),ve=i(),Y=s("p"),Y.textContent=Pe,Ce=i(),Q=s("p"),Q.innerHTML=Oe,Xe=i(),p(z.$$.fragment),je=i(),p(D.$$.fragment),Le=i(),P=s("p"),this.h()},l(e){const t=it("svelte-u9bgzb",document.head);c=o(t,"META",{name:!0,content:!0}),t.forEach(l),q=a(e),N=o(e,"P",{}),qe(N).forEach(l),K=a(e),r(T.$$.fragment,e),ee=a(e),r(_.$$.fragment,e),te=a(e),g=o(e,"P",{"data-svelte-h":!0}),m(g)!=="svelte-qp9xqj"&&(g.textContent=Re),le=a(e),y=o(e,"UL",{"data-svelte-h":!0}),m(y)!=="svelte-rop6rm"&&(y.innerHTML=Se),ne=a(e),r(J.$$.fragment,e),ie=a(e),$=o(e,"P",{"data-svelte-h":!0}),m($)!=="svelte-se31mi"&&($.textContent=We),ae=a(e),U=o(e,"UL",{"data-svelte-h":!0}),m(U)!=="svelte-1ei7enr"&&(U.innerHTML=Ae),se=a(e),r(I.$$.fragment,e),oe=a(e),b=o(e,"P",{"data-svelte-h":!0}),m(b)!=="svelte-1vji76k"&&(b.innerHTML=Ge),me=a(e),r(v.$$.fragment,e),pe=a(e),C=o(e,"P",{"data-svelte-h":!0}),m(C)!=="svelte-1dltmt2"&&(C.innerHTML=He),re=a(e),r(X.$$.fragment,e),ue=a(e),j=o(e,"P",{"data-svelte-h":!0}),m(j)!=="svelte-idgjfb"&&(j.innerHTML=Ve),de=a(e),L=o(e,"P",{"data-svelte-h":!0}),m(L)!=="svelte-ce1e0i"&&(L.textContent=ke),fe=a(e),B=o(e,"OL",{"data-svelte-h":!0}),m(B)!=="svelte-2kg0w5"&&(B.innerHTML=Fe),he=a(e),r(Z.$$.fragment,e),ce=a(e),x=o(e,"P",{"data-svelte-h":!0}),m(x)!=="svelte-nmkk8f"&&(x.innerHTML=Ee),we=a(e),w=o(e,"OL",{start:!0,"data-svelte-h":!0}),m(w)!=="svelte-1sdtmds"&&(w.innerHTML=Ye),Me=a(e),r(R.$$.fragment,e),Te=a(e),r(S.$$.fragment,e),_e=a(e),W=o(e,"P",{"data-svelte-h":!0}),m(W)!=="svelte-1vzj3zu"&&(W.innerHTML=Qe),ge=a(e),r(A.$$.fragment,e),ye=a(e),r(G.$$.fragment,e),Je=a(e),H=o(e,"UL",{"data-svelte-h":!0}),m(H)!=="svelte-1rp99nc"&&(H.innerHTML=ze),$e=a(e),r(V.$$.fragment,e),Ue=a(e),k=o(e,"P",{"data-svelte-h":!0}),m(k)!=="svelte-13cuqwv"&&(k.innerHTML=De),Ie=a(e),F=o(e,"P",{"data-svelte-h":!0}),m(F)!=="svelte-1y1hqcv"&&(F.textContent=Ne),be=a(e),r(E.$$.fragment,e),ve=a(e),Y=o(e,"P",{"data-svelte-h":!0}),m(Y)!=="svelte-472z3o"&&(Y.textContent=Pe),Ce=a(e),Q=o(e,"P",{"data-svelte-h":!0}),m(Q)!=="svelte-1lr4pmk"&&(Q.innerHTML=Oe),Xe=a(e),r(z.$$.fragment,e),je=a(e),r(D.$$.fragment,e),Le=a(e),P=o(e,"P",{}),qe(P).forEach(l),this.h()},h(){Ze(c,"name","hf:doc:metadata"),Ze(c,"content",pt),Ze(w,"start","2")},m(e,t){at(document.head,c),n(e,q,t),n(e,N,t),n(e,K,t),u(T,e,t),n(e,ee,t),u(_,e,t),n(e,te,t),n(e,g,t),n(e,le,t),n(e,y,t),n(e,ne,t),u(J,e,t),n(e,ie,t),n(e,$,t),n(e,ae,t),n(e,U,t),n(e,se,t),u(I,e,t),n(e,oe,t),n(e,b,t),n(e,me,t),u(v,e,t),n(e,pe,t),n(e,C,t),n(e,re,t),u(X,e,t),n(e,ue,t),n(e,j,t),n(e,de,t),n(e,L,t),n(e,fe,t),n(e,B,t),n(e,he,t),u(Z,e,t),n(e,ce,t),n(e,x,t),n(e,we,t),n(e,w,t),n(e,Me,t),u(R,e,t),n(e,Te,t),u(S,e,t),n(e,_e,t),n(e,W,t),n(e,ge,t),u(A,e,t),n(e,ye,t),u(G,e,t),n(e,Je,t),n(e,H,t),n(e,$e,t),u(V,e,t),n(e,Ue,t),n(e,k,t),n(e,Ie,t),n(e,F,t),n(e,be,t),u(E,e,t),n(e,ve,t),n(e,Y,t),n(e,Ce,t),n(e,Q,t),n(e,Xe,t),u(z,e,t),n(e,je,t),u(D,e,t),n(e,Le,t),n(e,P,t),Be=!0},p:et,i(e){Be||(d(T.$$.fragment,e),d(_.$$.fragment,e),d(J.$$.fragment,e),d(I.$$.fragment,e),d(v.$$.fragment,e),d(X.$$.fragment,e),d(Z.$$.fragment,e),d(R.$$.fragment,e),d(S.$$.fragment,e),d(A.$$.fragment,e),d(G.$$.fragment,e),d(V.$$.fragment,e),d(E.$$.fragment,e),d(z.$$.fragment,e),d(D.$$.fragment,e),Be=!0)},o(e){f(T.$$.fragment,e),f(_.$$.fragment,e),f(J.$$.fragment,e),f(I.$$.fragment,e),f(v.$$.fragment,e),f(X.$$.fragment,e),f(Z.$$.fragment,e),f(R.$$.fragment,e),f(S.$$.fragment,e),f(A.$$.fragment,e),f(G.$$.fragment,e),f(V.$$.fragment,e),f(E.$$.fragment,e),f(z.$$.fragment,e),f(D.$$.fragment,e),Be=!1},d(e){e&&(l(q),l(N),l(K),l(ee),l(te),l(g),l(le),l(y),l(ne),l(ie),l($),l(ae),l(U),l(se),l(oe),l(b),l(me),l(pe),l(C),l(re),l(ue),l(j),l(de),l(L),l(fe),l(B),l(he),l(ce),l(x),l(we),l(w),l(Me),l(Te),l(_e),l(W),l(ge),l(ye),l(Je),l(H),l($e),l(Ue),l(k),l(Ie),l(F),l(be),l(ve),l(Y),l(Ce),l(Q),l(Xe),l(je),l(Le),l(P)),l(c),h(T,e),h(_,e),h(J,e),h(I,e),h(v,e),h(X,e),h(Z,e),h(R,e),h(S,e),h(A,e),h(G,e),h(V,e),h(E,e),h(z,e),h(D,e)}}}const pt='{"title":"Multi-node Training","local":"multi-node-training","sections":[{"title":"Setting up several Gaudi instances","local":"setting-up-several-gaudi-instances","sections":[{"title":"On premises","local":"on-premises","sections":[],"depth":3},{"title":"Intel Tiber AI Cloud instances","local":"intel-tiber-ai-cloud-instances","sections":[],"depth":3}],"depth":2},{"title":"Launching a Multi-node Run","local":"launching-a-multi-node-run","sections":[],"depth":2},{"title":"Environment Variables","local":"environment-variables","sections":[],"depth":2},{"title":"Recommendations","local":"recommendations","sections":[],"depth":2},{"title":"Example","local":"example","sections":[],"depth":2}],"depth":1}';function rt(xe){return tt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ct extends lt{constructor(c){super(),nt(this,c,rt,mt,Ke,{})}}export{ct as component};
