import{s as oa,o as ra,n as be}from"../chunks/scheduler.d9d004e7.js";import{S as pa,i as ma,e as r,s,c as d,h as ua,a as p,d as a,b as l,f as sa,g as f,j as m,k as la,l as da,m as i,n as c,t as h,o as $,p as g}from"../chunks/index.4a2b774c.js";import{T as _e}from"../chunks/Tip.cf0287e4.js";import{C as fa,H as _,E as ca}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.c93fcd94.js";import{C as vt}from"../chunks/CodeBlock.8bd69f21.js";function ha(T){let n,u=`Not all models are yet supported with Eager mode and Eager mode with torch.compile (still in development).
Lazy mode is the default mode.`;return{c(){n=r("p"),n.textContent=u},l(o){n=p(o,"P",{"data-svelte-h":!0}),m(n)!=="svelte-1k0xkhh"&&(n.textContent=u)},m(o,w){i(o,n,w)},p:be,d(o){o&&a(n)}}}function $a(T){let n,u=`In lazy mode, the last batch is filled with extra samples by default so that it has the same dimensions as previous batches.
This enables to avoid extra graph compilations during training.
You can also discard the last batch with <code>dataloader_drop_last=True</code>.`;return{c(){n=r("p"),n.innerHTML=u},l(o){n=p(o,"P",{"data-svelte-h":!0}),m(n)!=="svelte-1l937np"&&(n.innerHTML=u)},m(o,w){i(o,n,w)},p:be,d(o){o&&a(n)}}}function ga(T){let n,u=`In lazy mode, the first two or three training iterations may be slower due to graph compilations.
To not take them into account in the computation of the throughput at the end of the training, you can add the following training argument: <code>throughput_warmup_steps=3</code>.`;return{c(){n=r("p"),n.innerHTML=u},l(o){n=p(o,"P",{"data-svelte-h":!0}),m(n)!=="svelte-1vfbovy"&&(n.innerHTML=u)},m(o,w){i(o,n,w)},p:be,d(o){o&&a(n)}}}function wa(T){let n,u='Please refer to the <a href="https://docs.habana.ai/en/latest/PyTorch/PyTorch_Mixed_Precision/Autocast.html" rel="nofollow">advanced autocast usage on Gaudi</a> for more informations regarding:',o,w,v="<li>default autocast operations</li> <li>default autocast operations override</li>";return{c(){n=r("p"),n.innerHTML=u,o=s(),w=r("ul"),w.innerHTML=v},l(y){n=p(y,"P",{"data-svelte-h":!0}),m(n)!=="svelte-io8h3z"&&(n.innerHTML=u),o=l(y),w=p(y,"UL",{"data-svelte-h":!0}),m(w)!=="svelte-12uyio6"&&(w.innerHTML=v)},m(y,b){i(y,n,b),i(y,o,b),i(y,w,b)},p:be,d(y){y&&(a(n),a(o),a(w))}}}function Ta(T){let n,u=`Use with caution: currently using HPU Graphs for training may not support all the possible cases.
However, the potential performance gain could be dramatic!`;return{c(){n=r("p"),n.textContent=u},l(o){n=p(o,"P",{"data-svelte-h":!0}),m(n)!=="svelte-1q7fkv4"&&(n.textContent=u)},m(o,w){i(o,n,w)},p:be,d(o){o&&a(n)}}}function ya(T){let n,u="The default value of <em>epsilon</em> is <code>1e-6</code> for the Intel Gaudi fused ADAM optimizer, while it is <code>1e-8</code> for <code>torch.optim.AdamW</code>.";return{c(){n=r("p"),n.innerHTML=u},l(o){n=p(o,"P",{"data-svelte-h":!0}),m(n)!=="svelte-m1ldcz"&&(n.innerHTML=u)},m(o,w){i(o,n,w)},p:be,d(o){o&&a(n)}}}function _a(T){let n,u="In distributed mode, memory stats are communicated only by the main process.";return{c(){n=r("p"),n.textContent=u},l(o){n=p(o,"P",{"data-svelte-h":!0}),m(n)!=="svelte-1tabw0"&&(n.textContent=u)},m(o,w){i(o,n,w)},p:be,d(o){o&&a(n)}}}function ba(T){let n,u,o,w,v,y,b,Me,U,Mt=`Gaudi offers several possibilities to make training faster.
They are all compatible with each other and can be coupled with <a href="https://huggingface.co/docs/optimum/habana/usage_guides/distributed" rel="nofollow">distributed training</a>.`,Le,J,Pe,I,Lt="The following execution modes are supported:",Ce,j,Pt=`<li><em>Lazy mode</em>, where operations are accumulated in a graph whose execution is triggered in a lazy manner.
This allows the graph compiler to optimize the device execution for these operations.</li> <li><em>Eager mode</em>, where one operation at a time is executed.</li> <li><em>Eager mode</em> with <em>torch.compile</em>, where a model (or part of a model) is enclosed into a graph.</li>`,xe,M,He,G,Ct="In lazy mode, the graph compiler generates optimized binary code that implements the given model topology on Gaudi. It performs operator fusion, data layout management, parallelization, pipelining and memory management, as well as graph-level optimizations.",ke,z,xt="To execute your training in lazy mode, you must provide the following training arguments:",Ue,B,Je,L,Ie,P,je,F,Ge,A,Ht=`Mixed-precision training enables to compute some operations using lighter data types to accelerate training.
Optimum for Intel Gaudi enables mixed precision training in a similar fashion as 🤗 Transformers:`,ze,Z,kt="<li>argument <code>--bf16</code> enables usage of PyTorch autocast</li> <li>argument <code>--half_precision_backend [hpu_amp, cpu_amp]</code> is used to specify a device on which mixed precision operations should be performed</li>",Be,C,Fe,Y,Ae,W,Ut=`The flexibility of PyTorch comes at a price - usually the same pythonic logic is processed every training step over and over.
This may lead to a situation where it takes longer for CPU to schedule the work on Gaudi than it is effectively computed by it.
To cope with such host-bound workloads, you may want to try enabling the <em>HPU Graphs</em> feature, which records the computational graph once, then only triggers it for execution much faster multiple times.`,Ze,D,Jt=`To do so, specify <code>--use_hpu_graphs_for_training True</code>.
This option will wrap the model in <a href="https://docs.habana.ai/en/latest/PyTorch/Model_Optimization_PyTorch/HPU_Graphs_Training.html#training-loop-with-modulecacher" rel="nofollow"><code>habana_frameworks.torch.hpu.ModuleCacher</code></a>, which automatically records <em>HPU Graphs</em> on the model’s usage.`,Ye,V,It=`For multi-worker distributed training, you also need to specify <code>--distribution_strategy fast_ddp</code>.
This option replaces the usage of <code>torch.nn.parallel.DistributedDataParallel</code> with much simpler and usually faster <code>optimum.habana.distributed.all_reduce_gradients</code>.`,We,x,De,X,Ve,E,jt=`For distributed training on several devices, you can also specify <code>--distribution_strategy fast_ddp</code>.
This option replaces the usage of <code>torch.nn.parallel.DistributedDataParallel</code> with much simpler and usually faster <code>optimum.habana.distributed.all_reduce_gradients</code>.`,Xe,O,Ee,S,Gt=`There are two stages when running models on Intel Gaudi HPU: python code interpretation on CPU and HPU recipe computation.
The HPU computation stage can be triggered manually or when a copy to the CPU is requested, and generally HPU computation is triggered after <code>loss.backward()</code> to make the CPU code interpretation and HPU recipe computation overlap as shown in the following illustration:`,Oe,N,Se,q,zt=`However, when CPU code interpretation takes longer than HPU computation, it becomes the bottleneck and HPU computation can not be triggered until CPU code interpretation is done.
So one potential optimization for such cases is to trigger the HPU <em>forward</em> computation right after the CPU <em>forward</em> interpretation and before the CPU <em>backward</em> interpretation.
You can see an example below where the CPU <em>backward</em> interpretation overlaps with the HPU <em>forward</em> computation:`,Ne,Q,qe,R,Bt="To enable this optimization, you can set the following training argument <code>--pipelining_fwd_bwd True</code>.",Qe,K,Ft=`<strong>We recommend using it on Gaudi2</strong> as the host will often be the bottleneck.
You should be able to see a speedup on first-generation Gaudi too, but it will be less significant than on Gaudi2 because your run is more likely to be HPU-bound.`,Re,ee,At="Furthermore, <em>when training models that require large device memory</em>, we suggest disabling this optimization because <em>it will increase the HPU memory usage</em>.",Ke,te,et,ae,Zt=`If the workload of the data loader is heavy, you can increase the number of workers to make your run faster.
You can enable this with the training argument <a href="https://huggingface.co/docs/transformers/main_classes/trainer#transformers.TrainingArguments.dataloader_num_workers" rel="nofollow"><code>--dataloader_num_workers N</code></a> with <code>N</code> being the number of workers to use.`,tt,ie,Yt=`<strong>We recommend using it with datasets containing images.</strong>
Besides, using <code>--dataloader_num_workers 1</code> should help in most cases as it enables data loading in a thread different from the main one.`,at,ne,it,se,Wt=`This optimization is well-suited for models with a high cost of copying data from the host to the device (e.g. vision models like ViT or Swin).
You can enable it with the training argument <code>--non_blocking_data_copy True</code>.`,nt,le,Dt=`<strong>We recommend using it on Gaudi2</strong> where the host can continue to execute other tasks (e.g. graph building) to get a better pipelining between the host and the device.
On first-generation Gaudi, the device executing time is longer so one should not expect to get any speedup.`,st,oe,lt,re,Vt=`Intel Gaudi provides a few custom operators that achieve better performance than their PyTorch counterparts on Gaudi.
You can also define your own custom operator for Gaudi as described <a href="https://docs.habana.ai/en/latest/PyTorch/PyTorch_CustomOp_API/page_index.html" rel="nofollow">here</a>.`,ot,pe,rt,me,Xt=`Intel Gaudi offers a <a href="https://docs.habana.ai/en/latest/PyTorch/Model_Optimization_PyTorch/Custom_Ops_PyTorch.html#custom-optimizers" rel="nofollow">custom fused ADAM implementation</a>.
It can be used by specifying <code>&quot;use_fused_adam&quot;: true</code> in the Gaudi configuration file.`,pt,H,mt,ue,ut,de,Et=`Intel Gaudi provides a <a href="https://docs.habana.ai/en/latest/PyTorch/Model_Optimization_PyTorch/Custom_Ops_PyTorch.html#other-custom-ops" rel="nofollow">custom gradient norm clipping implementation</a>.
It can be used by specifying <code>&quot;use_fused_clip_norm&quot;: true</code> in the Gaudi configuration file.`,dt,fe,ft,ce,Ot=`Flash attention algorithm with additional Intel® Gaudi® AI Accelerator optimizetions is supported for both Lazy and Eager mode.
See <a href="https://docs.habana.ai/en/latest/PyTorch/Model_Optimization_PyTorch/Optimization_in_PyTorch_Models.html#using-fused-scaled-dot-product-attention-fusedsdpa" rel="nofollow">Using Fused Scaled Dot Product Attention (FusedSDPA)</a>.`,ct,he,ht,$e,St="Live memory statistics are displayed every <code>logging_steps</code> (default is 500) steps:",$t,ge,Nt="<li><code>memory_allocated (GB)</code> refers to the <em>current</em> memory consumption in GB,</li> <li><code>max_memory_allocated (GB)</code> refers to the <em>maximum</em> memory consumption reached during the run in GB,</li> <li><code>total_memory_available (GB)</code> refers to the <em>total</em> memory available on the device in GB.</li>",gt,we,qt="These metrics can help you to adjust the batch size of your runs.",wt,k,Tt,Te,Qt='You can take a look at <a href="https://docs.habana.ai/en/latest/PyTorch/PyTorch_User_Guide/Python_Packages.html#memory-stats-apis" rel="nofollow">Intel Gaudi AI Accelerator’s official documentation</a> for more information about the memory stats API.',yt,ye,_t,ve,bt;return v=new fa({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),b=new _({props:{title:"Accelerating Training",local:"accelerating-training",headingTag:"h1"}}),J=new _({props:{title:"Execution Modes",local:"execution-modes",headingTag:"h2"}}),M=new _e({props:{warning:!0,$$slots:{default:[ha]},$$scope:{ctx:T}}}),B=new vt({props:{code:"YXJncyUyMCUzRCUyMEdhdWRpVHJhaW5pbmdBcmd1bWVudHMoJTBBJTIwJTIwJTIwJTIwJTIzJTIwc2FtZSUyMGFyZ3VtZW50cyUyMGFzJTIwaW4lMjBUcmFuc2Zvcm1lcnMlMkMlMEElMjAlMjAlMjAlMjB1c2VfaGFiYW5hJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMHVzZV9sYXp5X21vZGUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwZ2F1ZGlfY29uZmlnX25hbWUlM0RwYXRoX3RvX215X2dhdWRpX2NvbmZpZyUwQSk=",highlighted:`args = GaudiTrainingArguments(
    <span class="hljs-comment"># same arguments as in Transformers,</span>
    use_habana=<span class="hljs-literal">True</span>,
    use_lazy_mode=<span class="hljs-literal">True</span>,
    gaudi_config_name=path_to_my_gaudi_config
)`,wrap:!1}}),L=new _e({props:{$$slots:{default:[$a]},$$scope:{ctx:T}}}),P=new _e({props:{$$slots:{default:[ga]},$$scope:{ctx:T}}}),F=new _({props:{title:"Mixed-Precision Training",local:"mixed-precision-training",headingTag:"h2"}}),C=new _e({props:{warning:!0,$$slots:{default:[wa]},$$scope:{ctx:T}}}),Y=new _({props:{title:"HPU Graphs",local:"hpu-graphs",headingTag:"h2"}}),x=new _e({props:{warning:!0,$$slots:{default:[Ta]},$$scope:{ctx:T}}}),X=new _({props:{title:"Fast DDP",local:"fast-ddp",headingTag:"h2"}}),O=new _({props:{title:"Pipelining Forward and Backward Passes",local:"pipelining-forward-and-backward-passes",headingTag:"h2"}}),N=new vt({props:{code:"Q1BVJTNBLi4uZm9yd2FyZCUyMCUyQiUyMGJhY2t3YXJkJTIwJTIwJTIwLi4ub3B0aW1pemVyJTIwJTIwLi4uZm9yd2FyZCUyMCUyQiUyMGJhY2t3YXJkJTIwJTIwJTIwLi4ub3B0aW1pemVyJTIwJTIwLi4uJTBBSFBVJTNBLi4uLi4uLi4uLi4uLi4uLi4uLi4uLi4uZm9yd2FyZCUyMCUyQiUyMGJhY2t3YXJkLi4ub3B0aW1pemVyLi4uLi4uZm9yd2FyZCUyMCUyQiUyMGJhY2t3YXJkLi4ub3B0aW1pemVy",highlighted:`CPU:<span class="hljs-params">...</span>forward + backward   <span class="hljs-params">...</span>optimizer  <span class="hljs-params">...</span>forward + backward   <span class="hljs-params">...</span>optimizer  <span class="hljs-params">...</span>
HPU:<span class="hljs-params">...</span><span class="hljs-params">...</span><span class="hljs-params">...</span><span class="hljs-params">...</span><span class="hljs-params">...</span><span class="hljs-params">...</span><span class="hljs-params">...</span><span class="hljs-params">...</span>forward + backward<span class="hljs-params">...</span>optimizer<span class="hljs-params">...</span><span class="hljs-params">...</span>forward + backward<span class="hljs-params">...</span>optimizer`,wrap:!1}}),Q=new vt({props:{code:"Q1BVJTNBLi4uZm9yd2FyZCUyMCUyMCUyMC4uLmJhY2t3YXJkJTIwJTIwJTIwLi4ub3B0aW1pemVyJTIwJTIwLi4uZm9yd2FyZCUyMCUyMCUyMC4uLmJhY2t3YXJkJTIwJTIwJTIwLi4ub3B0aW1pemVyJTIwJTIwJTIwLi4uJTBBSFBVJTNBLi4uLi4uLi4uLi4uLmZvcndhcmQuLi4uLi4uYmFja3dhcmQuLi4uLi5vcHRpbWl6ZXIuLi4uLi5mb3J3YXJkLi4uLi5iYWNrd2FyZC4uLi4uLi5vcHRpbWl6ZXI=",highlighted:`CPU:<span class="hljs-params">...</span>forward   <span class="hljs-params">...</span>backward   <span class="hljs-params">...</span>optimizer  <span class="hljs-params">...</span>forward   <span class="hljs-params">...</span>backward   <span class="hljs-params">...</span>optimizer   <span class="hljs-params">...</span>
HPU:<span class="hljs-params">...</span><span class="hljs-params">...</span><span class="hljs-params">...</span><span class="hljs-params">...</span>.forward<span class="hljs-params">...</span><span class="hljs-params">...</span>.backward<span class="hljs-params">...</span><span class="hljs-params">...</span>optimizer<span class="hljs-params">...</span><span class="hljs-params">...</span>forward<span class="hljs-params">...</span>..backward<span class="hljs-params">...</span><span class="hljs-params">...</span>.optimizer`,wrap:!1}}),te=new _({props:{title:"Use More Workers for Data Loading",local:"use-more-workers-for-data-loading",headingTag:"h2"}}),ne=new _({props:{title:"Non-Blocking Data Copy",local:"non-blocking-data-copy",headingTag:"h2"}}),oe=new _({props:{title:"Custom Operators",local:"custom-operators",headingTag:"h2"}}),pe=new _({props:{title:"Fused ADAM",local:"fused-adam",headingTag:"h3"}}),H=new _e({props:{warning:!0,$$slots:{default:[ya]},$$scope:{ctx:T}}}),ue=new _({props:{title:"Fused Gradient Norm Clipping",local:"fused-gradient-norm-clipping",headingTag:"h3"}}),fe=new _({props:{title:"Gaudi Optimized Flash Attention",local:"gaudi-optimized-flash-attention",headingTag:"h3"}}),he=new _({props:{title:"Tracking Memory Usage",local:"tracking-memory-usage",headingTag:"h2"}}),k=new _e({props:{warning:!0,$$slots:{default:[_a]},$$scope:{ctx:T}}}),ye=new ca({props:{source:"https://github.com/huggingface/optimum-habana/blob/main/docs/source/usage_guides/accelerate_training.mdx"}}),{c(){n=r("meta"),u=s(),o=r("p"),w=s(),d(v.$$.fragment),y=s(),d(b.$$.fragment),Me=s(),U=r("p"),U.innerHTML=Mt,Le=s(),d(J.$$.fragment),Pe=s(),I=r("p"),I.textContent=Lt,Ce=s(),j=r("ul"),j.innerHTML=Pt,xe=s(),d(M.$$.fragment),He=s(),G=r("p"),G.textContent=Ct,ke=s(),z=r("p"),z.textContent=xt,Ue=s(),d(B.$$.fragment),Je=s(),d(L.$$.fragment),Ie=s(),d(P.$$.fragment),je=s(),d(F.$$.fragment),Ge=s(),A=r("p"),A.textContent=Ht,ze=s(),Z=r("ul"),Z.innerHTML=kt,Be=s(),d(C.$$.fragment),Fe=s(),d(Y.$$.fragment),Ae=s(),W=r("p"),W.innerHTML=Ut,Ze=s(),D=r("p"),D.innerHTML=Jt,Ye=s(),V=r("p"),V.innerHTML=It,We=s(),d(x.$$.fragment),De=s(),d(X.$$.fragment),Ve=s(),E=r("p"),E.innerHTML=jt,Xe=s(),d(O.$$.fragment),Ee=s(),S=r("p"),S.innerHTML=Gt,Oe=s(),d(N.$$.fragment),Se=s(),q=r("p"),q.innerHTML=zt,Ne=s(),d(Q.$$.fragment),qe=s(),R=r("p"),R.innerHTML=Bt,Qe=s(),K=r("p"),K.innerHTML=Ft,Re=s(),ee=r("p"),ee.innerHTML=At,Ke=s(),d(te.$$.fragment),et=s(),ae=r("p"),ae.innerHTML=Zt,tt=s(),ie=r("p"),ie.innerHTML=Yt,at=s(),d(ne.$$.fragment),it=s(),se=r("p"),se.innerHTML=Wt,nt=s(),le=r("p"),le.innerHTML=Dt,st=s(),d(oe.$$.fragment),lt=s(),re=r("p"),re.innerHTML=Vt,ot=s(),d(pe.$$.fragment),rt=s(),me=r("p"),me.innerHTML=Xt,pt=s(),d(H.$$.fragment),mt=s(),d(ue.$$.fragment),ut=s(),de=r("p"),de.innerHTML=Et,dt=s(),d(fe.$$.fragment),ft=s(),ce=r("p"),ce.innerHTML=Ot,ct=s(),d(he.$$.fragment),ht=s(),$e=r("p"),$e.innerHTML=St,$t=s(),ge=r("ul"),ge.innerHTML=Nt,gt=s(),we=r("p"),we.textContent=qt,wt=s(),d(k.$$.fragment),Tt=s(),Te=r("p"),Te.innerHTML=Qt,yt=s(),d(ye.$$.fragment),_t=s(),ve=r("p"),this.h()},l(e){const t=ua("svelte-u9bgzb",document.head);n=p(t,"META",{name:!0,content:!0}),t.forEach(a),u=l(e),o=p(e,"P",{}),sa(o).forEach(a),w=l(e),f(v.$$.fragment,e),y=l(e),f(b.$$.fragment,e),Me=l(e),U=p(e,"P",{"data-svelte-h":!0}),m(U)!=="svelte-1pibpy9"&&(U.innerHTML=Mt),Le=l(e),f(J.$$.fragment,e),Pe=l(e),I=p(e,"P",{"data-svelte-h":!0}),m(I)!=="svelte-1n8dewo"&&(I.textContent=Lt),Ce=l(e),j=p(e,"UL",{"data-svelte-h":!0}),m(j)!=="svelte-j2rpi"&&(j.innerHTML=Pt),xe=l(e),f(M.$$.fragment,e),He=l(e),G=p(e,"P",{"data-svelte-h":!0}),m(G)!=="svelte-q1o82l"&&(G.textContent=Ct),ke=l(e),z=p(e,"P",{"data-svelte-h":!0}),m(z)!=="svelte-916mlc"&&(z.textContent=xt),Ue=l(e),f(B.$$.fragment,e),Je=l(e),f(L.$$.fragment,e),Ie=l(e),f(P.$$.fragment,e),je=l(e),f(F.$$.fragment,e),Ge=l(e),A=p(e,"P",{"data-svelte-h":!0}),m(A)!=="svelte-1h5k1db"&&(A.textContent=Ht),ze=l(e),Z=p(e,"UL",{"data-svelte-h":!0}),m(Z)!=="svelte-1076mfd"&&(Z.innerHTML=kt),Be=l(e),f(C.$$.fragment,e),Fe=l(e),f(Y.$$.fragment,e),Ae=l(e),W=p(e,"P",{"data-svelte-h":!0}),m(W)!=="svelte-1cw5c95"&&(W.innerHTML=Ut),Ze=l(e),D=p(e,"P",{"data-svelte-h":!0}),m(D)!=="svelte-lrw3oq"&&(D.innerHTML=Jt),Ye=l(e),V=p(e,"P",{"data-svelte-h":!0}),m(V)!=="svelte-8cls23"&&(V.innerHTML=It),We=l(e),f(x.$$.fragment,e),De=l(e),f(X.$$.fragment,e),Ve=l(e),E=p(e,"P",{"data-svelte-h":!0}),m(E)!=="svelte-11ovj5i"&&(E.innerHTML=jt),Xe=l(e),f(O.$$.fragment,e),Ee=l(e),S=p(e,"P",{"data-svelte-h":!0}),m(S)!=="svelte-x0iehm"&&(S.innerHTML=Gt),Oe=l(e),f(N.$$.fragment,e),Se=l(e),q=p(e,"P",{"data-svelte-h":!0}),m(q)!=="svelte-1k82u8a"&&(q.innerHTML=zt),Ne=l(e),f(Q.$$.fragment,e),qe=l(e),R=p(e,"P",{"data-svelte-h":!0}),m(R)!=="svelte-17kd987"&&(R.innerHTML=Bt),Qe=l(e),K=p(e,"P",{"data-svelte-h":!0}),m(K)!=="svelte-1mb0f47"&&(K.innerHTML=Ft),Re=l(e),ee=p(e,"P",{"data-svelte-h":!0}),m(ee)!=="svelte-1yzl4nc"&&(ee.innerHTML=At),Ke=l(e),f(te.$$.fragment,e),et=l(e),ae=p(e,"P",{"data-svelte-h":!0}),m(ae)!=="svelte-7vvao4"&&(ae.innerHTML=Zt),tt=l(e),ie=p(e,"P",{"data-svelte-h":!0}),m(ie)!=="svelte-1xpoavn"&&(ie.innerHTML=Yt),at=l(e),f(ne.$$.fragment,e),it=l(e),se=p(e,"P",{"data-svelte-h":!0}),m(se)!=="svelte-uej40d"&&(se.innerHTML=Wt),nt=l(e),le=p(e,"P",{"data-svelte-h":!0}),m(le)!=="svelte-e7qbak"&&(le.innerHTML=Dt),st=l(e),f(oe.$$.fragment,e),lt=l(e),re=p(e,"P",{"data-svelte-h":!0}),m(re)!=="svelte-1yoy3p9"&&(re.innerHTML=Vt),ot=l(e),f(pe.$$.fragment,e),rt=l(e),me=p(e,"P",{"data-svelte-h":!0}),m(me)!=="svelte-1sqdus5"&&(me.innerHTML=Xt),pt=l(e),f(H.$$.fragment,e),mt=l(e),f(ue.$$.fragment,e),ut=l(e),de=p(e,"P",{"data-svelte-h":!0}),m(de)!=="svelte-102oxu7"&&(de.innerHTML=Et),dt=l(e),f(fe.$$.fragment,e),ft=l(e),ce=p(e,"P",{"data-svelte-h":!0}),m(ce)!=="svelte-14at5sk"&&(ce.innerHTML=Ot),ct=l(e),f(he.$$.fragment,e),ht=l(e),$e=p(e,"P",{"data-svelte-h":!0}),m($e)!=="svelte-1l14050"&&($e.innerHTML=St),$t=l(e),ge=p(e,"UL",{"data-svelte-h":!0}),m(ge)!=="svelte-fbwrvf"&&(ge.innerHTML=Nt),gt=l(e),we=p(e,"P",{"data-svelte-h":!0}),m(we)!=="svelte-sxgaes"&&(we.textContent=qt),wt=l(e),f(k.$$.fragment,e),Tt=l(e),Te=p(e,"P",{"data-svelte-h":!0}),m(Te)!=="svelte-x608ek"&&(Te.innerHTML=Qt),yt=l(e),f(ye.$$.fragment,e),_t=l(e),ve=p(e,"P",{}),sa(ve).forEach(a),this.h()},h(){la(n,"name","hf:doc:metadata"),la(n,"content",va)},m(e,t){da(document.head,n),i(e,u,t),i(e,o,t),i(e,w,t),c(v,e,t),i(e,y,t),c(b,e,t),i(e,Me,t),i(e,U,t),i(e,Le,t),c(J,e,t),i(e,Pe,t),i(e,I,t),i(e,Ce,t),i(e,j,t),i(e,xe,t),c(M,e,t),i(e,He,t),i(e,G,t),i(e,ke,t),i(e,z,t),i(e,Ue,t),c(B,e,t),i(e,Je,t),c(L,e,t),i(e,Ie,t),c(P,e,t),i(e,je,t),c(F,e,t),i(e,Ge,t),i(e,A,t),i(e,ze,t),i(e,Z,t),i(e,Be,t),c(C,e,t),i(e,Fe,t),c(Y,e,t),i(e,Ae,t),i(e,W,t),i(e,Ze,t),i(e,D,t),i(e,Ye,t),i(e,V,t),i(e,We,t),c(x,e,t),i(e,De,t),c(X,e,t),i(e,Ve,t),i(e,E,t),i(e,Xe,t),c(O,e,t),i(e,Ee,t),i(e,S,t),i(e,Oe,t),c(N,e,t),i(e,Se,t),i(e,q,t),i(e,Ne,t),c(Q,e,t),i(e,qe,t),i(e,R,t),i(e,Qe,t),i(e,K,t),i(e,Re,t),i(e,ee,t),i(e,Ke,t),c(te,e,t),i(e,et,t),i(e,ae,t),i(e,tt,t),i(e,ie,t),i(e,at,t),c(ne,e,t),i(e,it,t),i(e,se,t),i(e,nt,t),i(e,le,t),i(e,st,t),c(oe,e,t),i(e,lt,t),i(e,re,t),i(e,ot,t),c(pe,e,t),i(e,rt,t),i(e,me,t),i(e,pt,t),c(H,e,t),i(e,mt,t),c(ue,e,t),i(e,ut,t),i(e,de,t),i(e,dt,t),c(fe,e,t),i(e,ft,t),i(e,ce,t),i(e,ct,t),c(he,e,t),i(e,ht,t),i(e,$e,t),i(e,$t,t),i(e,ge,t),i(e,gt,t),i(e,we,t),i(e,wt,t),c(k,e,t),i(e,Tt,t),i(e,Te,t),i(e,yt,t),c(ye,e,t),i(e,_t,t),i(e,ve,t),bt=!0},p(e,[t]){const Rt={};t&2&&(Rt.$$scope={dirty:t,ctx:e}),M.$set(Rt);const Kt={};t&2&&(Kt.$$scope={dirty:t,ctx:e}),L.$set(Kt);const ea={};t&2&&(ea.$$scope={dirty:t,ctx:e}),P.$set(ea);const ta={};t&2&&(ta.$$scope={dirty:t,ctx:e}),C.$set(ta);const aa={};t&2&&(aa.$$scope={dirty:t,ctx:e}),x.$set(aa);const ia={};t&2&&(ia.$$scope={dirty:t,ctx:e}),H.$set(ia);const na={};t&2&&(na.$$scope={dirty:t,ctx:e}),k.$set(na)},i(e){bt||(h(v.$$.fragment,e),h(b.$$.fragment,e),h(J.$$.fragment,e),h(M.$$.fragment,e),h(B.$$.fragment,e),h(L.$$.fragment,e),h(P.$$.fragment,e),h(F.$$.fragment,e),h(C.$$.fragment,e),h(Y.$$.fragment,e),h(x.$$.fragment,e),h(X.$$.fragment,e),h(O.$$.fragment,e),h(N.$$.fragment,e),h(Q.$$.fragment,e),h(te.$$.fragment,e),h(ne.$$.fragment,e),h(oe.$$.fragment,e),h(pe.$$.fragment,e),h(H.$$.fragment,e),h(ue.$$.fragment,e),h(fe.$$.fragment,e),h(he.$$.fragment,e),h(k.$$.fragment,e),h(ye.$$.fragment,e),bt=!0)},o(e){$(v.$$.fragment,e),$(b.$$.fragment,e),$(J.$$.fragment,e),$(M.$$.fragment,e),$(B.$$.fragment,e),$(L.$$.fragment,e),$(P.$$.fragment,e),$(F.$$.fragment,e),$(C.$$.fragment,e),$(Y.$$.fragment,e),$(x.$$.fragment,e),$(X.$$.fragment,e),$(O.$$.fragment,e),$(N.$$.fragment,e),$(Q.$$.fragment,e),$(te.$$.fragment,e),$(ne.$$.fragment,e),$(oe.$$.fragment,e),$(pe.$$.fragment,e),$(H.$$.fragment,e),$(ue.$$.fragment,e),$(fe.$$.fragment,e),$(he.$$.fragment,e),$(k.$$.fragment,e),$(ye.$$.fragment,e),bt=!1},d(e){e&&(a(u),a(o),a(w),a(y),a(Me),a(U),a(Le),a(Pe),a(I),a(Ce),a(j),a(xe),a(He),a(G),a(ke),a(z),a(Ue),a(Je),a(Ie),a(je),a(Ge),a(A),a(ze),a(Z),a(Be),a(Fe),a(Ae),a(W),a(Ze),a(D),a(Ye),a(V),a(We),a(De),a(Ve),a(E),a(Xe),a(Ee),a(S),a(Oe),a(Se),a(q),a(Ne),a(qe),a(R),a(Qe),a(K),a(Re),a(ee),a(Ke),a(et),a(ae),a(tt),a(ie),a(at),a(it),a(se),a(nt),a(le),a(st),a(lt),a(re),a(ot),a(rt),a(me),a(pt),a(mt),a(ut),a(de),a(dt),a(ft),a(ce),a(ct),a(ht),a($e),a($t),a(ge),a(gt),a(we),a(wt),a(Tt),a(Te),a(yt),a(_t),a(ve)),a(n),g(v,e),g(b,e),g(J,e),g(M,e),g(B,e),g(L,e),g(P,e),g(F,e),g(C,e),g(Y,e),g(x,e),g(X,e),g(O,e),g(N,e),g(Q,e),g(te,e),g(ne,e),g(oe,e),g(pe,e),g(H,e),g(ue,e),g(fe,e),g(he,e),g(k,e),g(ye,e)}}}const va='{"title":"Accelerating Training","local":"accelerating-training","sections":[{"title":"Execution Modes","local":"execution-modes","sections":[],"depth":2},{"title":"Mixed-Precision Training","local":"mixed-precision-training","sections":[],"depth":2},{"title":"HPU Graphs","local":"hpu-graphs","sections":[],"depth":2},{"title":"Fast DDP","local":"fast-ddp","sections":[],"depth":2},{"title":"Pipelining Forward and Backward Passes","local":"pipelining-forward-and-backward-passes","sections":[],"depth":2},{"title":"Use More Workers for Data Loading","local":"use-more-workers-for-data-loading","sections":[],"depth":2},{"title":"Non-Blocking Data Copy","local":"non-blocking-data-copy","sections":[],"depth":2},{"title":"Custom Operators","local":"custom-operators","sections":[{"title":"Fused ADAM","local":"fused-adam","sections":[],"depth":3},{"title":"Fused Gradient Norm Clipping","local":"fused-gradient-norm-clipping","sections":[],"depth":3},{"title":"Gaudi Optimized Flash Attention","local":"gaudi-optimized-flash-attention","sections":[],"depth":3}],"depth":2},{"title":"Tracking Memory Usage","local":"tracking-memory-usage","sections":[],"depth":2}],"depth":1}';function Ma(T){return ra(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ka extends pa{constructor(n){super(),ma(this,n,Ma,ba,oa,{})}}export{ka as component};
