import{s as Ve,o as Ee,n as $e}from"../chunks/scheduler.d9d004e7.js";import{S as xe,i as Se,e as o,s as i,c as d,h as Fe,a as p,d as a,b as s,f as He,g as f,j as m,k as Re,l as Pe,m as l,n as c,t as h,o as M,p as y}from"../chunks/index.4a2b774c.js";import{T as ye}from"../chunks/Tip.cf0287e4.js";import{C as Be,H as A,E as Ye}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.02c268bd.js";import{C as Ue}from"../chunks/CodeBlock.67ad8ed0.js";function ke($){let n,u=`Not all models are yet supported with Eager mode and Eager mode with torch.compile (still in development).
Lazy mode is the default mode.`;return{c(){n=o("p"),n.textContent=u},l(r){n=p(r,"P",{"data-svelte-h":!0}),m(n)!=="svelte-1k0xkhh"&&(n.textContent=u)},m(r,U){l(r,n,U)},p:$e,d(r){r&&a(n)}}}function Ae($){let n,u=`In lazy mode, the last batch may trigger an extra compilation because it could be smaller than previous batches.
To avoid this, you can discard the last batch with <code>dataloader_drop_last=True</code>.`;return{c(){n=o("p"),n.innerHTML=u},l(r){n=p(r,"P",{"data-svelte-h":!0}),m(n)!=="svelte-12atxwz"&&(n.innerHTML=u)},m(r,U){l(r,n,U)},p:$e,d(r){r&&a(n)}}}function Le($){let n,u="With HPU Graphs and in lazy mode, the <em>first couple of iterations</em> may be slower due to graph compilations.";return{c(){n=o("p"),n.innerHTML=u},l(r){n=p(r,"P",{"data-svelte-h":!0}),m(n)!=="svelte-1khopef"&&(n.innerHTML=u)},m(r,U){l(r,n,U)},p:$e,d(r){r&&a(n)}}}function Xe($){let n,u,r,U,_,L,b,X,J,Te="Intel Gaudi offers several possibilities to make inference faster.",Q,C,N,G,we="The following execution modes are supported:",D,Z,ge=`<li><em>Lazy mode</em>, where operations are accumulated in a graph whose execution is triggered in a lazy manner.
This allows the graph compiler to optimize the device execution for these operations.</li> <li><em>Eager mode</em>, where one operation at a time is executed.</li> <li><em>Eager mode</em> with <em>torch.compile</em>, where a model (or part of a model) is enclosed into a graph.</li>`,q,T,O,j,_e=`In lazy mode, the graph compiler generates optimized binary code that implements the given model topology on Gaudi.
It performs operator fusion, data layout management, parallelization, pipelining and memory management, as well as graph-level optimizations.`,K,I,be="To execute inference in lazy mode, you must provide the following arguments:",ee,v,te,w,ae,z,le,W,Je=`Gaudi provides a way to run fast inference with HPU Graphs.
It consists in capturing a series of operations (i.e. graphs) in an HPU stream and then replaying them in an optimized way (more information <a href="https://docs.habana.ai/en/latest/PyTorch/Inference_on_Gaudi/Inference_using_HPU_Graphs/Inference_using_HPU_Graphs.html" rel="nofollow">here</a>).
Thus, you can apply this to the <code>forward</code> method of your model to run it efficiently at inference.`,ne,H,Ce="HPU Graphs are integrated into the <code>GaudiTrainer</code> and the <code>GaudiStableDiffusionPipeline</code> so that one can use them very easily:",ie,R,Ge="<li><code>GaudiTrainer</code> needs the training argument <code>use_hpu_graphs_for_inference</code> to be set to <code>True</code> as follows:</li>",se,V,re,E,Ze="<li><code>GaudiStableDiffusionPipeline</code> needs its argument <code>use_hpu_graphs</code> to be set to <code>True</code> such as:</li>",oe,x,pe,g,me,S,ue,F,je=`Intel Gaudi provides a few custom operators that achieve better performance than their PyTorch counterparts.
You can also define your own custom operator for Gaudi as described <a href="https://docs.habana.ai/en/latest/PyTorch/PyTorch_CustomOp_API/page_index.html" rel="nofollow">here</a>.`,de,P,fe,B,Ie=`Flash attention algorithm with additional Intel Gaudi AI Accelerator optimizations is supported for both Lazy and Eager mode.
See <a href="https://docs.habana.ai/en/latest/PyTorch/Model_Optimization_PyTorch/Optimization_in_PyTorch_Models.html#using-fused-scaled-dot-product-attention-fusedsdpa" rel="nofollow">Using Fused Scaled Dot Product Attention (FusedSDPA)</a>.`,ce,Y,he,k,Me;return _=new Be({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),b=new A({props:{title:"Accelerating Inference",local:"accelerating-inference",headingTag:"h1"}}),C=new A({props:{title:"Lazy Mode",local:"lazy-mode",headingTag:"h2"}}),T=new ye({props:{warning:!0,$$slots:{default:[ke]},$$scope:{ctx:$}}}),v=new Ue({props:{code:"YXJncyUyMCUzRCUyMEdhdWRpVHJhaW5pbmdBcmd1bWVudHMoJTBBJTIwJTIwJTIwJTIwJTIzJTIwc2FtZSUyMGFyZ3VtZW50cyUyMGFzJTIwaW4lMjBUcmFuc2Zvcm1lcnMlMkMlMEElMjAlMjAlMjAlMjB1c2VfaGFiYW5hJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMHVzZV9sYXp5X21vZGUlM0RUcnVlJTJDJTBBKQ==",highlighted:`args = GaudiTrainingArguments(
    <span class="hljs-comment"># same arguments as in Transformers,</span>
    use_habana=<span class="hljs-literal">True</span>,
    use_lazy_mode=<span class="hljs-literal">True</span>,
)`,wrap:!1}}),w=new ye({props:{$$slots:{default:[Ae]},$$scope:{ctx:$}}}),z=new A({props:{title:"HPU Graphs",local:"hpu-graphs",headingTag:"h2"}}),V=new Ue({props:{code:"ZnJvbSUyMG9wdGltdW0uaGFiYW5hJTIwaW1wb3J0JTIwR2F1ZGlUcmFpbmVyJTJDJTIwR2F1ZGlUcmFpbmluZ0FyZ3VtZW50cyUwQSUwQSUyMyUyMGRlZmluZSUyMHRoZSUyMHRyYWluaW5nJTIwYXJndW1lbnRzJTBBdHJhaW5pbmdfYXJncyUyMCUzRCUyMEdhdWRpVHJhaW5pbmdBcmd1bWVudHMoJTBBJTIwJTIwJTIwJTIwdXNlX2hhYmFuYSUzRFRydWUlMkMlMEElMjAlMjAlMjAlMjB1c2VfbGF6eV9tb2RlJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMHVzZV9ocHVfZ3JhcGhzX2Zvcl9pbmZlcmVuY2UlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwZ2F1ZGlfY29uZmlnX25hbWUlM0RnYXVkaV9jb25maWdfbmFtZSUyQyUwQSUyMCUyMCUyMCUyMC4uLiUwQSklMEElMEElMjMlMjBJbml0aWFsaXplJTIwb3VyJTIwVHJhaW5lciUwQXRyYWluZXIlMjAlM0QlMjBHYXVkaVRyYWluZXIoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0Rtb2RlbCUyQyUwQSUyMCUyMCUyMCUyMGFyZ3MlM0R0cmFpbmluZ19hcmdzJTJDJTBBJTIwJTIwJTIwJTIwdHJhaW5fZGF0YXNldCUzRHRyYWluX2RhdGFzZXQlMEElMjAlMjAlMjAlMjAuLi4lMjAlMjMlMjBvdGhlciUyMGFyZ3VtZW50cyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> optimum.habana <span class="hljs-keyword">import</span> GaudiTrainer, GaudiTrainingArguments

<span class="hljs-comment"># define the training arguments</span>
training_args = GaudiTrainingArguments(
    use_habana=<span class="hljs-literal">True</span>,
    use_lazy_mode=<span class="hljs-literal">True</span>,
    use_hpu_graphs_for_inference=<span class="hljs-literal">True</span>,
    gaudi_config_name=gaudi_config_name,
    ...
)

<span class="hljs-comment"># Initialize our Trainer</span>
trainer = GaudiTrainer(
    model=model,
    args=training_args,
    train_dataset=train_dataset
    ... <span class="hljs-comment"># other arguments</span>
)`,wrap:!1}}),x=new Ue({props:{code:"ZnJvbSUyMG9wdGltdW0uaGFiYW5hLmRpZmZ1c2VycyUyMGltcG9ydCUyMEdhdWRpRERJTVNjaGVkdWxlciUyQyUyMEdhdWRpU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUlMEElMEFtb2RlbF9uYW1lJTIwJTNEJTIwJTIyQ29tcFZpcyUyRnN0YWJsZS1kaWZmdXNpb24tdjEtNCUyMiUwQSUwQXNjaGVkdWxlciUyMCUzRCUyMEdhdWRpRERJTVNjaGVkdWxlci5mcm9tX3ByZXRyYWluZWQobW9kZWxfbmFtZSUyQyUyMHN1YmZvbGRlciUzRCUyMnNjaGVkdWxlciUyMiklMEElMEFwaXBlbGluZSUyMCUzRCUyMEdhdWRpU3RhYmxlRGlmZnVzaW9uUGlwZWxpbmUuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMG1vZGVsX25hbWUlMkMlMEElMjAlMjAlMjAlMjBzY2hlZHVsZXIlM0RzY2hlZHVsZXIlMkMlMEElMjAlMjAlMjAlMjB1c2VfaGFiYW5hJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMHVzZV9ocHVfZ3JhcGhzJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGdhdWRpX2NvbmZpZyUzRCUyMkhhYmFuYSUyRnN0YWJsZS1kaWZmdXNpb24lMjIlMkMlMEEpJTBBJTBBb3V0cHV0cyUyMCUzRCUyMGdlbmVyYXRvciglMEElMjAlMjAlMjAlMjAlNUIlMjJBbiUyMGltYWdlJTIwb2YlMjBhJTIwc3F1aXJyZWwlMjBpbiUyMFBpY2Fzc28lMjBzdHlsZSUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMG51bV9pbWFnZXNfcGVyX3Byb21wdCUzRDE2JTJDJTBBJTIwJTIwJTIwJTIwYmF0Y2hfc2l6ZSUzRDQlMkMlMEEp",highlighted:`<span class="hljs-keyword">from</span> optimum.habana.diffusers <span class="hljs-keyword">import</span> GaudiDDIMScheduler, GaudiStableDiffusionPipeline

model_name = <span class="hljs-string">&quot;CompVis/stable-diffusion-v1-4&quot;</span>

scheduler = GaudiDDIMScheduler.from_pretrained(model_name, subfolder=<span class="hljs-string">&quot;scheduler&quot;</span>)

pipeline = GaudiStableDiffusionPipeline.from_pretrained(
    model_name,
    scheduler=scheduler,
    use_habana=<span class="hljs-literal">True</span>,
    use_hpu_graphs=<span class="hljs-literal">True</span>,
    gaudi_config=<span class="hljs-string">&quot;Habana/stable-diffusion&quot;</span>,
)

outputs = generator(
    [<span class="hljs-string">&quot;An image of a squirrel in Picasso style&quot;</span>],
    num_images_per_prompt=<span class="hljs-number">16</span>,
    batch_size=<span class="hljs-number">4</span>,
)`,wrap:!1}}),g=new ye({props:{warning:!0,$$slots:{default:[Le]},$$scope:{ctx:$}}}),S=new A({props:{title:"Custom Operators",local:"custom-operators",headingTag:"h2"}}),P=new A({props:{title:"Gaudi Optimized Flash Attention",local:"gaudi-optimized-flash-attention",headingTag:"h3"}}),Y=new Ye({props:{source:"https://github.com/huggingface/optimum-habana/blob/main/docs/source/usage_guides/accelerate_inference.mdx"}}),{c(){n=o("meta"),u=i(),r=o("p"),U=i(),d(_.$$.fragment),L=i(),d(b.$$.fragment),X=i(),J=o("p"),J.textContent=Te,Q=i(),d(C.$$.fragment),N=i(),G=o("p"),G.textContent=we,D=i(),Z=o("ul"),Z.innerHTML=ge,q=i(),d(T.$$.fragment),O=i(),j=o("p"),j.textContent=_e,K=i(),I=o("p"),I.textContent=be,ee=i(),d(v.$$.fragment),te=i(),d(w.$$.fragment),ae=i(),d(z.$$.fragment),le=i(),W=o("p"),W.innerHTML=Je,ne=i(),H=o("p"),H.innerHTML=Ce,ie=i(),R=o("ul"),R.innerHTML=Ge,se=i(),d(V.$$.fragment),re=i(),E=o("ul"),E.innerHTML=Ze,oe=i(),d(x.$$.fragment),pe=i(),d(g.$$.fragment),me=i(),d(S.$$.fragment),ue=i(),F=o("p"),F.innerHTML=je,de=i(),d(P.$$.fragment),fe=i(),B=o("p"),B.innerHTML=Ie,ce=i(),d(Y.$$.fragment),he=i(),k=o("p"),this.h()},l(e){const t=Fe("svelte-u9bgzb",document.head);n=p(t,"META",{name:!0,content:!0}),t.forEach(a),u=s(e),r=p(e,"P",{}),He(r).forEach(a),U=s(e),f(_.$$.fragment,e),L=s(e),f(b.$$.fragment,e),X=s(e),J=p(e,"P",{"data-svelte-h":!0}),m(J)!=="svelte-4rm23h"&&(J.textContent=Te),Q=s(e),f(C.$$.fragment,e),N=s(e),G=p(e,"P",{"data-svelte-h":!0}),m(G)!=="svelte-1n8dewo"&&(G.textContent=we),D=s(e),Z=p(e,"UL",{"data-svelte-h":!0}),m(Z)!=="svelte-j2rpi"&&(Z.innerHTML=ge),q=s(e),f(T.$$.fragment,e),O=s(e),j=p(e,"P",{"data-svelte-h":!0}),m(j)!=="svelte-7su8zv"&&(j.textContent=_e),K=s(e),I=p(e,"P",{"data-svelte-h":!0}),m(I)!=="svelte-13wgip2"&&(I.textContent=be),ee=s(e),f(v.$$.fragment,e),te=s(e),f(w.$$.fragment,e),ae=s(e),f(z.$$.fragment,e),le=s(e),W=p(e,"P",{"data-svelte-h":!0}),m(W)!=="svelte-19okodl"&&(W.innerHTML=Je),ne=s(e),H=p(e,"P",{"data-svelte-h":!0}),m(H)!=="svelte-1v21clw"&&(H.innerHTML=Ce),ie=s(e),R=p(e,"UL",{"data-svelte-h":!0}),m(R)!=="svelte-1xy7kaq"&&(R.innerHTML=Ge),se=s(e),f(V.$$.fragment,e),re=s(e),E=p(e,"UL",{"data-svelte-h":!0}),m(E)!=="svelte-4ihtcv"&&(E.innerHTML=Ze),oe=s(e),f(x.$$.fragment,e),pe=s(e),f(g.$$.fragment,e),me=s(e),f(S.$$.fragment,e),ue=s(e),F=p(e,"P",{"data-svelte-h":!0}),m(F)!=="svelte-1eucnbs"&&(F.innerHTML=je),de=s(e),f(P.$$.fragment,e),fe=s(e),B=p(e,"P",{"data-svelte-h":!0}),m(B)!=="svelte-11599gg"&&(B.innerHTML=Ie),ce=s(e),f(Y.$$.fragment,e),he=s(e),k=p(e,"P",{}),He(k).forEach(a),this.h()},h(){Re(n,"name","hf:doc:metadata"),Re(n,"content",Qe)},m(e,t){Pe(document.head,n),l(e,u,t),l(e,r,t),l(e,U,t),c(_,e,t),l(e,L,t),c(b,e,t),l(e,X,t),l(e,J,t),l(e,Q,t),c(C,e,t),l(e,N,t),l(e,G,t),l(e,D,t),l(e,Z,t),l(e,q,t),c(T,e,t),l(e,O,t),l(e,j,t),l(e,K,t),l(e,I,t),l(e,ee,t),c(v,e,t),l(e,te,t),c(w,e,t),l(e,ae,t),c(z,e,t),l(e,le,t),l(e,W,t),l(e,ne,t),l(e,H,t),l(e,ie,t),l(e,R,t),l(e,se,t),c(V,e,t),l(e,re,t),l(e,E,t),l(e,oe,t),c(x,e,t),l(e,pe,t),c(g,e,t),l(e,me,t),c(S,e,t),l(e,ue,t),l(e,F,t),l(e,de,t),c(P,e,t),l(e,fe,t),l(e,B,t),l(e,ce,t),c(Y,e,t),l(e,he,t),l(e,k,t),Me=!0},p(e,[t]){const ve={};t&2&&(ve.$$scope={dirty:t,ctx:e}),T.$set(ve);const ze={};t&2&&(ze.$$scope={dirty:t,ctx:e}),w.$set(ze);const We={};t&2&&(We.$$scope={dirty:t,ctx:e}),g.$set(We)},i(e){Me||(h(_.$$.fragment,e),h(b.$$.fragment,e),h(C.$$.fragment,e),h(T.$$.fragment,e),h(v.$$.fragment,e),h(w.$$.fragment,e),h(z.$$.fragment,e),h(V.$$.fragment,e),h(x.$$.fragment,e),h(g.$$.fragment,e),h(S.$$.fragment,e),h(P.$$.fragment,e),h(Y.$$.fragment,e),Me=!0)},o(e){M(_.$$.fragment,e),M(b.$$.fragment,e),M(C.$$.fragment,e),M(T.$$.fragment,e),M(v.$$.fragment,e),M(w.$$.fragment,e),M(z.$$.fragment,e),M(V.$$.fragment,e),M(x.$$.fragment,e),M(g.$$.fragment,e),M(S.$$.fragment,e),M(P.$$.fragment,e),M(Y.$$.fragment,e),Me=!1},d(e){e&&(a(u),a(r),a(U),a(L),a(X),a(J),a(Q),a(N),a(G),a(D),a(Z),a(q),a(O),a(j),a(K),a(I),a(ee),a(te),a(ae),a(le),a(W),a(ne),a(H),a(ie),a(R),a(se),a(re),a(E),a(oe),a(pe),a(me),a(ue),a(F),a(de),a(fe),a(B),a(ce),a(he),a(k)),a(n),y(_,e),y(b,e),y(C,e),y(T,e),y(v,e),y(w,e),y(z,e),y(V,e),y(x,e),y(g,e),y(S,e),y(P,e),y(Y,e)}}}const Qe='{"title":"Accelerating Inference","local":"accelerating-inference","sections":[{"title":"Lazy Mode","local":"lazy-mode","sections":[],"depth":2},{"title":"HPU Graphs","local":"hpu-graphs","sections":[],"depth":2},{"title":"Custom Operators","local":"custom-operators","sections":[{"title":"Gaudi Optimized Flash Attention","local":"gaudi-optimized-flash-attention","sections":[],"depth":3}],"depth":2}],"depth":1}';function Ne($){return Ee(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class tt extends xe{constructor(n){super(),Se(this,n,Ne,Xe,Ve,{})}}export{tt as component};
