import{s as Z,n as ee,o as te}from"../chunks/scheduler.d9d004e7.js";import{S as ne,i as ae,e as r,s as i,c as b,h as ie,a as o,d as n,b as s,f as X,g as v,j as I,k as Y,l as se,m as a,n as y,t as w,o as C,p as U}from"../chunks/index.4a2b774c.js";import{C as le,H as D,E as re}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.c93fcd94.js";function oe(Q){let l,L,P,M,f,F,u,N,m,W=`Intel® Gaudi® offers several possibilities to make inference faster. For examples of FP8 and UINT4 for Inference, see the
<a href="/examples/text-generation">text-generation</a> example.`,R,p,B=`This guide provides the steps required to enable FP8 and UINT4 precision on your Intel® Gaudi® AI
accelerator using the Intel® Neural Compressor (INC) package.`,k,c,q,h,O=`When running inference on large language models (LLMs), high memory usage is often the bottleneck. Therefore
using FP8 data type for inference on large language models halves the required memory bandwidth. In addition,
FP8 compute is twice as fast as BF16 compute, so even compute-bound workloads, such as offline inference on
large batch sizes benefit.`,z,g,J=`References to <a href="https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_FP8.html" rel="nofollow">Run Inference Using FP8</a>
section on <a href="https://docs.habana.ai/en/latest/index.html" rel="nofollow">Intel® Gaudi® AI Accelerator Documentation</a>.`,H,d,A,$,K=`When running inference on large language models (LLMs), high memory usage is often the bottleneck. Therefore,
using UINT4 data type for inference on large language models halves the required memory bandwidth compared to
running inference in FP8.`,E,_,V=`References to <a href="https://docs.habana.ai/en/latest/PyTorch/Inference_on_PyTorch/Inference_Using_UINT4.html" rel="nofollow">Run Inference Using UINT4</a>
section on <a href="https://docs.habana.ai/en/latest/index.html" rel="nofollow">Intel® Gaudi® AI Accelerator Documentation</a>.`,G,x,S,T,j;return f=new le({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),u=new D({props:{title:"Quantization",local:"quantization",headingTag:"h1"}}),c=new D({props:{title:"Run Inference Using FP8",local:"run-inference-using-fp8",headingTag:"h2"}}),d=new D({props:{title:"Run Inference Using UINT4",local:"run-inference-using-uint4",headingTag:"h2"}}),x=new re({props:{source:"https://github.com/huggingface/optimum-habana/blob/main/docs/source/usage_guides/quantization.mdx"}}),{c(){l=r("meta"),L=i(),P=r("p"),M=i(),b(f.$$.fragment),F=i(),b(u.$$.fragment),N=i(),m=r("p"),m.innerHTML=W,R=i(),p=r("p"),p.textContent=B,k=i(),b(c.$$.fragment),q=i(),h=r("p"),h.textContent=O,z=i(),g=r("p"),g.innerHTML=J,H=i(),b(d.$$.fragment),A=i(),$=r("p"),$.textContent=K,E=i(),_=r("p"),_.innerHTML=V,G=i(),b(x.$$.fragment),S=i(),T=r("p"),this.h()},l(e){const t=ie("svelte-u9bgzb",document.head);l=o(t,"META",{name:!0,content:!0}),t.forEach(n),L=s(e),P=o(e,"P",{}),X(P).forEach(n),M=s(e),v(f.$$.fragment,e),F=s(e),v(u.$$.fragment,e),N=s(e),m=o(e,"P",{"data-svelte-h":!0}),I(m)!=="svelte-1jcbq3h"&&(m.innerHTML=W),R=s(e),p=o(e,"P",{"data-svelte-h":!0}),I(p)!=="svelte-1dy6srl"&&(p.textContent=B),k=s(e),v(c.$$.fragment,e),q=s(e),h=o(e,"P",{"data-svelte-h":!0}),I(h)!=="svelte-pk1td8"&&(h.textContent=O),z=s(e),g=o(e,"P",{"data-svelte-h":!0}),I(g)!=="svelte-158uhkl"&&(g.innerHTML=J),H=s(e),v(d.$$.fragment,e),A=s(e),$=o(e,"P",{"data-svelte-h":!0}),I($)!=="svelte-18vusud"&&($.textContent=K),E=s(e),_=o(e,"P",{"data-svelte-h":!0}),I(_)!=="svelte-5xnwd5"&&(_.innerHTML=V),G=s(e),v(x.$$.fragment,e),S=s(e),T=o(e,"P",{}),X(T).forEach(n),this.h()},h(){Y(l,"name","hf:doc:metadata"),Y(l,"content",fe)},m(e,t){se(document.head,l),a(e,L,t),a(e,P,t),a(e,M,t),y(f,e,t),a(e,F,t),y(u,e,t),a(e,N,t),a(e,m,t),a(e,R,t),a(e,p,t),a(e,k,t),y(c,e,t),a(e,q,t),a(e,h,t),a(e,z,t),a(e,g,t),a(e,H,t),y(d,e,t),a(e,A,t),a(e,$,t),a(e,E,t),a(e,_,t),a(e,G,t),y(x,e,t),a(e,S,t),a(e,T,t),j=!0},p:ee,i(e){j||(w(f.$$.fragment,e),w(u.$$.fragment,e),w(c.$$.fragment,e),w(d.$$.fragment,e),w(x.$$.fragment,e),j=!0)},o(e){C(f.$$.fragment,e),C(u.$$.fragment,e),C(c.$$.fragment,e),C(d.$$.fragment,e),C(x.$$.fragment,e),j=!1},d(e){e&&(n(L),n(P),n(M),n(F),n(N),n(m),n(R),n(p),n(k),n(q),n(h),n(z),n(g),n(H),n(A),n($),n(E),n(_),n(G),n(S),n(T)),n(l),U(f,e),U(u,e),U(c,e),U(d,e),U(x,e)}}}const fe='{"title":"Quantization","local":"quantization","sections":[{"title":"Run Inference Using FP8","local":"run-inference-using-fp8","sections":[],"depth":2},{"title":"Run Inference Using UINT4","local":"run-inference-using-uint4","sections":[],"depth":2}],"depth":1}';function ue(Q){return te(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class he extends ne{constructor(l){super(),ae(this,l,ue,oe,Z,{})}}export{he as component};
