import{s as $e,o as Te,n as _e}from"../chunks/scheduler.9bc65507.js";import{S as ge,i as we,g as l,s as o,r as k,A as xe,h as s,f as n,c as r,j as ue,u as E,x as m,k as he,y as Pe,a as i,v as H,d as q,t as z,w as S}from"../chunks/index.707bf1b6.js";import{T as ve}from"../chunks/Tip.c2ecdbf4.js";import{H as U}from"../chunks/Heading.342b1fa6.js";function ye(X){let a,$="for PyTorch >= 1.14.0. JIT-mode could benefit any models for prediction and evaluaion since dict input is supported in jit.trace",d,p,u=`for PyTorch < 1.14.0. JIT-mode could benefit models whose forward parameter order matches the tuple input order in jit.trace, like question-answering model
In the case where the forward parameter order does not match the tuple input order in jit.trace, like text-classification models, jit.trace will fail and we are capturing this with the exception here to make it fallback. Logging is used to notify users.`;return{c(){a=l("p"),a.textContent=$,d=o(),p=l("p"),p.textContent=u},l(f){a=s(f,"P",{"data-svelte-h":!0}),m(a)!=="svelte-1eeqt0g"&&(a.textContent=$),d=r(f),p=s(f,"P",{"data-svelte-h":!0}),m(p)!=="svelte-122cw99"&&(p.textContent=u)},m(f,c){i(f,a,c),i(f,d,c),i(f,p,c)},p:_e,d(f){f&&(n(a),n(d),n(p))}}}function be(X){let a,$,d,p,u,f,c,ie="This guide focuses on inferencing large models efficiently on CPU.",A,T,B,_,oe='We have recently integrated <code>BetterTransformer</code> for faster inference on CPU for text, image and audio models. Check the documentation about this integration <a href="https://huggingface.co/docs/optimum/bettertransformer/overview" rel="nofollow">here</a> for more details.',G,g,F,w,re=`TorchScript is a way to create serializable and optimizable models from PyTorch code. Any TorchScript program can be saved from a Python process and loaded in a process where there is no Python dependency.
Comparing to default eager mode, jit mode in PyTorch normally yields better performance for model inference from optimization methodologies like operator fusion.`,O,x,ae='For a gentle introduction to TorchScript, see the Introduction to <a href="https://pytorch.org/tutorials/beginner/Intro_to_TorchScript_tutorial.html#tracing-modules" rel="nofollow">PyTorch TorchScript tutorial</a>.',N,P,R,v,le="Intel® Extension for PyTorch provides further optimizations in jit mode for Transformers series models. It is highly recommended for users to take advantage of Intel® Extension for PyTorch with jit mode. Some frequently used operator patterns from Transformers models are already supported in Intel® Extension for PyTorch with jit mode fusions. Those fusion patterns like Multi-head-attention fusion, Concat Linear, Linear+Add, Linear+Gelu, Add+LayerNorm fusion and etc. are enabled and perform well. The benefit of the fusion is delivered to users in a transparent fashion. According to the analysis, ~70% of most popular NLP tasks in question-answering, text-classification, and token-classification can get performance benefits with these fusion patterns for both Float32 precision and BFloat16 Mixed precision.",W,y,se='Check more detailed information for <a href="https://intel.github.io/intel-extension-for-pytorch/cpu/latest/tutorials/features/graph_optimization.html" rel="nofollow">IPEX Graph Optimization</a>.',D,b,K,C,fe='IPEX release is following PyTorch, check the approaches for <a href="https://intel.github.io/intel-extension-for-pytorch/" rel="nofollow">IPEX installation</a>.',Q,I,V,L,pe="To enable JIT-mode in Trainer for evaluaion or prediction, users should add <code>jit_mode_eval</code> in Trainer command arguments.",Y,h,Z,j,me='Take an example of the use cases on <a href="https://github.com/huggingface/transformers/tree/main/examples/pytorch/question-answering" rel="nofollow">Transformers question-answering</a>',ee,M,ce=`<li><p>Inference using jit mode on CPU:</p> <pre>python run_qa.py \\
--model_name_or_path csarron/bert-base-uncased-squad-v1 \\
--dataset_name squad \\
--do_eval \\
--max_seq_length 384 \\
--doc_stride 128 \\
--output_dir /tmp/ \\
--no_cuda \\
<b>--jit_mode_eval </b></pre></li> <li><p>Inference with IPEX using jit mode on CPU:</p> <pre>python run_qa.py \\
--model_name_or_path csarron/bert-base-uncased-squad-v1 \\
--dataset_name squad \\
--do_eval \\
--max_seq_length 384 \\
--doc_stride 128 \\
--output_dir /tmp/ \\
--no_cuda \\
<b>--use_ipex \\</b>
<b>--jit_mode_eval</b></pre></li>`,te,J,ne;return u=new U({props:{title:"Efficient Inference on CPU",local:"efficient-inference-on-cpu",headingTag:"h1"}}),T=new U({props:{title:"BetterTransformer for faster inference",local:"bettertransformer-for-faster-inference",headingTag:"h2"}}),g=new U({props:{title:"PyTorch JIT-mode (TorchScript)",local:"pytorch-jit-mode-torchscript",headingTag:"h2"}}),P=new U({props:{title:"IPEX Graph Optimization with JIT-mode",local:"ipex-graph-optimization-with-jit-mode",headingTag:"h3"}}),b=new U({props:{title:"IPEX installation:",local:"ipex-installation",headingTag:"h4"}}),I=new U({props:{title:"Usage of JIT-mode",local:"usage-of-jit-mode",headingTag:"h3"}}),h=new ve({props:{warning:!0,$$slots:{default:[ye]},$$scope:{ctx:X}}}),{c(){a=l("meta"),$=o(),d=l("p"),p=o(),k(u.$$.fragment),f=o(),c=l("p"),c.textContent=ie,A=o(),k(T.$$.fragment),B=o(),_=l("p"),_.innerHTML=oe,G=o(),k(g.$$.fragment),F=o(),w=l("p"),w.textContent=re,O=o(),x=l("p"),x.innerHTML=ae,N=o(),k(P.$$.fragment),R=o(),v=l("p"),v.textContent=le,W=o(),y=l("p"),y.innerHTML=se,D=o(),k(b.$$.fragment),K=o(),C=l("p"),C.innerHTML=fe,Q=o(),k(I.$$.fragment),V=o(),L=l("p"),L.innerHTML=pe,Y=o(),k(h.$$.fragment),Z=o(),j=l("p"),j.innerHTML=me,ee=o(),M=l("ul"),M.innerHTML=ce,te=o(),J=l("p"),this.h()},l(e){const t=xe("svelte-u9bgzb",document.head);a=s(t,"META",{name:!0,content:!0}),t.forEach(n),$=r(e),d=s(e,"P",{}),ue(d).forEach(n),p=r(e),E(u.$$.fragment,e),f=r(e),c=s(e,"P",{"data-svelte-h":!0}),m(c)!=="svelte-remud"&&(c.textContent=ie),A=r(e),E(T.$$.fragment,e),B=r(e),_=s(e,"P",{"data-svelte-h":!0}),m(_)!=="svelte-1tf28r"&&(_.innerHTML=oe),G=r(e),E(g.$$.fragment,e),F=r(e),w=s(e,"P",{"data-svelte-h":!0}),m(w)!=="svelte-19ymrru"&&(w.textContent=re),O=r(e),x=s(e,"P",{"data-svelte-h":!0}),m(x)!=="svelte-p4cknk"&&(x.innerHTML=ae),N=r(e),E(P.$$.fragment,e),R=r(e),v=s(e,"P",{"data-svelte-h":!0}),m(v)!=="svelte-17b7ikm"&&(v.textContent=le),W=r(e),y=s(e,"P",{"data-svelte-h":!0}),m(y)!=="svelte-1mws6j8"&&(y.innerHTML=se),D=r(e),E(b.$$.fragment,e),K=r(e),C=s(e,"P",{"data-svelte-h":!0}),m(C)!=="svelte-hnoa5g"&&(C.innerHTML=fe),Q=r(e),E(I.$$.fragment,e),V=r(e),L=s(e,"P",{"data-svelte-h":!0}),m(L)!=="svelte-1t82n9g"&&(L.innerHTML=pe),Y=r(e),E(h.$$.fragment,e),Z=r(e),j=s(e,"P",{"data-svelte-h":!0}),m(j)!=="svelte-1t7mun5"&&(j.innerHTML=me),ee=r(e),M=s(e,"UL",{"data-svelte-h":!0}),m(M)!=="svelte-143w2h7"&&(M.innerHTML=ce),te=r(e),J=s(e,"P",{}),ue(J).forEach(n),this.h()},h(){he(a,"name","hf:doc:metadata"),he(a,"content",Ce)},m(e,t){Pe(document.head,a),i(e,$,t),i(e,d,t),i(e,p,t),H(u,e,t),i(e,f,t),i(e,c,t),i(e,A,t),H(T,e,t),i(e,B,t),i(e,_,t),i(e,G,t),H(g,e,t),i(e,F,t),i(e,w,t),i(e,O,t),i(e,x,t),i(e,N,t),H(P,e,t),i(e,R,t),i(e,v,t),i(e,W,t),i(e,y,t),i(e,D,t),H(b,e,t),i(e,K,t),i(e,C,t),i(e,Q,t),H(I,e,t),i(e,V,t),i(e,L,t),i(e,Y,t),H(h,e,t),i(e,Z,t),i(e,j,t),i(e,ee,t),i(e,M,t),i(e,te,t),i(e,J,t),ne=!0},p(e,[t]){const de={};t&2&&(de.$$scope={dirty:t,ctx:e}),h.$set(de)},i(e){ne||(q(u.$$.fragment,e),q(T.$$.fragment,e),q(g.$$.fragment,e),q(P.$$.fragment,e),q(b.$$.fragment,e),q(I.$$.fragment,e),q(h.$$.fragment,e),ne=!0)},o(e){z(u.$$.fragment,e),z(T.$$.fragment,e),z(g.$$.fragment,e),z(P.$$.fragment,e),z(b.$$.fragment,e),z(I.$$.fragment,e),z(h.$$.fragment,e),ne=!1},d(e){e&&(n($),n(d),n(p),n(f),n(c),n(A),n(B),n(_),n(G),n(F),n(w),n(O),n(x),n(N),n(R),n(v),n(W),n(y),n(D),n(K),n(C),n(Q),n(V),n(L),n(Y),n(Z),n(j),n(ee),n(M),n(te),n(J)),n(a),S(u,e),S(T,e),S(g,e),S(P,e),S(b,e),S(I,e),S(h,e)}}}const Ce='{"title":"Efficient Inference on CPU","local":"efficient-inference-on-cpu","sections":[{"title":"BetterTransformer for faster inference","local":"bettertransformer-for-faster-inference","sections":[],"depth":2},{"title":"PyTorch JIT-mode (TorchScript)","local":"pytorch-jit-mode-torchscript","sections":[{"title":"IPEX Graph Optimization with JIT-mode","local":"ipex-graph-optimization-with-jit-mode","sections":[{"title":"IPEX installation:","local":"ipex-installation","sections":[],"depth":4}],"depth":3},{"title":"Usage of JIT-mode","local":"usage-of-jit-mode","sections":[],"depth":3}],"depth":2}],"depth":1}';function Ie(X){return Te(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ee extends ge{constructor(a){super(),we(this,a,Ie,be,$e,{})}}export{Ee as component};
