import{s as P,n as F,o as X}from"../chunks/scheduler.d9d004e7.js";import{S as q,i as z,e as g,s as l,c as J,h as O,a as d,d as a,b as i,f as Y,g as w,j as S,k as D,l as K,m as n,n as U,t as I,o as b,p as $}from"../chunks/index.4a2b774c.js";import{C as ee,H as te,E as ae}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.2d77c8e6.js";import{C as H}from"../chunks/CodeBlock.8806415f.js";function ne(W){let s,G,y,C,o,x,m,j,r,N=`Text Generation Inference (TGI) on Intel® Gaudi® AI Accelerator is supported via <a href="https://github.com/huggingface/tgi-gaudi" rel="nofollow">Intel® Gaudi® TGI repository</a>.
Start TGI service on Gaudi system simply by <a href="https://github.com/huggingface/tgi-gaudi/pkgs/container/tgi-gaudi" rel="nofollow">pulling a TGI Gaudi Docker image</a> and launching a local TGI service instance.`,_,u,Q="For example, TGI service on Gaudi for <em>Llama 2 7B</em> model can be started with:",v,p,E,c,Z="You can then send a simple request:",B,T,L,f,R=`To run static benchmark test, please refer to
<a href="https://github.com/huggingface/text-generation-inference/tree/main/benchmark" rel="nofollow">TGI’s benchmark tool</a>.
More examples of running the service instances on single or multi HPU device system are available
<a href="https://github.com/huggingface/tgi-gaudi?tab=readme-ov-file#running-tgi-on-gaudi" rel="nofollow">here</a>.`,A,M,V,h,k;return o=new ee({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),m=new te({props:{title:"TGI on Gaudi",local:"tgi-on-gaudi",headingTag:"h1"}}),p=new H({props:{code:"ZG9ja2VyJTIwcnVuJTIwJTVDJTBBJTIwJTIwLXAlMjA4MDgwJTNBODAlMjAlNUMlMEElMjAlMjAtdiUyMCUyNFBXRCUyRmRhdGElM0ElMkZkYXRhJTIwJTVDJTBBJTIwJTIwLS1ydW50aW1lJTNEaGFiYW5hJTIwJTVDJTBBJTIwJTIwLWUlMjBIQUJBTkFfVklTSUJMRV9ERVZJQ0VTJTNEYWxsJTIwJTVDJTBBJTIwJTIwLWUlMjBPTVBJX01DQV9idGxfdmFkZXJfc2luZ2xlX2NvcHlfbWVjaGFuaXNtJTNEbm9uZSUyMCU1QyUwQSUyMCUyMC0tY2FwLWFkZCUzRHN5c19uaWNlJTIwJTVDJTBBJTIwJTIwLS1pcGMlM0Rob3N0JTIwZ2hjci5pbyUyRmh1Z2dpbmdmYWNlJTJGdGdpLWdhdWRpJTNBMi4wLjElMjAlNUMlMEElMjAlMjAtLW1vZGVsLWlkJTIwbWV0YS1sbGFtYSUyRkxsYW1hLTItN2ItaGYlMjAlNUMlMEElMjAlMjAtLW1heC1pbnB1dC10b2tlbnMlMjAxMDI0JTIwJTVDJTBBJTIwJTIwLS1tYXgtdG90YWwtdG9rZW5zJTIwMjA0OA==",highlighted:`docker run \\
  -p 8080:80 \\
  -v <span class="hljs-variable">$PWD</span>/data:/data \\
  --runtime=habana \\
  -e HABANA_VISIBLE_DEVICES=all \\
  -e OMPI_MCA_btl_vader_single_copy_mechanism=none \\
  --cap-add=sys_nice \\
  --ipc=host ghcr.io/huggingface/tgi-gaudi:2.0.1 \\
  --model-id meta-llama/Llama-2-7b-hf \\
  --max-input-tokens 1024 \\
  --max-total-tokens 2048`,wrap:!1}}),T=new H({props:{code:"Y3VybCUyMDEyNy4wLjAuMSUzQTgwODAlMkZnZW5lcmF0ZSUyMCU1QyUwQSUyMCUyMC1YJTIwUE9TVCUyMCU1QyUwQSUyMCUyMC1kJTIwJyU3QiUyMmlucHV0cyUyMiUzQSUyMldoYXQlMjBpcyUyMERlZXAlMjBMZWFybmluZyUzRiUyMiUyQyUyMnBhcmFtZXRlcnMlMjIlM0ElN0IlMjJtYXhfbmV3X3Rva2VucyUyMiUzQTMyJTdEJTdEJyUyMCU1QyUwQSUyMCUyMC1IJTIwJ0NvbnRlbnQtVHlwZSUzQSUyMGFwcGxpY2F0aW9uJTJGanNvbic=",highlighted:`curl 127.0.0.1:8080/generate \\
  -X POST \\
  -d <span class="hljs-string">&#x27;{&quot;inputs&quot;:&quot;What is Deep Learning?&quot;,&quot;parameters&quot;:{&quot;max_new_tokens&quot;:32}}&#x27;</span> \\
  -H <span class="hljs-string">&#x27;Content-Type: application/json&#x27;</span>`,wrap:!1}}),M=new ae({props:{source:"https://github.com/huggingface/optimum-habana/blob/main/docs/source/tutorials/tgi.mdx"}}),{c(){s=g("meta"),G=l(),y=g("p"),C=l(),J(o.$$.fragment),x=l(),J(m.$$.fragment),j=l(),r=g("p"),r.innerHTML=N,_=l(),u=g("p"),u.innerHTML=Q,v=l(),J(p.$$.fragment),E=l(),c=g("p"),c.textContent=Z,B=l(),J(T.$$.fragment),L=l(),f=g("p"),f.innerHTML=R,A=l(),J(M.$$.fragment),V=l(),h=g("p"),this.h()},l(e){const t=O("svelte-u9bgzb",document.head);s=d(t,"META",{name:!0,content:!0}),t.forEach(a),G=i(e),y=d(e,"P",{}),Y(y).forEach(a),C=i(e),w(o.$$.fragment,e),x=i(e),w(m.$$.fragment,e),j=i(e),r=d(e,"P",{"data-svelte-h":!0}),S(r)!=="svelte-9joj6i"&&(r.innerHTML=N),_=i(e),u=d(e,"P",{"data-svelte-h":!0}),S(u)!=="svelte-16oq2kc"&&(u.innerHTML=Q),v=i(e),w(p.$$.fragment,e),E=i(e),c=d(e,"P",{"data-svelte-h":!0}),S(c)!=="svelte-1fjbq7q"&&(c.textContent=Z),B=i(e),w(T.$$.fragment,e),L=i(e),f=d(e,"P",{"data-svelte-h":!0}),S(f)!=="svelte-ciua25"&&(f.innerHTML=R),A=i(e),w(M.$$.fragment,e),V=i(e),h=d(e,"P",{}),Y(h).forEach(a),this.h()},h(){D(s,"name","hf:doc:metadata"),D(s,"content",le)},m(e,t){K(document.head,s),n(e,G,t),n(e,y,t),n(e,C,t),U(o,e,t),n(e,x,t),U(m,e,t),n(e,j,t),n(e,r,t),n(e,_,t),n(e,u,t),n(e,v,t),U(p,e,t),n(e,E,t),n(e,c,t),n(e,B,t),U(T,e,t),n(e,L,t),n(e,f,t),n(e,A,t),U(M,e,t),n(e,V,t),n(e,h,t),k=!0},p:F,i(e){k||(I(o.$$.fragment,e),I(m.$$.fragment,e),I(p.$$.fragment,e),I(T.$$.fragment,e),I(M.$$.fragment,e),k=!0)},o(e){b(o.$$.fragment,e),b(m.$$.fragment,e),b(p.$$.fragment,e),b(T.$$.fragment,e),b(M.$$.fragment,e),k=!1},d(e){e&&(a(G),a(y),a(C),a(x),a(j),a(r),a(_),a(u),a(v),a(E),a(c),a(B),a(L),a(f),a(A),a(V),a(h)),a(s),$(o,e),$(m,e),$(p,e),$(T,e),$(M,e)}}}const le='{"title":"TGI on Gaudi","local":"tgi-on-gaudi","sections":[],"depth":1}';function ie(W){return X(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ue extends q{constructor(s){super(),z(this,s,ie,ne,P,{})}}export{ue as component};
