import{s as ve,o as be,n as we}from"../chunks/scheduler.18a86fab.js";import{S as Me,i as ye,g as m,s,r as p,A as Je,h as o,f as n,c as r,j as Te,u as d,x as g,k as ge,y as Le,a as l,v as $,d as c,t as u,w as T}from"../chunks/index.98837b22.js";import{T as he}from"../chunks/Tip.77304350.js";import{C as ae}from"../chunks/CodeBlock.8d0c2e8a.js";import{H as se}from"../chunks/Heading.7e7e0c5b.js";import{E as Ue}from"../chunks/index.499f366c.js";function Ce(V){let a,f='Refer to the <a href="https://huggingface.co/docs/text-generation-inference/basic_tutorials/non_core_models" rel="nofollow">Non-core model serving</a> guide for more details.';return{c(){a=m("p"),a.innerHTML=f},l(i){a=o(i,"P",{"data-svelte-h":!0}),g(a)!=="svelte-188p87p"&&(a.innerHTML=f)},m(i,h){l(i,a,h)},p:we,d(i){i&&n(a)}}}function Ie(V){let a,f='Refer to the <a href="https://docs.vllm.ai/en/latest/models/supported_models.html#transformers-fallback" rel="nofollow">Transformers fallback</a> section for more details.';return{c(){a=m("p"),a.innerHTML=f},l(i){a=o(i,"P",{"data-svelte-h":!0}),g(a)!=="svelte-1eqlmcr"&&(a.innerHTML=f)},m(i,h){l(i,a,h)},p:we,d(i){i&&n(a)}}}function _e(V){let a,f,i,h,b,N,M,re="Transformer models can be served for inference with specialized libraries such as Text Generation Inference (TGI) and vLLM. These libraries are specifically designed to optimize performance with LLMs and include many unique optimization features that may not be included in Transformers.",j,y,B,J,ie='<a href="https://huggingface.co/docs/text-generation-inference/index" rel="nofollow">TGI</a> can serve models that aren’t <a href="https://huggingface.co/docs/text-generation-inference/supported_models" rel="nofollow">natively implemented</a> by falling back on the Transformers implementation of the model. Some of TGIs high-performance features aren’t available in the Transformers implementation, but other features like continuous batching and streaming are still supported.',X,w,P,L,me="Serve a Transformers implementation the same way you’d serve a TGI model.",Q,U,A,C,oe="Add <code>--trust-remote_code</code> to the command to serve a custom Transformers model.",D,I,F,_,W,x,fe='<a href="https://docs.vllm.ai/en/latest/index.html" rel="nofollow">vLLM</a> can also serve a Transformers implementation of a model if it isn’t <a href="https://docs.vllm.ai/en/latest/models/supported_models.html#list-of-text-only-language-models" rel="nofollow">natively implemented</a> in vLLM.',E,Z,pe="Many features like quantization, LoRA adapters, and distributed inference and serving are supported for the Transformers implementation.",Y,v,q,S,de="By default, vLLM serves the native implementation and if it doesn’t exist, it falls back on the Transformers implementation. But you can also set <code>--model-impl transformers</code> to explicitly use the Transformers model implementation.",O,H,K,G,$e="Add the <code>trust-remote-code</code> parameter to enable loading a remote code model.",ee,k,te,R,ne,z,le;return b=new se({props:{title:"Serving",local:"serving",headingTag:"h1"}}),y=new se({props:{title:"TGI",local:"tgi",headingTag:"h2"}}),w=new he({props:{warning:!1,$$slots:{default:[Ce]},$$scope:{ctx:V}}}),U=new ae({props:{code:"ZG9ja2VyJTIwcnVuJTIwLS1ncHVzJTIwYWxsJTIwLS1zaG0tc2l6ZSUyMDFnJTIwLXAlMjA4MDgwJTNBODAlMjAtdiUyMCUyNHZvbHVtZSUzQSUyRmRhdGElMjBnaGNyLmlvJTJGaHVnZ2luZ2ZhY2UlMkZ0ZXh0LWdlbmVyYXRpb24taW5mZXJlbmNlJTNBbGF0ZXN0JTIwLS1tb2RlbC1pZCUyMGdwdDI=",highlighted:'docker <span class="hljs-keyword">run</span><span class="language-bash"> --gpus all --shm-size 1g -p 8080:80 -v <span class="hljs-variable">$volume</span>:/data ghcr.io/huggingface/text-generation-inference:latest --model-id gpt2</span>',wrap:!1}}),I=new ae({props:{code:"ZG9ja2VyJTIwcnVuJTIwLS1ncHVzJTIwYWxsJTIwLS1zaG0tc2l6ZSUyMDFnJTIwLXAlMjA4MDgwJTNBODAlMjAtdiUyMCUyNHZvbHVtZSUzQSUyRmRhdGElMjBnaGNyLmlvJTJGaHVnZ2luZ2ZhY2UlMkZ0ZXh0LWdlbmVyYXRpb24taW5mZXJlbmNlJTNBbGF0ZXN0JTIwLS1tb2RlbC1pZCUyMCUzQ0NVU1RPTV9NT0RFTF9JRCUzRSUyMC0tdHJ1c3QtcmVtb3RlLWNvZGU=",highlighted:'docker <span class="hljs-keyword">run</span><span class="language-bash"> --gpus all --shm-size 1g -p 8080:80 -v <span class="hljs-variable">$volume</span>:/data ghcr.io/huggingface/text-generation-inference:latest --model-id &lt;CUSTOM_MODEL_ID&gt; --trust-remote-code</span>',wrap:!1}}),_=new se({props:{title:"vLLM",local:"vllm",headingTag:"h2"}}),v=new he({props:{warning:!1,$$slots:{default:[Ie]},$$scope:{ctx:V}}}),H=new ae({props:{code:"dmxsbSUyMHNlcnZlJTIwUXdlbiUyRlF3ZW4yLjUtMS41Qi1JbnN0cnVjdCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tdGFzayUyMGdlbmVyYXRlJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tb2RlbC1pbXBsJTIwdHJhbnNmb3JtZXJz",highlighted:`vllm serve Qwen/Qwen2.5-1.5B-Instruct \\
    --task generate \\
    --model-impl transformers`,wrap:!1}}),k=new ae({props:{code:"dmxsbSUyMHNlcnZlJTIwUXdlbiUyRlF3ZW4yLjUtMS41Qi1JbnN0cnVjdCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tdGFzayUyMGdlbmVyYXRlJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tb2RlbC1pbXBsJTIwdHJhbnNmb3JtZXJzJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS10cnVzdC1yZW1vdGUtY29kZQ==",highlighted:`vllm serve Qwen/Qwen2.5-1.5B-Instruct \\
    --task generate \\
    --model-impl transformers \\
    --trust-remote-code`,wrap:!1}}),R=new Ue({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/serving.md"}}),{c(){a=m("meta"),f=s(),i=m("p"),h=s(),p(b.$$.fragment),N=s(),M=m("p"),M.textContent=re,j=s(),p(y.$$.fragment),B=s(),J=m("p"),J.innerHTML=ie,X=s(),p(w.$$.fragment),P=s(),L=m("p"),L.textContent=me,Q=s(),p(U.$$.fragment),A=s(),C=m("p"),C.innerHTML=oe,D=s(),p(I.$$.fragment),F=s(),p(_.$$.fragment),W=s(),x=m("p"),x.innerHTML=fe,E=s(),Z=m("p"),Z.textContent=pe,Y=s(),p(v.$$.fragment),q=s(),S=m("p"),S.innerHTML=de,O=s(),p(H.$$.fragment),K=s(),G=m("p"),G.innerHTML=$e,ee=s(),p(k.$$.fragment),te=s(),p(R.$$.fragment),ne=s(),z=m("p"),this.h()},l(e){const t=Je("svelte-u9bgzb",document.head);a=o(t,"META",{name:!0,content:!0}),t.forEach(n),f=r(e),i=o(e,"P",{}),Te(i).forEach(n),h=r(e),d(b.$$.fragment,e),N=r(e),M=o(e,"P",{"data-svelte-h":!0}),g(M)!=="svelte-4fem0o"&&(M.textContent=re),j=r(e),d(y.$$.fragment,e),B=r(e),J=o(e,"P",{"data-svelte-h":!0}),g(J)!=="svelte-1kihfet"&&(J.innerHTML=ie),X=r(e),d(w.$$.fragment,e),P=r(e),L=o(e,"P",{"data-svelte-h":!0}),g(L)!=="svelte-84rke5"&&(L.textContent=me),Q=r(e),d(U.$$.fragment,e),A=r(e),C=o(e,"P",{"data-svelte-h":!0}),g(C)!=="svelte-1h3jbpj"&&(C.innerHTML=oe),D=r(e),d(I.$$.fragment,e),F=r(e),d(_.$$.fragment,e),W=r(e),x=o(e,"P",{"data-svelte-h":!0}),g(x)!=="svelte-11h5k2g"&&(x.innerHTML=fe),E=r(e),Z=o(e,"P",{"data-svelte-h":!0}),g(Z)!=="svelte-13x4tbc"&&(Z.textContent=pe),Y=r(e),d(v.$$.fragment,e),q=r(e),S=o(e,"P",{"data-svelte-h":!0}),g(S)!=="svelte-pkwvzk"&&(S.innerHTML=de),O=r(e),d(H.$$.fragment,e),K=r(e),G=o(e,"P",{"data-svelte-h":!0}),g(G)!=="svelte-15zxqlm"&&(G.innerHTML=$e),ee=r(e),d(k.$$.fragment,e),te=r(e),d(R.$$.fragment,e),ne=r(e),z=o(e,"P",{}),Te(z).forEach(n),this.h()},h(){ge(a,"name","hf:doc:metadata"),ge(a,"content",xe)},m(e,t){Le(document.head,a),l(e,f,t),l(e,i,t),l(e,h,t),$(b,e,t),l(e,N,t),l(e,M,t),l(e,j,t),$(y,e,t),l(e,B,t),l(e,J,t),l(e,X,t),$(w,e,t),l(e,P,t),l(e,L,t),l(e,Q,t),$(U,e,t),l(e,A,t),l(e,C,t),l(e,D,t),$(I,e,t),l(e,F,t),$(_,e,t),l(e,W,t),l(e,x,t),l(e,E,t),l(e,Z,t),l(e,Y,t),$(v,e,t),l(e,q,t),l(e,S,t),l(e,O,t),$(H,e,t),l(e,K,t),l(e,G,t),l(e,ee,t),$(k,e,t),l(e,te,t),$(R,e,t),l(e,ne,t),l(e,z,t),le=!0},p(e,[t]){const ce={};t&2&&(ce.$$scope={dirty:t,ctx:e}),w.$set(ce);const ue={};t&2&&(ue.$$scope={dirty:t,ctx:e}),v.$set(ue)},i(e){le||(c(b.$$.fragment,e),c(y.$$.fragment,e),c(w.$$.fragment,e),c(U.$$.fragment,e),c(I.$$.fragment,e),c(_.$$.fragment,e),c(v.$$.fragment,e),c(H.$$.fragment,e),c(k.$$.fragment,e),c(R.$$.fragment,e),le=!0)},o(e){u(b.$$.fragment,e),u(y.$$.fragment,e),u(w.$$.fragment,e),u(U.$$.fragment,e),u(I.$$.fragment,e),u(_.$$.fragment,e),u(v.$$.fragment,e),u(H.$$.fragment,e),u(k.$$.fragment,e),u(R.$$.fragment,e),le=!1},d(e){e&&(n(f),n(i),n(h),n(N),n(M),n(j),n(B),n(J),n(X),n(P),n(L),n(Q),n(A),n(C),n(D),n(F),n(W),n(x),n(E),n(Z),n(Y),n(q),n(S),n(O),n(K),n(G),n(ee),n(te),n(ne),n(z)),n(a),T(b,e),T(y,e),T(w,e),T(U,e),T(I,e),T(_,e),T(v,e),T(H,e),T(k,e),T(R,e)}}}const xe='{"title":"Serving","local":"serving","sections":[{"title":"TGI","local":"tgi","sections":[],"depth":2},{"title":"vLLM","local":"vllm","sections":[],"depth":2}],"depth":1}';function Ze(V){return be(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ze extends Me{constructor(a){super(),ye(this,a,Ze,_e,ve,{})}}export{ze as component};
