import{s as Ve,n as Ee,o as qe}from"../chunks/scheduler.56725da7.js";import{S as Re,i as Ye,e as o,s as a,c as p,h as Fe,a as i,d as l,b as s,f as Qe,g as u,j as m,k as ze,l as Pe,m as n,n as r,t as M,o as c,p as y}from"../chunks/index.18a26576.js";import{C as De}from"../chunks/CopyLLMTxtMenu.f4001e5e.js";import{C as V}from"../chunks/CodeBlock.e59f1853.js";import{H as R}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.d7e7b6b3.js";function Oe(Ce){let f,Y,E,F,d,P,T,D,w,$e=`The <code>optimum-neuron</code> package includes a <a href="https://docs.vllm.ai/en/latest/" rel="nofollow">vLLM</a> plugin
that registers an ‘optimum-neuron’ vLLM platform specifically designed to ease the deployment
of models hosted on the Hugging Face hub to AWS Trainium and Inferentia.`,O,J,ve="This platform supports two modes of operation:",K,h,Ie='<li>it can be used for the inference of pre-exported Neuron models directly from the hub,</li> <li>but it allows also the simplified deployment of vanilla models directly without recompilation using <a href="#hugging-face-neuron-cache">cached artifacts</a>.</li>',ee,U,xe="Notes",te,b,Be=`<li>only a relevant subset of all possible configurations for a given model are cached,</li> <li>you can use the <code>optimum-cli</code> to get all <a href="https://huggingface.co/docs/optimum-neuron/guides/cache_system#neuron-model-cache-lookup-inferentia-only" rel="nofollow">cached configurations</a> for each model.</li> <li>to deploy models that are not cached on the Hugging Face hub, you need to <a href="https://huggingface.co/docs/optimum-neuron/main/en/guides/export_model" rel="nofollow">export</a>
them beforehand.</li>`,le,g,ne,j,Le=`The easiest way to use the <code>optimum-neuron</code> vLLM platform is to launch an Amazon ec2 instance using
the <a href="https://aws.amazon.com/marketplace/pp/prodview-gr3e6yiscria2" rel="nofollow">Hugging Face Neuron Deep Learning AMI</a>.  If you decide NOT to make your life easier by using <a href="https://aws.amazon.com/marketplace/pp/prodview-gr3e6yiscria2" rel="nofollow">Hugging Face Neuron Deep Learning AMI</a>, you can install this functionality into your Neuron environment with <code>pip install optimum-neuron[neuronx,vllm]</code>.`,ae,C,_e="Note: Trn2 instances are not supported by the <code>optimum-neuron</code> platform yet.",se,$,Ae='<li>After launching the instance, follow the instructions in <a href="https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AccessingInstancesLinux.html" rel="nofollow">Connect to your instance</a> to connect to the instance</li> <li>Once inside your instance, activate the pre-installed <code>optimum-neuron</code> virtual environment by running</li>',oe,v,ie,I,me,x,Ge="The easiest way to test a model is to use the python API:",pe,B,ue,L,re,_,Ze="The easiest way to serve a model is to use the <code>optimum-cli</code>:",Me,A,ce,G,Xe="The model can be a pre-exported neuron model or a standard hub model.",ye,Z,He="When deploying a standard hub model, you can customize the way it will be exported:",fe,X,de,H,Se=`Note: by default <code>optimum-cli</code> will only <code>serve</code> standard models for which a cached configuration exists.
This behaviour can be overridden using the <code>--allow_non_cached_model</code> argument.`,Te,S,ke=`If you omit one parameter, <code>optimum-neuron</code> will select a default value for you based
on the deployment target and prioritizing cached configurations.`,we,k,Ne="Use the following command to test the model:",Je,N,he,W,Ue,Q,We="You can also launch an Open AI compatible inference server directly using vLLM entry points:",be,z,ge,q,je;return d=new De({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),T=new R({props:{title:"optimum-neuron plugin for vLLM",local:"optimum-neuron-plugin-for-vllm",headingTag:"h1"}}),g=new R({props:{title:"Setup",local:"setup",headingTag:"h2"}}),v=new V({props:{code:"c291cmNlJTIwJTJGb3B0JTJGYXdzX25ldXJvbnhfdmVudl9weXRvcmNoXzJfNyUyRmJpbiUyRmFjdGl2YXRl",highlighted:"source /opt/aws_neuronx_venv_pytorch_2_7/bin/activate",wrap:!1}}),I=new R({props:{title:"Generating content programmatically",local:"generating-content-programmatically",headingTag:"h2"}}),B=new V({props:{code:"ZnJvbSUyMHZsbG0lMjBpbXBvcnQlMjBMTE0lMkMlMjBTYW1wbGluZ1BhcmFtcyUwQSUwQXByb21wdHMlMjAlM0QlMjAlNUIlMEElMjAlMjAlMjAlMjAlMjJIZWxsbyUyQyUyMG15JTIwbmFtZSUyMGlzJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyVGhlJTIwcHJlc2lkZW50JTIwb2YlMjB0aGUlMjBVbml0ZWQlMjBTdGF0ZXMlMjBpcyUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMlRoZSUyMGNhcGl0YWwlMjBvZiUyMEZyYW5jZSUyMGlzJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyVGhlJTIwZnV0dXJlJTIwb2YlMjBBSSUyMGlzJTIyJTJDJTBBJTVEJTBBc2FtcGxpbmdfcGFyYW1zJTIwJTNEJTIwU2FtcGxpbmdQYXJhbXModGVtcGVyYXR1cmUlM0QwLjglMkMlMjB0b3BfcCUzRDAuOTUpJTBBJTBBbGxtJTIwJTNEJTIwTExNKG1vZGVsJTNEJTIydW5zbG90aCUyRkxsYW1hLTMuMi0xQi1JbnN0cnVjdCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1heF9udW1fc2VxcyUzRDQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtYXhfbW9kZWxfbGVuJTNENDA5NiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRlbnNvcl9wYXJhbGxlbF9zaXplJTNEMiklMEElMEFvdXRwdXRzJTIwJTNEJTIwbGxtLmdlbmVyYXRlKHByb21wdHMlMkMlMjBzYW1wbGluZ19wYXJhbXMpJTBBJTBBZm9yJTIwb3V0cHV0JTIwaW4lMjBvdXRwdXRzJTNBJTBBJTIwJTIwJTIwJTIwcHJvbXB0JTIwJTNEJTIwb3V0cHV0LnByb21wdCUwQSUyMCUyMCUyMCUyMGdlbmVyYXRlZF90ZXh0JTIwJTNEJTIwb3V0cHV0Lm91dHB1dHMlNUIwJTVELnRleHQlMEElMjAlMjAlMjAlMjBwcmludChmJTIyUHJvbXB0JTNBJTIwJTdCcHJvbXB0IXIlN0QlMkMlMjBHZW5lcmF0ZWQlMjB0ZXh0JTNBJTIwJTdCZ2VuZXJhdGVkX3RleHQhciU3RCUyMik=",highlighted:`<span class="hljs-keyword">from</span> vllm <span class="hljs-keyword">import</span> LLM, SamplingParams

prompts = [
    <span class="hljs-string">&quot;Hello, my name is&quot;</span>,
    <span class="hljs-string">&quot;The president of the United States is&quot;</span>,
    <span class="hljs-string">&quot;The capital of France is&quot;</span>,
    <span class="hljs-string">&quot;The future of AI is&quot;</span>,
]
sampling_params = SamplingParams(temperature=<span class="hljs-number">0.8</span>, top_p=<span class="hljs-number">0.95</span>)

llm = LLM(model=<span class="hljs-string">&quot;unsloth/Llama-3.2-1B-Instruct&quot;</span>,
          max_num_seqs=<span class="hljs-number">4</span>,
          max_model_len=<span class="hljs-number">4096</span>,
          tensor_parallel_size=<span class="hljs-number">2</span>)

outputs = llm.generate(prompts, sampling_params)

<span class="hljs-keyword">for</span> output <span class="hljs-keyword">in</span> outputs:
    prompt = output.prompt
    generated_text = output.outputs[<span class="hljs-number">0</span>].text
    <span class="hljs-built_in">print</span>(<span class="hljs-string">f&quot;Prompt: <span class="hljs-subst">{prompt!r}</span>, Generated text: <span class="hljs-subst">{generated_text!r}</span>&quot;</span>)`,wrap:!1}}),L=new R({props:{title:"Serving a model",local:"serving-a-model",headingTag:"h2"}}),A=new V({props:{code:"b3B0aW11bS1jbGklMjBuZXVyb24lMjBzZXJ2ZSUyMC0tbW9kZWwlM0QlM0Ntb2RlbF9uYW1lX29yX3BhdGglM0U=",highlighted:"optimum-cli neuron serve --model=&lt;model_name_or_path&gt;",wrap:!1}}),X=new V({props:{code:"b3B0aW11bS1jbGklMjBuZXVyb24lMjBzZXJ2ZSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbW9kZWwlM0QlMjJ1bnNsb3RoJTJGTGxhbWEtMy4xLTFCLUludHJ1Y3QlMjIlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWJhdGNoX3NpemUlM0Q0JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1zZXF1ZW5jZV9sZW5ndGglM0Q0MDk2JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS10ZW5zb3JfcGFyYWxsZWxfc2l6ZSUzRDIlMjAlNUM=",highlighted:`optimum-cli neuron serve \\
    --model=&quot;unsloth/Llama-3.1-1B-Intruct&quot; \\
    --batch_size=4 \\
    --sequence_length=4096 \\
    --tensor_parallel_size=2 \\`,wrap:!1}}),N=new V({props:{code:"Y3VybCUyMDEyNy4wLjAuMSUzQTgwODAlMkZ2MSUyRmNvbXBsZXRpb25zJTIwJTVDJTBBJTIwJTIwJTIwJTIwLUglMjAnQ29udGVudC1UeXBlJTNBJTIwYXBwbGljYXRpb24lMkZqc29uJyUyMCU1QyUwQSUyMCUyMCUyMCUyMC1YJTIwUE9TVCUyMCU1QyUwQSUyMCUyMCUyMCUyMC1kJTIwJyU3QiUyMnByb21wdCUyMiUzQSUyMk9uZSUyMG9mJTIwbXklMjBmb25kZXN0JTIwbWVtb3J5JTIwaXMlMjIlMkMlMjAlMjJ0ZW1wZXJhdHVyZSUyMiUzQSUyMDAuOCUyQyUyMCUyMm1heF90b2tlbnMlMjIlM0ExMjglN0Qn",highlighted:`curl 127.0.0.1:8080/v1/completions \\
    -H &#x27;Content-Type: application/json&#x27; \\
    -X POST \\
    -d &#x27;{&quot;prompt&quot;:&quot;One of my fondest memory is&quot;, &quot;temperature&quot;: 0.8, &quot;max_tokens&quot;:128}&#x27;`,wrap:!1}}),W=new R({props:{title:"Custom deployment for advanced users",local:"custom-deployment-for-advanced-users",headingTag:"h2"}}),z=new V({props:{code:"cHl0aG9uJTIwLW0lMjB2bGxtLmVudHJ5cG9pbnRzLm9wZW5haS5hcGlfc2VydmVyJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tb2RlbCUzRCUyMnVuc2xvdGglMkZMbGFtYS0zLjItMUItSW5zdHJ1Y3QlMjIlMjAlNUMlMEElMjAlMjAlMjAlMjAtLW1heC1udW0tc2VxcyUzRDQlMjAlNUMlMEElMjAlMjAlMjAlMjAtLW1heC1tb2RlbC1sZW4lM0Q0MDk2JTIwJTVDJTBBJTIwJTIwJTIwJTIwLS10ZW5zb3ItcGFyYWxsZWwtc2l6ZSUzRDIlMjAlNUMlMEElMjAlMjAlMjAlMjAtLXBvcnQlM0Q4MDgw",highlighted:`python -m vllm.entrypoints.openai.api_server \\
    --model=&quot;unsloth/Llama-3.2-1B-Instruct&quot; \\
    --max-num-seqs=4 \\
    --max-model-len=4096 \\
    --tensor-parallel-size=2 \\
    --port=8080`,wrap:!1}}),{c(){f=o("meta"),Y=a(),E=o("p"),F=a(),p(d.$$.fragment),P=a(),p(T.$$.fragment),D=a(),w=o("p"),w.innerHTML=$e,O=a(),J=o("p"),J.textContent=ve,K=a(),h=o("ul"),h.innerHTML=Ie,ee=a(),U=o("p"),U.textContent=xe,te=a(),b=o("ul"),b.innerHTML=Be,le=a(),p(g.$$.fragment),ne=a(),j=o("p"),j.innerHTML=Le,ae=a(),C=o("p"),C.innerHTML=_e,se=a(),$=o("ul"),$.innerHTML=Ae,oe=a(),p(v.$$.fragment),ie=a(),p(I.$$.fragment),me=a(),x=o("p"),x.textContent=Ge,pe=a(),p(B.$$.fragment),ue=a(),p(L.$$.fragment),re=a(),_=o("p"),_.innerHTML=Ze,Me=a(),p(A.$$.fragment),ce=a(),G=o("p"),G.textContent=Xe,ye=a(),Z=o("p"),Z.textContent=He,fe=a(),p(X.$$.fragment),de=a(),H=o("p"),H.innerHTML=Se,Te=a(),S=o("p"),S.innerHTML=ke,we=a(),k=o("p"),k.textContent=Ne,Je=a(),p(N.$$.fragment),he=a(),p(W.$$.fragment),Ue=a(),Q=o("p"),Q.textContent=We,be=a(),p(z.$$.fragment),ge=a(),q=o("p"),this.h()},l(e){const t=Fe("svelte-u9bgzb",document.head);f=i(t,"META",{name:!0,content:!0}),t.forEach(l),Y=s(e),E=i(e,"P",{}),Qe(E).forEach(l),F=s(e),u(d.$$.fragment,e),P=s(e),u(T.$$.fragment,e),D=s(e),w=i(e,"P",{"data-svelte-h":!0}),m(w)!=="svelte-28yqn4"&&(w.innerHTML=$e),O=s(e),J=i(e,"P",{"data-svelte-h":!0}),m(J)!=="svelte-z6gb1b"&&(J.textContent=ve),K=s(e),h=i(e,"UL",{"data-svelte-h":!0}),m(h)!=="svelte-1y9py5l"&&(h.innerHTML=Ie),ee=s(e),U=i(e,"P",{"data-svelte-h":!0}),m(U)!=="svelte-1y2vhyh"&&(U.textContent=xe),te=s(e),b=i(e,"UL",{"data-svelte-h":!0}),m(b)!=="svelte-1isgnu3"&&(b.innerHTML=Be),le=s(e),u(g.$$.fragment,e),ne=s(e),j=i(e,"P",{"data-svelte-h":!0}),m(j)!=="svelte-valqmb"&&(j.innerHTML=Le),ae=s(e),C=i(e,"P",{"data-svelte-h":!0}),m(C)!=="svelte-18hhclg"&&(C.innerHTML=_e),se=s(e),$=i(e,"UL",{"data-svelte-h":!0}),m($)!=="svelte-140kn6a"&&($.innerHTML=Ae),oe=s(e),u(v.$$.fragment,e),ie=s(e),u(I.$$.fragment,e),me=s(e),x=i(e,"P",{"data-svelte-h":!0}),m(x)!=="svelte-12fxvlq"&&(x.textContent=Ge),pe=s(e),u(B.$$.fragment,e),ue=s(e),u(L.$$.fragment,e),re=s(e),_=i(e,"P",{"data-svelte-h":!0}),m(_)!=="svelte-3wro4u"&&(_.innerHTML=Ze),Me=s(e),u(A.$$.fragment,e),ce=s(e),G=i(e,"P",{"data-svelte-h":!0}),m(G)!=="svelte-7r6j2m"&&(G.textContent=Xe),ye=s(e),Z=i(e,"P",{"data-svelte-h":!0}),m(Z)!=="svelte-li5n6k"&&(Z.textContent=He),fe=s(e),u(X.$$.fragment,e),de=s(e),H=i(e,"P",{"data-svelte-h":!0}),m(H)!=="svelte-1cus4pe"&&(H.innerHTML=Se),Te=s(e),S=i(e,"P",{"data-svelte-h":!0}),m(S)!=="svelte-55sgvw"&&(S.innerHTML=ke),we=s(e),k=i(e,"P",{"data-svelte-h":!0}),m(k)!=="svelte-1fua9k5"&&(k.textContent=Ne),Je=s(e),u(N.$$.fragment,e),he=s(e),u(W.$$.fragment,e),Ue=s(e),Q=i(e,"P",{"data-svelte-h":!0}),m(Q)!=="svelte-16sg5pw"&&(Q.textContent=We),be=s(e),u(z.$$.fragment,e),ge=s(e),q=i(e,"P",{}),Qe(q).forEach(l),this.h()},h(){ze(f,"name","hf:doc:metadata"),ze(f,"content",Ke)},m(e,t){Pe(document.head,f),n(e,Y,t),n(e,E,t),n(e,F,t),r(d,e,t),n(e,P,t),r(T,e,t),n(e,D,t),n(e,w,t),n(e,O,t),n(e,J,t),n(e,K,t),n(e,h,t),n(e,ee,t),n(e,U,t),n(e,te,t),n(e,b,t),n(e,le,t),r(g,e,t),n(e,ne,t),n(e,j,t),n(e,ae,t),n(e,C,t),n(e,se,t),n(e,$,t),n(e,oe,t),r(v,e,t),n(e,ie,t),r(I,e,t),n(e,me,t),n(e,x,t),n(e,pe,t),r(B,e,t),n(e,ue,t),r(L,e,t),n(e,re,t),n(e,_,t),n(e,Me,t),r(A,e,t),n(e,ce,t),n(e,G,t),n(e,ye,t),n(e,Z,t),n(e,fe,t),r(X,e,t),n(e,de,t),n(e,H,t),n(e,Te,t),n(e,S,t),n(e,we,t),n(e,k,t),n(e,Je,t),r(N,e,t),n(e,he,t),r(W,e,t),n(e,Ue,t),n(e,Q,t),n(e,be,t),r(z,e,t),n(e,ge,t),n(e,q,t),je=!0},p:Ee,i(e){je||(M(d.$$.fragment,e),M(T.$$.fragment,e),M(g.$$.fragment,e),M(v.$$.fragment,e),M(I.$$.fragment,e),M(B.$$.fragment,e),M(L.$$.fragment,e),M(A.$$.fragment,e),M(X.$$.fragment,e),M(N.$$.fragment,e),M(W.$$.fragment,e),M(z.$$.fragment,e),je=!0)},o(e){c(d.$$.fragment,e),c(T.$$.fragment,e),c(g.$$.fragment,e),c(v.$$.fragment,e),c(I.$$.fragment,e),c(B.$$.fragment,e),c(L.$$.fragment,e),c(A.$$.fragment,e),c(X.$$.fragment,e),c(N.$$.fragment,e),c(W.$$.fragment,e),c(z.$$.fragment,e),je=!1},d(e){e&&(l(Y),l(E),l(F),l(P),l(D),l(w),l(O),l(J),l(K),l(h),l(ee),l(U),l(te),l(b),l(le),l(ne),l(j),l(ae),l(C),l(se),l($),l(oe),l(ie),l(me),l(x),l(pe),l(ue),l(re),l(_),l(Me),l(ce),l(G),l(ye),l(Z),l(fe),l(de),l(H),l(Te),l(S),l(we),l(k),l(Je),l(he),l(Ue),l(Q),l(be),l(ge),l(q)),l(f),y(d,e),y(T,e),y(g,e),y(v,e),y(I,e),y(B,e),y(L,e),y(A,e),y(X,e),y(N,e),y(W,e),y(z,e)}}}const Ke='{"title":"optimum-neuron plugin for vLLM","local":"optimum-neuron-plugin-for-vllm","sections":[{"title":"Setup","local":"setup","sections":[],"depth":2},{"title":"Generating content programmatically","local":"generating-content-programmatically","sections":[],"depth":2},{"title":"Serving a model","local":"serving-a-model","sections":[],"depth":2},{"title":"Custom deployment for advanced users","local":"custom-deployment-for-advanced-users","sections":[],"depth":2}],"depth":1}';function et(Ce){return qe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ot extends Re{constructor(f){super(),Ye(this,f,et,Oe,Ve,{})}}export{ot as component};
