import{s as At,n as kt,o as Ht}from"../chunks/scheduler.9cda857a.js";import{S as _t,i as Yt,e as p,s as n,c as s,h as Et,a as o,d as l,b as i,f as Lt,g as m,j as r,k as wt,l as qt,m as a,n as u,t as c,o as M,p as f}from"../chunks/index.003612b0.js";import{C as Ft}from"../chunks/CopyLLMTxtMenu.44d08a39.js";import{C as N}from"../chunks/CodeBlock.a90632f1.js";import{H as ht,E as Nt}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.e5cfda8c.js";function St(dt){let y,R,S,B,h,W,d,Z,w,Ut='<p>You can now deploy any llama.cpp compatible GGUF on Hugging Face Endpoints, read more about it <a href="https://huggingface.co/docs/inference-endpoints/en/others/llamacpp_container" rel="nofollow">here</a></p>',P,U,bt='Llama.cpp allows you to download and run inference on a GGUF simply by providing a path to the Hugging Face repo path and the file name. llama.cpp downloads the model checkpoint and automatically caches it. The location of the cache is defined by <code>LLAMA_CACHE</code> environment variable; read more about it <a href="https://github.com/ggerganov/llama.cpp/pull/7826" rel="nofollow">here</a>.',V,b,$t='You can install llama.cpp through brew (works on Mac and Linux), or you can build it from source. There are also pre-built binaries and Docker images that you can <a href="https://github.com/ggerganov/llama.cpp?tab=readme-ov-file#usage" rel="nofollow">check in the official documentation</a>.',X,$,O,g,z,T,gt="or, on windows via winget",D,C,K,j,tt,J,Tt="Step 1: Clone llama.cpp from GitHub.",et,v,lt,G,Ct="Step 2: Move into the llama.cpp folder and build it. You can also add hardware-specific flags (for ex: <code>-DGGML_CUDA=1</code> for Nvidia GPUs).",at,I,nt,x,jt='Note: for other hardware support (for ex: AMD ROCm, Intel SYCL), please refer to <a href="https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md" rel="nofollow">llama.cpp’s build guide</a>',it,L,Jt="Once installed, you can use the <code>llama-cli</code> or <code>llama-server</code> as follows:",pt,A,ot,k,vt="Note: You can explicitly add <code>-no-cnv</code> to run the CLI in raw completion mode (non-chat mode).",st,H,Gt="Additionally, you can invoke an OpenAI spec chat completions endpoint directly using the llama.cpp server:",mt,_,rt,Y,It="After running the server you can simply utilise the endpoint as below:",ut,E,ct,q,xt="Replace <code>-hf</code> with any valid Hugging Face hub repo name - off you go! 🦙",Mt,F,ft,Q,yt;return h=new Ft({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),d=new ht({props:{title:"GGUF usage with llama.cpp",local:"gguf-usage-with-llamacpp",headingTag:"h1"}}),$=new ht({props:{title:"Option 1: Install with brew/ winget",local:"option-1-install-with-brew-winget",headingTag:"h3"}}),g=new N({props:{code:"YnJldyUyMGluc3RhbGwlMjBsbGFtYS5jcHA=",highlighted:"brew install llama.cpp",wrap:!1}}),C=new N({props:{code:"d2luZ2V0JTIwaW5zdGFsbCUyMGxsYW1hLmNwcA==",highlighted:"winget install llama.cpp",wrap:!1}}),j=new ht({props:{title:"Option 2: build from source",local:"option-2-build-from-source",headingTag:"h3"}}),v=new N({props:{code:"Z2l0JTIwY2xvbmUlMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZnZ2VyZ2Fub3YlMkZsbGFtYS5jcHA=",highlighted:'git clone https:<span class="hljs-regexp">//gi</span>thub.com<span class="hljs-regexp">/ggerganov/</span>llama.cpp',wrap:!1}}),I=new N({props:{code:"Y2QlMjBsbGFtYS5jcHAlMEFjbWFrZSUyMC1CJTIwYnVpbGQlMjAlMjAlMjAlMjMlMjBvcHRpb25hbGx5JTJDJTIwYWRkJTIwLURHR01MX0NVREElM0RPTiUyMHRvJTIwYWN0aXZhdGUlMjBDVURBJTBBY21ha2UlMjAtLWJ1aWxkJTIwYnVpbGQlMjAtLWNvbmZpZyUyMFJlbGVhc2U=",highlighted:`cd <span class="hljs-keyword">llama.cpp
</span>cmake -<span class="hljs-keyword">B </span><span class="hljs-keyword">build </span>  <span class="hljs-comment"># optionally, add -DGGML_CUDA=ON to activate CUDA</span>
cmake --<span class="hljs-keyword">build </span><span class="hljs-keyword">build </span>--<span class="hljs-built_in">config</span> Release`,wrap:!1}}),A=new N({props:{code:"bGxhbWEtY2xpJTIwLWhmJTIwYmFydG93c2tpJTJGTGxhbWEtMy4yLTNCLUluc3RydWN0LUdHVUYlM0FROF8w",highlighted:"llama-cli -hf bartowski/Llama-3.2-3B-Instruct-GGUF:Q8_0",wrap:!1}}),_=new N({props:{code:"bGxhbWEtc2VydmVyJTIwLWhmJTIwYmFydG93c2tpJTJGTGxhbWEtMy4yLTNCLUluc3RydWN0LUdHVUYlM0FROF8w",highlighted:"llama-server -hf bartowski/Llama-3.2-3B-Instruct-GGUF:Q8_0",wrap:!1}}),E=new N({props:{code:"Y3VybCUyMGh0dHAlM0ElMkYlMkZsb2NhbGhvc3QlM0E4MDgwJTJGdjElMkZjaGF0JTJGY29tcGxldGlvbnMlMjAlNUMlMEEtSCUyMCUyMkNvbnRlbnQtVHlwZSUzQSUyMGFwcGxpY2F0aW9uJTJGanNvbiUyMiUyMCU1QyUwQS1IJTIwJTIyQXV0aG9yaXphdGlvbiUzQSUyMEJlYXJlciUyMG5vLWtleSUyMiUyMCU1QyUwQS1kJTIwJyU3QiUwQSUyMm1lc3NhZ2VzJTIyJTNBJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycm9sZSUyMiUzQSUyMCUyMnN5c3RlbSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlMjJZb3UlMjBhcmUlMjBhbiUyMEFJJTIwYXNzaXN0YW50LiUyMFlvdXIlMjB0b3AlMjBwcmlvcml0eSUyMGlzJTIwYWNoaWV2aW5nJTIwdXNlciUyMGZ1bGZpbGxtZW50JTIwdmlhJTIwaGVscGluZyUyMHRoZW0lMjB3aXRoJTIwdGhlaXIlMjByZXF1ZXN0cy4lMjIlMEElMjAlMjAlMjAlMjAlN0QlMkMlMEElMjAlMjAlMjAlMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJyb2xlJTIyJTNBJTIwJTIydXNlciUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmNvbnRlbnQlMjIlM0ElMjAlMjJXcml0ZSUyMGElMjBsaW1lcmljayUyMGFib3V0JTIwUHl0aG9uJTIwZXhjZXB0aW9ucyUyMiUwQSUyMCUyMCUyMCUyMCU3RCUwQSUyMCUyMCU1RCUwQSU3RCc=",highlighted:`curl http://localhost:8080/v1/chat/completions \\
-H <span class="hljs-string">&quot;Content-Type: application/json&quot;</span> \\
-H <span class="hljs-string">&quot;Authorization: Bearer no-key&quot;</span> \\
-d <span class="hljs-string">&#x27;{
&quot;messages&quot;: [
    {
        &quot;role&quot;: &quot;system&quot;,
        &quot;content&quot;: &quot;You are an AI assistant. Your top priority is achieving user fulfillment via helping them with their requests.&quot;
    },
    {
        &quot;role&quot;: &quot;user&quot;,
        &quot;content&quot;: &quot;Write a limerick about Python exceptions&quot;
    }
  ]
}&#x27;</span>`,wrap:!1}}),F=new Nt({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/gguf-llamacpp.md"}}),{c(){y=p("meta"),R=n(),S=p("p"),B=n(),s(h.$$.fragment),W=n(),s(d.$$.fragment),Z=n(),w=p("blockquote"),w.innerHTML=Ut,P=n(),U=p("p"),U.innerHTML=bt,V=n(),b=p("p"),b.innerHTML=$t,X=n(),s($.$$.fragment),O=n(),s(g.$$.fragment),z=n(),T=p("p"),T.textContent=gt,D=n(),s(C.$$.fragment),K=n(),s(j.$$.fragment),tt=n(),J=p("p"),J.textContent=Tt,et=n(),s(v.$$.fragment),lt=n(),G=p("p"),G.innerHTML=Ct,at=n(),s(I.$$.fragment),nt=n(),x=p("p"),x.innerHTML=jt,it=n(),L=p("p"),L.innerHTML=Jt,pt=n(),s(A.$$.fragment),ot=n(),k=p("p"),k.innerHTML=vt,st=n(),H=p("p"),H.textContent=Gt,mt=n(),s(_.$$.fragment),rt=n(),Y=p("p"),Y.textContent=It,ut=n(),s(E.$$.fragment),ct=n(),q=p("p"),q.innerHTML=xt,Mt=n(),s(F.$$.fragment),ft=n(),Q=p("p"),this.h()},l(t){const e=Et("svelte-u9bgzb",document.head);y=o(e,"META",{name:!0,content:!0}),e.forEach(l),R=i(t),S=o(t,"P",{}),Lt(S).forEach(l),B=i(t),m(h.$$.fragment,t),W=i(t),m(d.$$.fragment,t),Z=i(t),w=o(t,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),r(w)!=="svelte-wkkgku"&&(w.innerHTML=Ut),P=i(t),U=o(t,"P",{"data-svelte-h":!0}),r(U)!=="svelte-llq74"&&(U.innerHTML=bt),V=i(t),b=o(t,"P",{"data-svelte-h":!0}),r(b)!=="svelte-kldsj5"&&(b.innerHTML=$t),X=i(t),m($.$$.fragment,t),O=i(t),m(g.$$.fragment,t),z=i(t),T=o(t,"P",{"data-svelte-h":!0}),r(T)!=="svelte-1but92h"&&(T.textContent=gt),D=i(t),m(C.$$.fragment,t),K=i(t),m(j.$$.fragment,t),tt=i(t),J=o(t,"P",{"data-svelte-h":!0}),r(J)!=="svelte-1xohd7n"&&(J.textContent=Tt),et=i(t),m(v.$$.fragment,t),lt=i(t),G=o(t,"P",{"data-svelte-h":!0}),r(G)!=="svelte-1r0jgu0"&&(G.innerHTML=Ct),at=i(t),m(I.$$.fragment,t),nt=i(t),x=o(t,"P",{"data-svelte-h":!0}),r(x)!=="svelte-3pqgmd"&&(x.innerHTML=jt),it=i(t),L=o(t,"P",{"data-svelte-h":!0}),r(L)!=="svelte-56179e"&&(L.innerHTML=Jt),pt=i(t),m(A.$$.fragment,t),ot=i(t),k=o(t,"P",{"data-svelte-h":!0}),r(k)!=="svelte-1pxj7s7"&&(k.innerHTML=vt),st=i(t),H=o(t,"P",{"data-svelte-h":!0}),r(H)!=="svelte-1aqbixr"&&(H.textContent=Gt),mt=i(t),m(_.$$.fragment,t),rt=i(t),Y=o(t,"P",{"data-svelte-h":!0}),r(Y)!=="svelte-1qbn3ou"&&(Y.textContent=It),ut=i(t),m(E.$$.fragment,t),ct=i(t),q=o(t,"P",{"data-svelte-h":!0}),r(q)!=="svelte-rc3gcp"&&(q.innerHTML=xt),Mt=i(t),m(F.$$.fragment,t),ft=i(t),Q=o(t,"P",{}),Lt(Q).forEach(l),this.h()},h(){wt(y,"name","hf:doc:metadata"),wt(y,"content",Qt),wt(w,"class","tip")},m(t,e){qt(document.head,y),a(t,R,e),a(t,S,e),a(t,B,e),u(h,t,e),a(t,W,e),u(d,t,e),a(t,Z,e),a(t,w,e),a(t,P,e),a(t,U,e),a(t,V,e),a(t,b,e),a(t,X,e),u($,t,e),a(t,O,e),u(g,t,e),a(t,z,e),a(t,T,e),a(t,D,e),u(C,t,e),a(t,K,e),u(j,t,e),a(t,tt,e),a(t,J,e),a(t,et,e),u(v,t,e),a(t,lt,e),a(t,G,e),a(t,at,e),u(I,t,e),a(t,nt,e),a(t,x,e),a(t,it,e),a(t,L,e),a(t,pt,e),u(A,t,e),a(t,ot,e),a(t,k,e),a(t,st,e),a(t,H,e),a(t,mt,e),u(_,t,e),a(t,rt,e),a(t,Y,e),a(t,ut,e),u(E,t,e),a(t,ct,e),a(t,q,e),a(t,Mt,e),u(F,t,e),a(t,ft,e),a(t,Q,e),yt=!0},p:kt,i(t){yt||(c(h.$$.fragment,t),c(d.$$.fragment,t),c($.$$.fragment,t),c(g.$$.fragment,t),c(C.$$.fragment,t),c(j.$$.fragment,t),c(v.$$.fragment,t),c(I.$$.fragment,t),c(A.$$.fragment,t),c(_.$$.fragment,t),c(E.$$.fragment,t),c(F.$$.fragment,t),yt=!0)},o(t){M(h.$$.fragment,t),M(d.$$.fragment,t),M($.$$.fragment,t),M(g.$$.fragment,t),M(C.$$.fragment,t),M(j.$$.fragment,t),M(v.$$.fragment,t),M(I.$$.fragment,t),M(A.$$.fragment,t),M(_.$$.fragment,t),M(E.$$.fragment,t),M(F.$$.fragment,t),yt=!1},d(t){t&&(l(R),l(S),l(B),l(W),l(Z),l(w),l(P),l(U),l(V),l(b),l(X),l(O),l(z),l(T),l(D),l(K),l(tt),l(J),l(et),l(lt),l(G),l(at),l(nt),l(x),l(it),l(L),l(pt),l(ot),l(k),l(st),l(H),l(mt),l(rt),l(Y),l(ut),l(ct),l(q),l(Mt),l(ft),l(Q)),l(y),f(h,t),f(d,t),f($,t),f(g,t),f(C,t),f(j,t),f(v,t),f(I,t),f(A,t),f(_,t),f(E,t),f(F,t)}}}const Qt='{"title":"GGUF usage with llama.cpp","local":"gguf-usage-with-llamacpp","sections":[{"title":"Option 1: Install with brew/ winget","local":"option-1-install-with-brew-winget","sections":[],"depth":3},{"title":"Option 2: build from source","local":"option-2-build-from-source","sections":[],"depth":3}],"depth":1}';function Rt(dt){return Ht(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Xt extends _t{constructor(y){super(),Yt(this,y,Rt,St,At,{})}}export{Xt as component};
