import{s as pt,o as ut,n as ct}from"../chunks/scheduler.18a86fab.js";import{S as ft,i as yt,g as r,s as l,r as u,A as ht,h as m,f as a,c as s,j as rt,u as c,x as w,k as mt,y as dt,a as n,v as f,d as y,t as h,w as d}from"../chunks/index.98837b22.js";import{T as wt}from"../chunks/Tip.77304350.js";import{C as D}from"../chunks/CodeBlock.8d0c2e8a.js";import{H as K}from"../chunks/Heading.7e7e0c5b.js";import{E as Mt}from"../chunks/index.499f366c.js";function bt(R){let o,M="The Transformers integration only supports weight quantization. Use the Quanto library directly if you need activation quantization, calibration, or QAT.";return{c(){o=r("p"),o.textContent=M},l(i){o=m(i,"P",{"data-svelte-h":!0}),w(o)!=="svelte-1qcj49f"&&(o.textContent=M)},m(i,F){n(i,o,F)},p:ct,d(i){i&&a(o)}}}function gt(R){let o,M,i,F,b,X,g,tt='<a href="https://github.com/huggingface/optimum-quanto" rel="nofollow">Quanto</a> is a PyTorch quantization backend for <a href="https://huggingface.co/docs/optimum/index" rel="nofollow">Optimum</a>. It features linear quantization for weights (float8, int8, int4, int2) with accuracy very similar to full-precision models. Quanto is compatible with any model modality and device, making it simple to use regardless of hardware.',z,$,et='Quanto is also compatible with <a href="https://pytorch.org/tutorials/intermediate/torch_compile_tutorial.html" rel="nofollow">torch.compile</a> for faster generation.',I,U,at="Install Quanto with the following command.",x,T,S,C,nt='Quantize a model by creating a <a href="/docs/transformers/main/en/main_classes/quantization#transformers.QuantoConfig">QuantoConfig</a> and specifying the <code>weights</code> parameter to quantize to. This works for any model in any modality as long as it contains <a href="https://pytorch.org/docs/stable/generated/torch.nn.Linear.html" rel="nofollow">torch.nn.Linear</a> layers.',E,p,N,v,H,J,Y,Z,ot='Wrap a Quanto model with <a href="https://pytorch.org/tutorials/intermediate/torch_compile_tutorial.html" rel="nofollow">torch.compile</a> for faster generation.',V,_,G,W,L,q,lt='Read the <a href="https://huggingface.co/blog/quanto-introduction" rel="nofollow">Quanto: a PyTorch quantization backend for Optimum</a> blog post to learn more about the library design and benchmarks.',B,Q,st='For more hands-on examples, take a look at the Quanto <a href="https://colab.research.google.com/drive/16CXfVmtdQvciSh9BopZUDYcmXCDpvgrT?usp=sharing" rel="nofollow">notebook</a>.',P,j,A,k,O;return b=new K({props:{title:"Optimum Quanto",local:"optimum-quanto",headingTag:"h1"}}),T=new D({props:{code:"cGlwJTIwaW5zdGFsbCUyMG9wdGltdW0tcXVhbnRvJTIwYWNjZWxlcmF0ZSUyMHRyYW5zZm9ybWVycw==",highlighted:"pip install optimum-quanto accelerate transformers",wrap:!1}}),p=new wt({props:{warning:!1,$$slots:{default:[bt]},$$scope:{ctx:R}}}),v=new D({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMFF1YW50b0NvbmZpZyUwQSUwQXF1YW50X2NvbmZpZyUyMCUzRCUyMFF1YW50b0NvbmZpZyh3ZWlnaHRzJTNEJTIyaW50OCUyMiklMEFtb2RlbCUyMCUzRCUyMHRyYW5zZm9ybWVycy5BdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIybWV0YS1sbGFtYSUyRkxsYW1hLTMuMS04QiUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEJTIyYXV0byUyMiUyQyUyMCUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTIwJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50X2NvbmZpZyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, QuantoConfig

quant_config = QuantoConfig(weights=<span class="hljs-string">&quot;int8&quot;</span>)
model = transformers.AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-3.1-8B&quot;</span>, 
    torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>, 
    device_map=<span class="hljs-string">&quot;auto&quot;</span>, 
    quantization_config=quant_config
)`,wrap:!1}}),J=new K({props:{title:"torch.compile",local:"torchcompile",headingTag:"h2"}}),_=new D({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yU3BlZWNoU2VxMlNlcSUyQyUyMFF1YW50b0NvbmZpZyUwQSUwQXF1YW50X2NvbmZpZyUyMCUzRCUyMFF1YW50b0NvbmZpZyh3ZWlnaHRzJTNEJTIyaW50OCUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclNwZWVjaFNlcTJTZXEuZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMm9wZW5haSUyRndoaXNwZXItbGFyZ2UtdjIlMjIlMkMlMEElMjAlMjB0b3JjaF9kdHlwZSUzRCUyMmF1dG8lMjIlMkMlMEElMjAlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUwQSUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudF9jb25maWclMEEpJTBBJTBBbW9kZWwlMjAlM0QlMjB0b3JjaC5jb21waWxlKG1vZGVsKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForSpeechSeq2Seq, QuantoConfig

quant_config = QuantoConfig(weights=<span class="hljs-string">&quot;int8&quot;</span>)
model = AutoModelForSpeechSeq2Seq.from_pretrained(
  <span class="hljs-string">&quot;openai/whisper-large-v2&quot;</span>,
  torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>,
  device_map=<span class="hljs-string">&quot;auto&quot;</span>,
  quantization_config=quant_config
)

model = torch.<span class="hljs-built_in">compile</span>(model)`,wrap:!1}}),W=new K({props:{title:"Resources",local:"resources",headingTag:"h2"}}),j=new Mt({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/quanto.md"}}),{c(){o=r("meta"),M=l(),i=r("p"),F=l(),u(b.$$.fragment),X=l(),g=r("p"),g.innerHTML=tt,z=l(),$=r("p"),$.innerHTML=et,I=l(),U=r("p"),U.textContent=at,x=l(),u(T.$$.fragment),S=l(),C=r("p"),C.innerHTML=nt,E=l(),u(p.$$.fragment),N=l(),u(v.$$.fragment),H=l(),u(J.$$.fragment),Y=l(),Z=r("p"),Z.innerHTML=ot,V=l(),u(_.$$.fragment),G=l(),u(W.$$.fragment),L=l(),q=r("p"),q.innerHTML=lt,B=l(),Q=r("p"),Q.innerHTML=st,P=l(),u(j.$$.fragment),A=l(),k=r("p"),this.h()},l(t){const e=ht("svelte-u9bgzb",document.head);o=m(e,"META",{name:!0,content:!0}),e.forEach(a),M=s(t),i=m(t,"P",{}),rt(i).forEach(a),F=s(t),c(b.$$.fragment,t),X=s(t),g=m(t,"P",{"data-svelte-h":!0}),w(g)!=="svelte-1cud5cn"&&(g.innerHTML=tt),z=s(t),$=m(t,"P",{"data-svelte-h":!0}),w($)!=="svelte-subwa2"&&($.innerHTML=et),I=s(t),U=m(t,"P",{"data-svelte-h":!0}),w(U)!=="svelte-djh72g"&&(U.textContent=at),x=s(t),c(T.$$.fragment,t),S=s(t),C=m(t,"P",{"data-svelte-h":!0}),w(C)!=="svelte-80bl5c"&&(C.innerHTML=nt),E=s(t),c(p.$$.fragment,t),N=s(t),c(v.$$.fragment,t),H=s(t),c(J.$$.fragment,t),Y=s(t),Z=m(t,"P",{"data-svelte-h":!0}),w(Z)!=="svelte-14sjas1"&&(Z.innerHTML=ot),V=s(t),c(_.$$.fragment,t),G=s(t),c(W.$$.fragment,t),L=s(t),q=m(t,"P",{"data-svelte-h":!0}),w(q)!=="svelte-1kv0bzr"&&(q.innerHTML=lt),B=s(t),Q=m(t,"P",{"data-svelte-h":!0}),w(Q)!=="svelte-yub5ma"&&(Q.innerHTML=st),P=s(t),c(j.$$.fragment,t),A=s(t),k=m(t,"P",{}),rt(k).forEach(a),this.h()},h(){mt(o,"name","hf:doc:metadata"),mt(o,"content",$t)},m(t,e){dt(document.head,o),n(t,M,e),n(t,i,e),n(t,F,e),f(b,t,e),n(t,X,e),n(t,g,e),n(t,z,e),n(t,$,e),n(t,I,e),n(t,U,e),n(t,x,e),f(T,t,e),n(t,S,e),n(t,C,e),n(t,E,e),f(p,t,e),n(t,N,e),f(v,t,e),n(t,H,e),f(J,t,e),n(t,Y,e),n(t,Z,e),n(t,V,e),f(_,t,e),n(t,G,e),f(W,t,e),n(t,L,e),n(t,q,e),n(t,B,e),n(t,Q,e),n(t,P,e),f(j,t,e),n(t,A,e),n(t,k,e),O=!0},p(t,[e]){const it={};e&2&&(it.$$scope={dirty:e,ctx:t}),p.$set(it)},i(t){O||(y(b.$$.fragment,t),y(T.$$.fragment,t),y(p.$$.fragment,t),y(v.$$.fragment,t),y(J.$$.fragment,t),y(_.$$.fragment,t),y(W.$$.fragment,t),y(j.$$.fragment,t),O=!0)},o(t){h(b.$$.fragment,t),h(T.$$.fragment,t),h(p.$$.fragment,t),h(v.$$.fragment,t),h(J.$$.fragment,t),h(_.$$.fragment,t),h(W.$$.fragment,t),h(j.$$.fragment,t),O=!1},d(t){t&&(a(M),a(i),a(F),a(X),a(g),a(z),a($),a(I),a(U),a(x),a(S),a(C),a(E),a(N),a(H),a(Y),a(Z),a(V),a(G),a(L),a(q),a(B),a(Q),a(P),a(A),a(k)),a(o),d(b,t),d(T,t),d(p,t),d(v,t),d(J,t),d(_,t),d(W,t),d(j,t)}}}const $t='{"title":"Optimum Quanto","local":"optimum-quanto","sections":[{"title":"torch.compile","local":"torchcompile","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}';function Ut(R){return ut(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Wt extends ft{constructor(o){super(),yt(this,o,Ut,gt,pt,{})}}export{Wt as component};
