import{s as K,o as ee,n as O}from"../chunks/scheduler.18a86fab.js";import{S as te,i as ae,g as y,s as m,r as c,A as ne,h as b,f as l,c as f,j as A,u as d,x as q,k as P,y as se,a as o,v as h,d as $,t as M,w as T}from"../chunks/index.98837b22.js";import{C as N}from"../chunks/CodeBlock.8d0c2e8a.js";import{H as le}from"../chunks/Heading.7e7e0c5b.js";import{H as oe,a as D}from"../chunks/HfOption.6641485e.js";import{E as re}from"../chunks/index.499f366c.js";function ie(w){let t,r;return t=new N({props:{code:"cGlwJTIwaW5zdGFsbCUyMC0tbm8tY2FjaGUtZGlyJTIwaHR0cHMlM0ElMkYlMkZnaXRodWIuY29tJTJGTmV0RWFzZS1GdVhpJTJGRUVUUSUyRnJlbGVhc2VzJTJGZG93bmxvYWQlMkZ2MS4wLjAlMkZFRVRRLTEuMC4wJTJCY3UxMjElMkJ0b3JjaDIuMS4yLWNwMzEwLWNwMzEwLWxpbnV4X3g4Nl82NC53aGw=",highlighted:"pip install --no-cache-dir https://github.com/NetEase-FuXi/EETQ/releases/download/v1.0.0/EETQ-1.0.0+cu121+torch2.1.2-cp310-cp310-linux_x86_64.whl",wrap:!1}}),{c(){c(t.$$.fragment)},l(a){d(t.$$.fragment,a)},m(a,i){h(t,a,i),r=!0},p:O,i(a){r||($(t.$$.fragment,a),r=!0)},o(a){M(t.$$.fragment,a),r=!1},d(a){T(t,a)}}}function pe(w){let t,r;return t=new N({props:{code:"Z2l0JTIwY2xvbmUlMjBodHRwcyUzQSUyRiUyRmdpdGh1Yi5jb20lMkZOZXRFYXNlLUZ1WGklMkZFRVRRLmdpdCUwQWNkJTIwRUVUUSUyRiUwQWdpdCUyMHN1Ym1vZHVsZSUyMHVwZGF0ZSUyMC0taW5pdCUyMC0tcmVjdXJzaXZlJTBBcGlwJTIwaW5zdGFsbCUyMC4=",highlighted:`git <span class="hljs-built_in">clone</span> https://github.com/NetEase-FuXi/EETQ.git
<span class="hljs-built_in">cd</span> EETQ/
git submodule update --init --recursive
pip install .`,wrap:!1}}),{c(){c(t.$$.fragment)},l(a){d(t.$$.fragment,a)},m(a,i){h(t,a,i),r=!0},p:O,i(a){r||($(t.$$.fragment,a),r=!0)},o(a){M(t.$$.fragment,a),r=!1},d(a){T(t,a)}}}function me(w){let t,r,a,i;return t=new D({props:{id:"install",option:"release page",$$slots:{default:[ie]},$$scope:{ctx:w}}}),a=new D({props:{id:"install",option:"source code",$$slots:{default:[pe]},$$scope:{ctx:w}}}),{c(){c(t.$$.fragment),r=m(),c(a.$$.fragment)},l(s){d(t.$$.fragment,s),r=f(s),d(a.$$.fragment,s)},m(s,p){h(t,s,p),o(s,r,p),h(a,s,p),i=!0},p(s,p){const u={};p&2&&(u.$$scope={dirty:p,ctx:s}),t.$set(u);const v={};p&2&&(v.$$scope={dirty:p,ctx:s}),a.$set(v)},i(s){i||($(t.$$.fragment,s),$(a.$$.fragment,s),i=!0)},o(s){M(t.$$.fragment,s),M(a.$$.fragment,s),i=!1},d(s){s&&l(r),T(t,s),T(a,s)}}}function fe(w){let t,r,a,i,s,p,u,v='The <a href="https://github.com/NetEase-FuXi/EETQ" rel="nofollow">Easy &amp; Efficient Quantization for Transformers (EETQ)</a> library supports int8 weight-only per-channel quantization for NVIDIA GPUs. It uses high-performance GEMM and GEMV kernels from <a href="https://github.com/NVIDIA/FasterTransformer" rel="nofollow">FasterTransformer</a> and <a href="https://github.com/NVIDIA/TensorRT-LLM" rel="nofollow">TensorRT-LLM</a>. The attention layer is optimized with <a href="https://github.com/Dao-AILab/flash-attention" rel="nofollow">FlashAttention2</a>. No calibration dataset is required, and the model doesn’t need to be pre-quantized. Accuracy degradation is negligible owing to the per-channel quantization.',Q,E,H='EETQ further supports fine-tuning with <a href="https://huggingface.co/docs/peft" rel="nofollow">PEFT</a>.',F,U,Y='Install EETQ from the <a href="https://github.com/NetEase-FuXi/EETQ/releases" rel="nofollow">release page</a> or <a href="https://github.com/NetEase-FuXi/EETQ" rel="nofollow">source code</a>. CUDA 11.4+ is required for EETQ.',G,g,V,_,x='Quantize a model on-the-fly by defining the quantization data type in <a href="/docs/transformers/main/en/main_classes/quantization#transformers.EetqConfig">EetqConfig</a>.',X,J,k,C,S='Save the quantized model with <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.save_pretrained">save_pretrained()</a> so it can be reused again with <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a>.',I,Z,z,R,L,W,j;return s=new le({props:{title:"EETQ",local:"eetq",headingTag:"h1"}}),g=new oe({props:{id:"install",options:["release page","source code"],$$slots:{default:[me]},$$scope:{ctx:w}}}),J=new N({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwRWV0cUNvbmZpZyUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBFZXRxQ29uZmlnKCUyMmludDglMjIpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIybWV0YS1sbGFtYSUyRkxsYW1hLTMuMS04QiUyMiUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMEEp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, EetqConfig

quantization_config = EetqConfig(<span class="hljs-string">&quot;int8&quot;</span>)
model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-3.1-8B&quot;</span>,
    torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
    quantization_config=quantization_config
)`,wrap:!1}}),Z=new N({props:{code:"cXVhbnRfcGF0aCUyMCUzRCUyMCUyMiUyRnBhdGglMkZ0byUyRnNhdmUlMkZxdWFudGl6ZWQlMkZtb2RlbCUyMiUwQW1vZGVsLnNhdmVfcHJldHJhaW5lZChxdWFudF9wYXRoKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKHF1YW50X3BhdGglMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMik=",highlighted:`quant_path = <span class="hljs-string">&quot;/path/to/save/quantized/model&quot;</span>
model.save_pretrained(quant_path)
model = AutoModelForCausalLM.from_pretrained(quant_path, device_map=<span class="hljs-string">&quot;auto&quot;</span>)`,wrap:!1}}),R=new re({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/eetq.md"}}),{c(){t=y("meta"),r=m(),a=y("p"),i=m(),c(s.$$.fragment),p=m(),u=y("p"),u.innerHTML=v,Q=m(),E=y("p"),E.innerHTML=H,F=m(),U=y("p"),U.innerHTML=Y,G=m(),c(g.$$.fragment),V=m(),_=y("p"),_.innerHTML=x,X=m(),c(J.$$.fragment),k=m(),C=y("p"),C.innerHTML=S,I=m(),c(Z.$$.fragment),z=m(),c(R.$$.fragment),L=m(),W=y("p"),this.h()},l(e){const n=ne("svelte-u9bgzb",document.head);t=b(n,"META",{name:!0,content:!0}),n.forEach(l),r=f(e),a=b(e,"P",{}),A(a).forEach(l),i=f(e),d(s.$$.fragment,e),p=f(e),u=b(e,"P",{"data-svelte-h":!0}),q(u)!=="svelte-7naszb"&&(u.innerHTML=v),Q=f(e),E=b(e,"P",{"data-svelte-h":!0}),q(E)!=="svelte-1lzxy86"&&(E.innerHTML=H),F=f(e),U=b(e,"P",{"data-svelte-h":!0}),q(U)!=="svelte-1kzfwx8"&&(U.innerHTML=Y),G=f(e),d(g.$$.fragment,e),V=f(e),_=b(e,"P",{"data-svelte-h":!0}),q(_)!=="svelte-1d2g0ok"&&(_.innerHTML=x),X=f(e),d(J.$$.fragment,e),k=f(e),C=b(e,"P",{"data-svelte-h":!0}),q(C)!=="svelte-1r4ki86"&&(C.innerHTML=S),I=f(e),d(Z.$$.fragment,e),z=f(e),d(R.$$.fragment,e),L=f(e),W=b(e,"P",{}),A(W).forEach(l),this.h()},h(){P(t,"name","hf:doc:metadata"),P(t,"content",ue)},m(e,n){se(document.head,t),o(e,r,n),o(e,a,n),o(e,i,n),h(s,e,n),o(e,p,n),o(e,u,n),o(e,Q,n),o(e,E,n),o(e,F,n),o(e,U,n),o(e,G,n),h(g,e,n),o(e,V,n),o(e,_,n),o(e,X,n),h(J,e,n),o(e,k,n),o(e,C,n),o(e,I,n),h(Z,e,n),o(e,z,n),h(R,e,n),o(e,L,n),o(e,W,n),j=!0},p(e,[n]){const B={};n&2&&(B.$$scope={dirty:n,ctx:e}),g.$set(B)},i(e){j||($(s.$$.fragment,e),$(g.$$.fragment,e),$(J.$$.fragment,e),$(Z.$$.fragment,e),$(R.$$.fragment,e),j=!0)},o(e){M(s.$$.fragment,e),M(g.$$.fragment,e),M(J.$$.fragment,e),M(Z.$$.fragment,e),M(R.$$.fragment,e),j=!1},d(e){e&&(l(r),l(a),l(i),l(p),l(u),l(Q),l(E),l(F),l(U),l(G),l(V),l(_),l(X),l(k),l(C),l(I),l(z),l(L),l(W)),l(t),T(s,e),T(g,e),T(J,e),T(Z,e),T(R,e)}}}const ue='{"title":"EETQ","local":"eetq","sections":[],"depth":1}';function ce(w){return ee(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ge extends te{constructor(t){super(),ae(this,t,ce,fe,K,{})}}export{ge as component};
