import{s as le,o as ie,n as re}from"../chunks/scheduler.18a86fab.js";import{S as oe,i as me,g as o,s as l,r as F,A as pe,h as m,f as a,c as i,j as ae,u as U,x as G,k as ne,y as fe,a as n,v as C,d as T,t as Z,w as E}from"../chunks/index.98837b22.js";import{T as ue}from"../chunks/Tip.77304350.js";import{C as Q}from"../chunks/CodeBlock.8d0c2e8a.js";import{H as se}from"../chunks/Heading.7e7e0c5b.js";import{E as de}from"../chunks/index.499f366c.js";function Me(B){let s,f='You need a GPU with <a href="https://developer.nvidia.com/cuda-gpus#collapseOne" rel="nofollow">compute capability 9+</a> like a H100.';return{c(){s=o("p"),s.innerHTML=f},l(r){s=m(r,"P",{"data-svelte-h":!0}),G(s)!=="svelte-65qec"&&(s.innerHTML=f)},m(r,k){n(r,s,k)},p:re,d(r){r&&a(s)}}}function ce(B){let s,f,r,k,u,V,d,A='<a href="https://github.com/pytorch/FBGEMM" rel="nofollow">FBGEMM (Facebook GEneral Matrix Multiplication)</a> is a low-precision matrix multiplication library for small batch sizes and support for accuracy-loss minimizing techniques such as row-wise quantization and outlier-aware quantization. With FBGEMM, quantize a models weights to 8-bits/channel and the activations to 8-bits/token (also known as fp8 or w8a8).',j,p,J,M,S="Install the FBGEMM_GPU package with the command below to ensure you have the latest version.",x,c,H,h,O='If you’re having installation issues, try installing the <a href="https://pytorch.org/FBGEMM/fbgemm_gpu-development/InstallationInstructions.html#fbgemm-gpu-install-libraries:~:text=found%20here.-,Install%20the%20FBGEMM_GPU%20Package,-Install%20through%20PyTorch" rel="nofollow">nightly release</a>.',P,b,K='Create a <a href="/docs/transformers/main/en/main_classes/quantization#transformers.FbgemmFp8Config">FbgemmFp8Config</a> and pass it to <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> to quantize a model to fp8.',R,g,z,$,D='<a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.save_pretrained">save_pretrained()</a> and <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a> enable saving and loading a quantized model.',L,y,W,_,X,w,ee='Read the <a href="https://engineering.fb.com/2018/11/07/ml-applications/fbgemm/" rel="nofollow">Open-sourcing FBGEMM for state-of-the-art server-side inference</a> blog post for more details on FBGEMM.',I,v,Y,q,N;return u=new se({props:{title:"FBGEMM",local:"fbgemm",headingTag:"h1"}}),p=new ue({props:{warning:!1,$$slots:{default:[Me]},$$scope:{ctx:B}}}),c=new Q({props:{code:"cGlwJTIwaW5zdGFsbCUyMC0tdXBncmFkZSUyMGFjY2VsZXJhdGUlMjBmYmdlbW0tZ3B1JTIwdG9yY2g=",highlighted:"pip install --upgrade accelerate fbgemm-gpu torch",wrap:!1}}),g=new Q({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEZiZ2VtbUZwOENvbmZpZyUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMEZiZ2VtbUZwOENvbmZpZygpJTBBcXVhbnRpemVkX21vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMm1ldGEtbGxhbWElMkZNZXRhLUxsYW1hLTMtOEIlMjIlMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudGl6YXRpb25fY29uZmlnJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> FbgemmFp8Config, AutoModelForCausalLM

quantization_config = FbgemmFp8Config()
quantized_model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Meta-Llama-3-8B&quot;</span>,
    torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
    quantization_config=quantization_config
)`,wrap:!1}}),y=new Q({props:{code:"cXVhbnRfcGF0aCUyMCUzRCUyMCUyMiUyRnBhdGglMkZ0byUyRnNhdmUlMkZxdWFudGl6ZWQlMkZtb2RlbCUyMiUwQW1vZGVsLnNhdmVfcHJldHJhaW5lZChxdWFudF9wYXRoKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKHF1YW50X3BhdGglMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMik=",highlighted:`quant_path = <span class="hljs-string">&quot;/path/to/save/quantized/model&quot;</span>
model.save_pretrained(quant_path)
model = AutoModelForCausalLM.from_pretrained(quant_path, device_map=<span class="hljs-string">&quot;auto&quot;</span>)`,wrap:!1}}),_=new se({props:{title:"Resources",local:"resources",headingTag:"h2"}}),v=new de({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/fbgemm_fp8.md"}}),{c(){s=o("meta"),f=l(),r=o("p"),k=l(),F(u.$$.fragment),V=l(),d=o("p"),d.innerHTML=A,j=l(),F(p.$$.fragment),J=l(),M=o("p"),M.textContent=S,x=l(),F(c.$$.fragment),H=l(),h=o("p"),h.innerHTML=O,P=l(),b=o("p"),b.innerHTML=K,R=l(),F(g.$$.fragment),z=l(),$=o("p"),$.innerHTML=D,L=l(),F(y.$$.fragment),W=l(),F(_.$$.fragment),X=l(),w=o("p"),w.innerHTML=ee,I=l(),F(v.$$.fragment),Y=l(),q=o("p"),this.h()},l(e){const t=pe("svelte-u9bgzb",document.head);s=m(t,"META",{name:!0,content:!0}),t.forEach(a),f=i(e),r=m(e,"P",{}),ae(r).forEach(a),k=i(e),U(u.$$.fragment,e),V=i(e),d=m(e,"P",{"data-svelte-h":!0}),G(d)!=="svelte-1953uft"&&(d.innerHTML=A),j=i(e),U(p.$$.fragment,e),J=i(e),M=m(e,"P",{"data-svelte-h":!0}),G(M)!=="svelte-1el6id2"&&(M.textContent=S),x=i(e),U(c.$$.fragment,e),H=i(e),h=m(e,"P",{"data-svelte-h":!0}),G(h)!=="svelte-19ap7pt"&&(h.innerHTML=O),P=i(e),b=m(e,"P",{"data-svelte-h":!0}),G(b)!=="svelte-1qm0jq2"&&(b.innerHTML=K),R=i(e),U(g.$$.fragment,e),z=i(e),$=m(e,"P",{"data-svelte-h":!0}),G($)!=="svelte-la0of2"&&($.innerHTML=D),L=i(e),U(y.$$.fragment,e),W=i(e),U(_.$$.fragment,e),X=i(e),w=m(e,"P",{"data-svelte-h":!0}),G(w)!=="svelte-18lkb5k"&&(w.innerHTML=ee),I=i(e),U(v.$$.fragment,e),Y=i(e),q=m(e,"P",{}),ae(q).forEach(a),this.h()},h(){ne(s,"name","hf:doc:metadata"),ne(s,"content",he)},m(e,t){fe(document.head,s),n(e,f,t),n(e,r,t),n(e,k,t),C(u,e,t),n(e,V,t),n(e,d,t),n(e,j,t),C(p,e,t),n(e,J,t),n(e,M,t),n(e,x,t),C(c,e,t),n(e,H,t),n(e,h,t),n(e,P,t),n(e,b,t),n(e,R,t),C(g,e,t),n(e,z,t),n(e,$,t),n(e,L,t),C(y,e,t),n(e,W,t),C(_,e,t),n(e,X,t),n(e,w,t),n(e,I,t),C(v,e,t),n(e,Y,t),n(e,q,t),N=!0},p(e,[t]){const te={};t&2&&(te.$$scope={dirty:t,ctx:e}),p.$set(te)},i(e){N||(T(u.$$.fragment,e),T(p.$$.fragment,e),T(c.$$.fragment,e),T(g.$$.fragment,e),T(y.$$.fragment,e),T(_.$$.fragment,e),T(v.$$.fragment,e),N=!0)},o(e){Z(u.$$.fragment,e),Z(p.$$.fragment,e),Z(c.$$.fragment,e),Z(g.$$.fragment,e),Z(y.$$.fragment,e),Z(_.$$.fragment,e),Z(v.$$.fragment,e),N=!1},d(e){e&&(a(f),a(r),a(k),a(V),a(d),a(j),a(J),a(M),a(x),a(H),a(h),a(P),a(b),a(R),a(z),a($),a(L),a(W),a(X),a(w),a(I),a(Y),a(q)),a(s),E(u,e),E(p,e),E(c,e),E(g,e),E(y,e),E(_,e),E(v,e)}}}const he='{"title":"FBGEMM","local":"fbgemm","sections":[{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}';function be(B){return ie(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Fe extends oe{constructor(s){super(),me(this,s,be,ce,le,{})}}export{Fe as component};
