import{s as ve,o as Qe,n as I}from"../chunks/scheduler.18a86fab.js";import{S as Ee,i as Ye,g as J,s as p,r as m,A as Ne,h as f,f as s,c as r,j as Pl,u as d,x as T,k as ke,y as G,a as o,v as M,d as h,t as y,w as b}from"../chunks/index.98837b22.js";import{T as Kl}from"../chunks/Tip.77304350.js";import{C as g}from"../chunks/CodeBlock.8d0c2e8a.js";import{H as X}from"../chunks/Heading.7e7e0c5b.js";import{H as vt,a as R}from"../chunks/HfOption.6641485e.js";import{E as He}from"../chunks/index.499f366c.js";function xe(w){let e,i='Refer to the torchao <a href="https://github.com/pytorch/ao#torchao-pytorch-architecture-optimization" rel="nofollow">README.md</a> for more details about the library.';return{c(){e=J("p"),e.innerHTML=i},l(l){e=f(l,"P",{"data-svelte-h":!0}),T(e)!=="svelte-1e1uvh6"&&(e.innerHTML=i)},m(l,c){o(l,e,c)},p:I,d(l){l&&s(e)}}}function Se(w){let e,i;return e=new g({props:{code:"JTIzJTIwVXBkYXRpbmclMjAlRjAlOUYlQTQlOTclMjBUcmFuc2Zvcm1lcnMlMjB0byUyMHRoZSUyMGxhdGVzdCUyMHZlcnNpb24lMkMlMjBhcyUyMHRoZSUyMGV4YW1wbGUlMjBzY3JpcHQlMjBiZWxvdyUyMHVzZXMlMjB0aGUlMjBuZXclMjBhdXRvJTIwY29tcGlsYXRpb24lMEElMjMlMjBTdGFibGUlMjByZWxlYXNlJTIwZnJvbSUyMFB5cGklMjB3aGljaCUyMHdpbGwlMjBkZWZhdWx0JTIwdG8lMjBDVURBJTIwMTIuNiUwQXBpcCUyMGluc3RhbGwlMjAtLXVwZ3JhZGUlMjB0b3JjaGFvJTIwdHJhbnNmb3JtZXJz",highlighted:`<span class="hljs-comment"># Updating 🤗 Transformers to the latest version, as the example script below uses the new auto compilation</span>
<span class="hljs-comment"># Stable release from Pypi which will default to CUDA 12.6</span>
pip install --upgrade torchao transformers`,wrap:!1}}),{c(){m(e.$$.fragment)},l(l){d(e.$$.fragment,l)},m(l,c){M(e,l,c),i=!0},p:I,i(l){i||(h(e.$$.fragment,l),i=!0)},o(l){y(e.$$.fragment,l),i=!1},d(l){b(e,l)}}}function Ae(w){let e,i="Stable Release from the PyTorch index",l,c,a;return c=new g({props:{code:"cGlwJTIwaW5zdGFsbCUyMHRvcmNoYW8lMjAtLWluZGV4LXVybCUyMGh0dHBzJTNBJTJGJTJGZG93bmxvYWQucHl0b3JjaC5vcmclMkZ3aGwlMkZjdTEyNiUyMCUyMyUyMG9wdGlvbnMlMjBhcmUlMjBjcHUlMkZjdTExOCUyRmN1MTI2JTJGY3UxMjg=",highlighted:'pip install torchao --index-url https://download.pytorch.org/whl/cu126 <span class="hljs-comment"># options are cpu/cu118/cu126/cu128</span>',wrap:!1}}),{c(){e=J("p"),e.textContent=i,l=p(),m(c.$$.fragment)},l(u){e=f(u,"P",{"data-svelte-h":!0}),T(e)!=="svelte-15ppzc6"&&(e.textContent=i),l=r(u),d(c.$$.fragment,u)},m(u,U){o(u,e,U),o(u,l,U),M(c,u,U),a=!0},p:I,i(u){a||(h(c.$$.fragment,u),a=!0)},o(u){y(c.$$.fragment,u),a=!1},d(u){u&&(s(e),s(l)),b(c,u)}}}function Le(w){let e,i,l,c;return e=new R({props:{id:"install torchao",option:"PyPi",$$slots:{default:[Se]},$$scope:{ctx:w}}}),l=new R({props:{id:"install torchao",option:"PyTorch Index",$$slots:{default:[Ae]},$$scope:{ctx:w}}}),{c(){m(e.$$.fragment),i=p(),m(l.$$.fragment)},l(a){d(e.$$.fragment,a),i=r(a),d(l.$$.fragment,a)},m(a,u){M(e,a,u),o(a,i,u),M(l,a,u),c=!0},p(a,u){const U={};u&2&&(U.$$scope={dirty:u,ctx:a}),e.$set(U);const Z={};u&2&&(Z.$$scope={dirty:u,ctx:a}),l.$set(Z)},i(a){c||(h(e.$$.fragment,a),h(l.$$.fragment,a),c=!0)},o(a){y(e.$$.fragment,a),y(l.$$.fragment,a),c=!1},d(a){a&&s(i),b(e,a),b(l,a)}}}function De(w){let e,i;return e=new g({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwVG9yY2hBb0NvbmZpZyUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQWZyb20lMjB0b3JjaGFvLnF1YW50aXphdGlvbiUyMGltcG9ydCUyMEZsb2F0OER5bmFtaWNBY3RpdmF0aW9uRmxvYXQ4V2VpZ2h0Q29uZmlnJTJDJTIwRmxvYXQ4V2VpZ2h0T25seUNvbmZpZyUwQSUwQXF1YW50X2NvbmZpZyUyMCUzRCUyMEZsb2F0OER5bmFtaWNBY3RpdmF0aW9uRmxvYXQ4V2VpZ2h0Q29uZmlnKCklMEElMjMlMjBvciUyMGZsb2F0OCUyMHdlaWdodCUyMG9ubHklMjBxdWFudGl6YXRpb24lMEElMjMlMjBxdWFudF9jb25maWclMjAlM0QlMjBGbG9hdDhXZWlnaHRPbmx5Q29uZmlnKCklMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwVG9yY2hBb0NvbmZpZyhxdWFudF90eXBlJTNEcXVhbnRfY29uZmlnKSUwQSUwQSUyMyUyMExvYWQlMjBhbmQlMjBxdWFudGl6ZSUyMHRoZSUyMG1vZGVsJTBBcXVhbnRpemVkX21vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMm1ldGEtbGxhbWElMkZMbGFtYS0zLjEtOEItSW5zdHJ1Y3QlMjIlMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudGl6YXRpb25fY29uZmlnJTBBKSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMm1ldGEtbGxhbWElMkZMbGFtYS0zLjEtOEItSW5zdHJ1Y3QlMjIpJTBBaW5wdXRfdGV4dCUyMCUzRCUyMCUyMldoYXQlMjBhcmUlMjB3ZSUyMGhhdmluZyUyMGZvciUyMGRpbm5lciUzRiUyMiUwQWlucHV0X2lkcyUyMCUzRCUyMHRva2VuaXplcihpbnB1dF90ZXh0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8oJTIyY3VkYSUyMiklMEElMEElMjMlMjBhdXRvLWNvbXBpbGUlMjB0aGUlMjBxdWFudGl6ZWQlMjBtb2RlbCUyMHdpdGglMjAlNjBjYWNoZV9pbXBsZW1lbnRhdGlvbiUzRCUyMnN0YXRpYyUyMiU2MCUyMHRvJTIwZ2V0JTIwc3BlZWQlMjB1cCUwQW91dHB1dCUyMCUzRCUyMHF1YW50aXplZF9tb2RlbC5nZW5lcmF0ZSgqKmlucHV0X2lkcyUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMTAlMkMlMjBjYWNoZV9pbXBsZW1lbnRhdGlvbiUzRCUyMnN0YXRpYyUyMiklMEFwcmludCh0b2tlbml6ZXIuZGVjb2RlKG91dHB1dCU1QjAlNUQlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSkp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TorchAoConfig, AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">from</span> torchao.quantization <span class="hljs-keyword">import</span> Float8DynamicActivationFloat8WeightConfig, Float8WeightOnlyConfig

quant_config = Float8DynamicActivationFloat8WeightConfig()
<span class="hljs-comment"># or float8 weight only quantization</span>
<span class="hljs-comment"># quant_config = Float8WeightOnlyConfig()</span>
quantization_config = TorchAoConfig(quant_type=quant_config)

<span class="hljs-comment"># Load and quantize the model</span>
quantized_model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-3.1-8B-Instruct&quot;</span>,
    torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
    quantization_config=quantization_config
)

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-3.1-8B-Instruct&quot;</span>)
input_text = <span class="hljs-string">&quot;What are we having for dinner?&quot;</span>
input_ids = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-comment"># auto-compile the quantized model with \`cache_implementation=&quot;static&quot;\` to get speed up</span>
output = quantized_model.generate(**input_ids, max_new_tokens=<span class="hljs-number">10</span>, cache_implementation=<span class="hljs-string">&quot;static&quot;</span>)
<span class="hljs-built_in">print</span>(tokenizer.decode(output[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`,wrap:!1}}),{c(){m(e.$$.fragment)},l(l){d(e.$$.fragment,l)},m(l,c){M(e,l,c),i=!0},p:I,i(l){i||(h(e.$$.fragment,l),i=!0)},o(l){y(e.$$.fragment,l),i=!1},d(l){b(e,l)}}}function Pe(w){let e,i;return e=new g({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwVG9yY2hBb0NvbmZpZyUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQWZyb20lMjB0b3JjaGFvLnF1YW50aXphdGlvbiUyMGltcG9ydCUyMEdlbWxpdGVVSW50WFdlaWdodE9ubHlDb25maWclMEElMEElMjMlMjBXZSUyMGludGVncmF0ZWQlMjB3aXRoJTIwZ2VtbGl0ZSUyQyUyMHdoaWNoJTIwb3B0aW1pemVzJTIwZm9yJTIwYmF0Y2glMjBzaXplJTIwTiUyMG9uJTIwQTEwMCUyMGFuZCUyMEgxMDAlMEFxdWFudF9jb25maWclMjAlM0QlMjBHZW1saXRlVUludFhXZWlnaHRPbmx5Q29uZmlnKGdyb3VwX3NpemUlM0QxMjgpJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMFRvcmNoQW9Db25maWcocXVhbnRfdHlwZSUzRHF1YW50X2NvbmZpZyklMEElMEElMjMlMjBMb2FkJTIwYW5kJTIwcXVhbnRpemUlMjB0aGUlMjBtb2RlbCUwQXF1YW50aXplZF9tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJtZXRhLWxsYW1hJTJGTGxhbWEtMy4xLThCLUluc3RydWN0JTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0QlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUwQSklMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJtZXRhLWxsYW1hJTJGTGxhbWEtMy4xLThCLUluc3RydWN0JTIyKSUwQWlucHV0X3RleHQlMjAlM0QlMjAlMjJXaGF0JTIwYXJlJTIwd2UlMjBoYXZpbmclMjBmb3IlMjBkaW5uZXIlM0YlMjIlMEFpbnB1dF9pZHMlMjAlM0QlMjB0b2tlbml6ZXIoaW5wdXRfdGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKCUyMmN1ZGElMjIpJTBBJTBBJTIzJTIwYXV0by1jb21waWxlJTIwdGhlJTIwcXVhbnRpemVkJTIwbW9kZWwlMjB3aXRoJTIwJTYwY2FjaGVfaW1wbGVtZW50YXRpb24lM0QlMjJzdGF0aWMlMjIlNjAlMjB0byUyMGdldCUyMHNwZWVkJTIwdXAlMEFvdXRwdXQlMjAlM0QlMjBxdWFudGl6ZWRfbW9kZWwuZ2VuZXJhdGUoKippbnB1dF9pZHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDEwJTJDJTIwY2FjaGVfaW1wbGVtZW50YXRpb24lM0QlMjJzdGF0aWMlMjIpJTBBcHJpbnQodG9rZW5pemVyLmRlY29kZShvdXRwdXQlNUIwJTVEJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TorchAoConfig, AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">from</span> torchao.quantization <span class="hljs-keyword">import</span> GemliteUIntXWeightOnlyConfig

<span class="hljs-comment"># We integrated with gemlite, which optimizes for batch size N on A100 and H100</span>
quant_config = GemliteUIntXWeightOnlyConfig(group_size=<span class="hljs-number">128</span>)
quantization_config = TorchAoConfig(quant_type=quant_config)

<span class="hljs-comment"># Load and quantize the model</span>
quantized_model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-3.1-8B-Instruct&quot;</span>,
    torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
    quantization_config=quantization_config
)

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-3.1-8B-Instruct&quot;</span>)
input_text = <span class="hljs-string">&quot;What are we having for dinner?&quot;</span>
input_ids = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-comment"># auto-compile the quantized model with \`cache_implementation=&quot;static&quot;\` to get speed up</span>
output = quantized_model.generate(**input_ids, max_new_tokens=<span class="hljs-number">10</span>, cache_implementation=<span class="hljs-string">&quot;static&quot;</span>)
<span class="hljs-built_in">print</span>(tokenizer.decode(output[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`,wrap:!1}}),{c(){m(e.$$.fragment)},l(l){d(e.$$.fragment,l)},m(l,c){M(e,l,c),i=!0},p:I,i(l){i||(h(e.$$.fragment,l),i=!0)},o(l){y(e.$$.fragment,l),i=!1},d(l){b(e,l)}}}function Ke(w){let e,i,l,c;return e=new R({props:{id:"examples-H100-GPU",option:"float8-dynamic-and-weight-only",$$slots:{default:[De]},$$scope:{ctx:w}}}),l=new R({props:{id:"examples-H100-GPU",option:"int4-weight-only",$$slots:{default:[Pe]},$$scope:{ctx:w}}}),{c(){m(e.$$.fragment),i=p(),m(l.$$.fragment)},l(a){d(e.$$.fragment,a),i=r(a),d(l.$$.fragment,a)},m(a,u){M(e,a,u),o(a,i,u),M(l,a,u),c=!0},p(a,u){const U={};u&2&&(U.$$scope={dirty:u,ctx:a}),e.$set(U);const Z={};u&2&&(Z.$$scope={dirty:u,ctx:a}),l.$set(Z)},i(a){c||(h(e.$$.fragment,a),h(l.$$.fragment,a),c=!0)},o(a){y(e.$$.fragment,a),y(l.$$.fragment,a),c=!1},d(a){a&&s(i),b(e,a),b(l,a)}}}function Oe(w){let e,i;return e=new g({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwVG9yY2hBb0NvbmZpZyUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQWZyb20lMjB0b3JjaGFvLnF1YW50aXphdGlvbiUyMGltcG9ydCUyMEludDhEeW5hbWljQWN0aXZhdGlvbkludDhXZWlnaHRDb25maWclMkMlMjBJbnQ4V2VpZ2h0T25seUNvbmZpZyUwQSUwQXF1YW50X2NvbmZpZyUyMCUzRCUyMEludDhEeW5hbWljQWN0aXZhdGlvbkludDhXZWlnaHRDb25maWcoKSUwQSUyMyUyMG9yJTIwaW50OCUyMHdlaWdodCUyMG9ubHklMjBxdWFudGl6YXRpb24lMEElMjMlMjBxdWFudF9jb25maWclMjAlM0QlMjBJbnQ4V2VpZ2h0T25seUNvbmZpZygpJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMFRvcmNoQW9Db25maWcocXVhbnRfdHlwZSUzRHF1YW50X2NvbmZpZyklMEElMEElMjMlMjBMb2FkJTIwYW5kJTIwcXVhbnRpemUlMjB0aGUlMjBtb2RlbCUwQXF1YW50aXplZF9tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJtZXRhLWxsYW1hJTJGTGxhbWEtMy4xLThCLUluc3RydWN0JTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0QlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRpemF0aW9uX2NvbmZpZyUwQSklMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJtZXRhLWxsYW1hJTJGTGxhbWEtMy4xLThCLUluc3RydWN0JTIyKSUwQWlucHV0X3RleHQlMjAlM0QlMjAlMjJXaGF0JTIwYXJlJTIwd2UlMjBoYXZpbmclMjBmb3IlMjBkaW5uZXIlM0YlMjIlMEFpbnB1dF9pZHMlMjAlM0QlMjB0b2tlbml6ZXIoaW5wdXRfdGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKCUyMmN1ZGElMjIpJTBBJTBBJTIzJTIwYXV0by1jb21waWxlJTIwdGhlJTIwcXVhbnRpemVkJTIwbW9kZWwlMjB3aXRoJTIwJTYwY2FjaGVfaW1wbGVtZW50YXRpb24lM0QlMjJzdGF0aWMlMjIlNjAlMjB0byUyMGdldCUyMHNwZWVkJTIwdXAlMEFvdXRwdXQlMjAlM0QlMjBxdWFudGl6ZWRfbW9kZWwuZ2VuZXJhdGUoKippbnB1dF9pZHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDEwJTJDJTIwY2FjaGVfaW1wbGVtZW50YXRpb24lM0QlMjJzdGF0aWMlMjIpJTBBcHJpbnQodG9rZW5pemVyLmRlY29kZShvdXRwdXQlNUIwJTVEJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpKQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TorchAoConfig, AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">from</span> torchao.quantization <span class="hljs-keyword">import</span> Int8DynamicActivationInt8WeightConfig, Int8WeightOnlyConfig

quant_config = Int8DynamicActivationInt8WeightConfig()
<span class="hljs-comment"># or int8 weight only quantization</span>
<span class="hljs-comment"># quant_config = Int8WeightOnlyConfig()</span>
quantization_config = TorchAoConfig(quant_type=quant_config)

<span class="hljs-comment"># Load and quantize the model</span>
quantized_model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-3.1-8B-Instruct&quot;</span>,
    torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
    quantization_config=quantization_config
)

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-3.1-8B-Instruct&quot;</span>)
input_text = <span class="hljs-string">&quot;What are we having for dinner?&quot;</span>
input_ids = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-comment"># auto-compile the quantized model with \`cache_implementation=&quot;static&quot;\` to get speed up</span>
output = quantized_model.generate(**input_ids, max_new_tokens=<span class="hljs-number">10</span>, cache_implementation=<span class="hljs-string">&quot;static&quot;</span>)
<span class="hljs-built_in">print</span>(tokenizer.decode(output[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`,wrap:!1}}),{c(){m(e.$$.fragment)},l(l){d(e.$$.fragment,l)},m(l,c){M(e,l,c),i=!0},p:I,i(l){i||(h(e.$$.fragment,l),i=!0)},o(l){y(e.$$.fragment,l),i=!1},d(l){b(e,l)}}}function ta(w){let e,i;return e=new g({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwVG9yY2hBb0NvbmZpZyUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQWZyb20lMjB0b3JjaGFvLnF1YW50aXphdGlvbiUyMGltcG9ydCUyMEdlbWxpdGVVSW50WFdlaWdodE9ubHlDb25maWclMkMlMjBJbnQ0V2VpZ2h0T25seUNvbmZpZyUwQSUwQSUyMyUyMEZvciUyMGJhdGNoJTIwc2l6ZSUyME4lMkMlMjB3ZSUyMHJlY29tbWVuZCUyMGdlbWxpdGUlMkMlMjB3aGljaCUyMG1heSUyMHJlcXVpcmUlMjBhdXRvdHVuaW5nJTBBJTIzJTIwZGVmYXVsdCUyMGlzJTIwNCUyMGJpdCUyQyUyMDglMjBiaXQlMjBpcyUyMGFsc28lMjBzdXBwb3J0ZWQlMjBieSUyMHBhc3NpbmclMjAlNjBiaXRfd2lkdGglM0Q4JTYwJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwR2VtbGl0ZVVJbnRYV2VpZ2h0T25seUNvbmZpZyhncm91cF9zaXplJTNEMTI4KSUwQSUwQSUyMyUyMEZvciUyMGJhdGNoJTIwc2l6ZSUyMDElMkMlMjB3ZSUyMGFsc28lMjBoYXZlJTIwY3VzdG9tJTIwdGlueWdlbW0lMjBrZXJuZWwlMjB0aGF0J3MlMjBvbmx5JTIwb3B0aW1pemVkJTIwZm9yJTIwdGhpcyUwQSUyMyUyMFdlJTIwY2FuJTIwc2V0JTIwJTYwdXNlX2hxcSU2MCUyMHRvJTIwJTYwVHJ1ZSU2MCUyMGZvciUyMGJldHRlciUyMGFjY3VyYWN5JTBBJTIzJTIwcXVhbnRfY29uZmlnJTIwJTNEJTIwSW50NFdlaWdodE9ubHlDb25maWcoZ3JvdXBfc2l6ZSUzRDEyOCUyQyUyMHVzZV9ocXElM0RUcnVlKSUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBUb3JjaEFvQ29uZmlnKHF1YW50X3R5cGUlM0RxdWFudF9jb25maWcpJTBBJTBBJTIzJTIwTG9hZCUyMGFuZCUyMHF1YW50aXplJTIwdGhlJTIwbW9kZWwlMEFxdWFudGl6ZWRfbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIybWV0YS1sbGFtYSUyRkxsYW1hLTMuMS04Qi1JbnN0cnVjdCUyMiUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMEEpJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIybWV0YS1sbGFtYSUyRkxsYW1hLTMuMS04Qi1JbnN0cnVjdCUyMiklMEFpbnB1dF90ZXh0JTIwJTNEJTIwJTIyV2hhdCUyMGFyZSUyMHdlJTIwaGF2aW5nJTIwZm9yJTIwZGlubmVyJTNGJTIyJTBBaW5wdXRfaWRzJTIwJTNEJTIwdG9rZW5pemVyKGlucHV0X3RleHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byglMjJjdWRhJTIyKSUwQSUwQSUyMyUyMGF1dG8tY29tcGlsZSUyMHRoZSUyMHF1YW50aXplZCUyMG1vZGVsJTIwd2l0aCUyMCU2MGNhY2hlX2ltcGxlbWVudGF0aW9uJTNEJTIyc3RhdGljJTIyJTYwJTIwdG8lMjBnZXQlMjBzcGVlZCUyMHVwJTBBb3V0cHV0JTIwJTNEJTIwcXVhbnRpemVkX21vZGVsLmdlbmVyYXRlKCoqaW5wdXRfaWRzJTJDJTIwbWF4X25ld190b2tlbnMlM0QxMCUyQyUyMGNhY2hlX2ltcGxlbWVudGF0aW9uJTNEJTIyc3RhdGljJTIyKSUwQXByaW50KHRva2VuaXplci5kZWNvZGUob3V0cHV0JTVCMCU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TorchAoConfig, AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">from</span> torchao.quantization <span class="hljs-keyword">import</span> GemliteUIntXWeightOnlyConfig, Int4WeightOnlyConfig

<span class="hljs-comment"># For batch size N, we recommend gemlite, which may require autotuning</span>
<span class="hljs-comment"># default is 4 bit, 8 bit is also supported by passing \`bit_width=8\`</span>
quant_config = GemliteUIntXWeightOnlyConfig(group_size=<span class="hljs-number">128</span>)

<span class="hljs-comment"># For batch size 1, we also have custom tinygemm kernel that&#x27;s only optimized for this</span>
<span class="hljs-comment"># We can set \`use_hqq\` to \`True\` for better accuracy</span>
<span class="hljs-comment"># quant_config = Int4WeightOnlyConfig(group_size=128, use_hqq=True)</span>

quantization_config = TorchAoConfig(quant_type=quant_config)

<span class="hljs-comment"># Load and quantize the model</span>
quantized_model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-3.1-8B-Instruct&quot;</span>,
    torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
    quantization_config=quantization_config
)

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-3.1-8B-Instruct&quot;</span>)
input_text = <span class="hljs-string">&quot;What are we having for dinner?&quot;</span>
input_ids = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-comment"># auto-compile the quantized model with \`cache_implementation=&quot;static&quot;\` to get speed up</span>
output = quantized_model.generate(**input_ids, max_new_tokens=<span class="hljs-number">10</span>, cache_implementation=<span class="hljs-string">&quot;static&quot;</span>)
<span class="hljs-built_in">print</span>(tokenizer.decode(output[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`,wrap:!1}}),{c(){m(e.$$.fragment)},l(l){d(e.$$.fragment,l)},m(l,c){M(e,l,c),i=!0},p:I,i(l){i||(h(e.$$.fragment,l),i=!0)},o(l){y(e.$$.fragment,l),i=!1},d(l){b(e,l)}}}function la(w){let e,i,l,c;return e=new R({props:{id:"examples-A100-GPU",option:"int8-dynamic-and-weight-only",$$slots:{default:[Oe]},$$scope:{ctx:w}}}),l=new R({props:{id:"examples-A100-GPU",option:"int4-weight-only",$$slots:{default:[ta]},$$scope:{ctx:w}}}),{c(){m(e.$$.fragment),i=p(),m(l.$$.fragment)},l(a){d(e.$$.fragment,a),i=r(a),d(l.$$.fragment,a)},m(a,u){M(e,a,u),o(a,i,u),M(l,a,u),c=!0},p(a,u){const U={};u&2&&(U.$$scope={dirty:u,ctx:a}),e.$set(U);const Z={};u&2&&(Z.$$scope={dirty:u,ctx:a}),l.$set(Z)},i(a){c||(h(e.$$.fragment,a),h(l.$$.fragment,a),c=!0)},o(a){y(e.$$.fragment,a),y(l.$$.fragment,a),c=!1},d(a){a&&s(i),b(e,a),b(l,a)}}}function ea(w){let e,i;return e=new g({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwVG9yY2hBb0NvbmZpZyUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQWZyb20lMjB0b3JjaGFvLnF1YW50aXphdGlvbiUyMGltcG9ydCUyMEludDhEeW5hbWljQWN0aXZhdGlvbkludDhXZWlnaHRDb25maWclMkMlMjBJbnQ4V2VpZ2h0T25seUNvbmZpZyUwQSUwQXF1YW50X2NvbmZpZyUyMCUzRCUyMEludDhEeW5hbWljQWN0aXZhdGlvbkludDhXZWlnaHRDb25maWcoKSUwQSUyMyUyMHF1YW50X2NvbmZpZyUyMCUzRCUyMEludDhXZWlnaHRPbmx5Q29uZmlnKCklMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwVG9yY2hBb0NvbmZpZyhxdWFudF90eXBlJTNEcXVhbnRfY29uZmlnKSUwQSUwQSUyMyUyMExvYWQlMjBhbmQlMjBxdWFudGl6ZSUyMHRoZSUyMG1vZGVsJTBBcXVhbnRpemVkX21vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMm1ldGEtbGxhbWElMkZMbGFtYS0zLjEtOEItSW5zdHJ1Y3QlMjIlMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjBkZXZpY2VfbWFwJTNEJTIyY3B1JTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMEEpJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIybWV0YS1sbGFtYSUyRkxsYW1hLTMuMS04Qi1JbnN0cnVjdCUyMiklMEFpbnB1dF90ZXh0JTIwJTNEJTIwJTIyV2hhdCUyMGFyZSUyMHdlJTIwaGF2aW5nJTIwZm9yJTIwZGlubmVyJTNGJTIyJTBBaW5wdXRfaWRzJTIwJTNEJTIwdG9rZW5pemVyKGlucHV0X3RleHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQSUwQSUyMyUyMGF1dG8tY29tcGlsZSUyMHRoZSUyMHF1YW50aXplZCUyMG1vZGVsJTIwd2l0aCUyMCU2MGNhY2hlX2ltcGxlbWVudGF0aW9uJTNEJTIyc3RhdGljJTIyJTYwJTIwdG8lMjBnZXQlMjBzcGVlZCUyMHVwJTBBb3V0cHV0JTIwJTNEJTIwcXVhbnRpemVkX21vZGVsLmdlbmVyYXRlKCoqaW5wdXRfaWRzJTJDJTIwbWF4X25ld190b2tlbnMlM0QxMCUyQyUyMGNhY2hlX2ltcGxlbWVudGF0aW9uJTNEJTIyc3RhdGljJTIyKSUwQXByaW50KHRva2VuaXplci5kZWNvZGUob3V0cHV0JTVCMCU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TorchAoConfig, AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">from</span> torchao.quantization <span class="hljs-keyword">import</span> Int8DynamicActivationInt8WeightConfig, Int8WeightOnlyConfig

quant_config = Int8DynamicActivationInt8WeightConfig()
<span class="hljs-comment"># quant_config = Int8WeightOnlyConfig()</span>
quantization_config = TorchAoConfig(quant_type=quant_config)

<span class="hljs-comment"># Load and quantize the model</span>
quantized_model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-3.1-8B-Instruct&quot;</span>,
    torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>,
    device_map=<span class="hljs-string">&quot;cpu&quot;</span>,
    quantization_config=quantization_config
)

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-3.1-8B-Instruct&quot;</span>)
input_text = <span class="hljs-string">&quot;What are we having for dinner?&quot;</span>
input_ids = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-comment"># auto-compile the quantized model with \`cache_implementation=&quot;static&quot;\` to get speed up</span>
output = quantized_model.generate(**input_ids, max_new_tokens=<span class="hljs-number">10</span>, cache_implementation=<span class="hljs-string">&quot;static&quot;</span>)
<span class="hljs-built_in">print</span>(tokenizer.decode(output[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`,wrap:!1}}),{c(){m(e.$$.fragment)},l(l){d(e.$$.fragment,l)},m(l,c){M(e,l,c),i=!0},p:I,i(l){i||(h(e.$$.fragment,l),i=!0)},o(l){y(e.$$.fragment,l),i=!1},d(l){b(e,l)}}}function aa(w){let e,i="Run the quantized model on a CPU by changing <code>device_map</code> to <code>&quot;cpu&quot;</code> and <code>layout</code> to <code>Int4CPULayout()</code>.";return{c(){e=J("p"),e.innerHTML=i},l(l){e=f(l,"P",{"data-svelte-h":!0}),T(e)!=="svelte-10s5k0"&&(e.innerHTML=i)},m(l,c){o(l,e,c)},p:I,d(l){l&&s(e)}}}function na(w){let e,i,l,c;return e=new Kl({props:{warning:!1,$$slots:{default:[aa]},$$scope:{ctx:w}}}),l=new g({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwVG9yY2hBb0NvbmZpZyUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQWZyb20lMjB0b3JjaGFvLnF1YW50aXphdGlvbiUyMGltcG9ydCUyMEludDRXZWlnaHRPbmx5Q29uZmlnJTBBZnJvbSUyMHRvcmNoYW8uZHR5cGVzJTIwaW1wb3J0JTIwSW50NENQVUxheW91dCUwQSUwQXF1YW50X2NvbmZpZyUyMCUzRCUyMEludDRXZWlnaHRPbmx5Q29uZmlnKGdyb3VwX3NpemUlM0QxMjglMkMlMjBsYXlvdXQlM0RJbnQ0Q1BVTGF5b3V0KCkpJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMFRvcmNoQW9Db25maWcocXVhbnRfdHlwZSUzRHF1YW50X2NvbmZpZyklMEElMEElMjMlMjBMb2FkJTIwYW5kJTIwcXVhbnRpemUlMjB0aGUlMjBtb2RlbCUwQXF1YW50aXplZF9tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJtZXRhLWxsYW1hJTJGTGxhbWEtMy4xLThCLUluc3RydWN0JTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0QlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRCUyMmNwdSUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudGl6YXRpb25fY29uZmlnJTBBKSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMm1ldGEtbGxhbWElMkZMbGFtYS0zLjEtOEItSW5zdHJ1Y3QlMjIpJTBBaW5wdXRfdGV4dCUyMCUzRCUyMCUyMldoYXQlMjBhcmUlMjB3ZSUyMGhhdmluZyUyMGZvciUyMGRpbm5lciUzRiUyMiUwQWlucHV0X2lkcyUyMCUzRCUyMHRva2VuaXplcihpbnB1dF90ZXh0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEElMEElMjMlMjBhdXRvLWNvbXBpbGUlMjB0aGUlMjBxdWFudGl6ZWQlMjBtb2RlbCUyMHdpdGglMjAlNjBjYWNoZV9pbXBsZW1lbnRhdGlvbiUzRCUyMnN0YXRpYyUyMiU2MCUyMHRvJTIwZ2V0JTIwc3BlZWQlMjB1cCUwQW91dHB1dCUyMCUzRCUyMHF1YW50aXplZF9tb2RlbC5nZW5lcmF0ZSgqKmlucHV0X2lkcyUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMTAlMkMlMjBjYWNoZV9pbXBsZW1lbnRhdGlvbiUzRCUyMnN0YXRpYyUyMiklMEFwcmludCh0b2tlbml6ZXIuZGVjb2RlKG91dHB1dCU1QjAlNUQlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSkp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TorchAoConfig, AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">from</span> torchao.quantization <span class="hljs-keyword">import</span> Int4WeightOnlyConfig
<span class="hljs-keyword">from</span> torchao.dtypes <span class="hljs-keyword">import</span> Int4CPULayout

quant_config = Int4WeightOnlyConfig(group_size=<span class="hljs-number">128</span>, layout=Int4CPULayout())
quantization_config = TorchAoConfig(quant_type=quant_config)

<span class="hljs-comment"># Load and quantize the model</span>
quantized_model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-3.1-8B-Instruct&quot;</span>,
    torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>,
    device_map=<span class="hljs-string">&quot;cpu&quot;</span>,
    quantization_config=quantization_config
)

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-3.1-8B-Instruct&quot;</span>)
input_text = <span class="hljs-string">&quot;What are we having for dinner?&quot;</span>
input_ids = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

<span class="hljs-comment"># auto-compile the quantized model with \`cache_implementation=&quot;static&quot;\` to get speed up</span>
output = quantized_model.generate(**input_ids, max_new_tokens=<span class="hljs-number">10</span>, cache_implementation=<span class="hljs-string">&quot;static&quot;</span>)
<span class="hljs-built_in">print</span>(tokenizer.decode(output[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`,wrap:!1}}),{c(){m(e.$$.fragment),i=p(),m(l.$$.fragment)},l(a){d(e.$$.fragment,a),i=r(a),d(l.$$.fragment,a)},m(a,u){M(e,a,u),o(a,i,u),M(l,a,u),c=!0},p(a,u){const U={};u&2&&(U.$$scope={dirty:u,ctx:a}),e.$set(U)},i(a){c||(h(e.$$.fragment,a),h(l.$$.fragment,a),c=!0)},o(a){y(e.$$.fragment,a),y(l.$$.fragment,a),c=!1},d(a){a&&s(i),b(e,a),b(l,a)}}}function sa(w){let e,i,l,c;return e=new R({props:{id:"examples-CPU",option:"int8-dynamic-and-weight-only",$$slots:{default:[ea]},$$scope:{ctx:w}}}),l=new R({props:{id:"examples-CPU",option:"int4-weight-only",$$slots:{default:[na]},$$scope:{ctx:w}}}),{c(){m(e.$$.fragment),i=p(),m(l.$$.fragment)},l(a){d(e.$$.fragment,a),i=r(a),d(l.$$.fragment,a)},m(a,u){M(e,a,u),o(a,i,u),M(l,a,u),c=!0},p(a,u){const U={};u&2&&(U.$$scope={dirty:u,ctx:a}),e.$set(U);const Z={};u&2&&(Z.$$scope={dirty:u,ctx:a}),l.$set(Z)},i(a){c||(h(e.$$.fragment,a),h(l.$$.fragment,a),c=!0)},o(a){y(e.$$.fragment,a),y(l.$$.fragment,a),c=!1},d(a){a&&s(i),b(e,a),b(l,a)}}}function oa(w){let e,i;return e=new g({props:{code:"JTIzJTIwZG9uJ3QlMjBzZXJpYWxpemUlMjBtb2RlbCUyMHdpdGglMjBTYWZldGVuc29ycyUwQW91dHB1dF9kaXIlMjAlM0QlMjAlMjJsbGFtYTMtOGItaW50NHdvLTEyOCUyMiUwQXF1YW50aXplZF9tb2RlbC5zYXZlX3ByZXRyYWluZWQoJTIybGxhbWEzLThiLWludDR3by0xMjglMjIlMkMlMjBzYWZlX3NlcmlhbGl6YXRpb24lM0RGYWxzZSk=",highlighted:`<span class="hljs-comment"># don&#x27;t serialize model with Safetensors</span>
output_dir = <span class="hljs-string">&quot;llama3-8b-int4wo-128&quot;</span>
quantized_model.save_pretrained(<span class="hljs-string">&quot;llama3-8b-int4wo-128&quot;</span>, safe_serialization=<span class="hljs-literal">False</span>)`,wrap:!1}}),{c(){m(e.$$.fragment)},l(l){d(e.$$.fragment,l)},m(l,c){M(e,l,c),i=!0},p:I,i(l){i||(h(e.$$.fragment,l),i=!0)},o(l){y(e.$$.fragment,l),i=!1},d(l){b(e,l)}}}function ia(w){let e,i;return e=new g({props:{code:"JTIzJTIwZG9uJ3QlMjBzZXJpYWxpemUlMjBtb2RlbCUyMHdpdGglMjBTYWZldGVuc29ycyUwQVVTRVJfSUQlMjAlM0QlMjAlMjJ5b3VyX2h1Z2dpbmdmYWNlX3VzZXJfaWQlMjIlMEFSRVBPX0lEJTIwJTNEJTIwJTIybGxhbWEzLThiLWludDR3by0xMjglMjIlMEFxdWFudGl6ZWRfbW9kZWwucHVzaF90b19odWIoZiUyMiU3QlVTRVJfSUQlN0QlMkZsbGFtYTMtOGItaW50NHdvLTEyOCUyMiUyQyUyMHNhZmVfc2VyaWFsaXphdGlvbiUzREZhbHNlKSUwQXRva2VuaXplci5wdXNoX3RvX2h1YihmJTIyJTdCVVNFUl9JRCU3RCUyRmxsYW1hMy04Yi1pbnQ0d28tMTI4JTIyKQ==",highlighted:`<span class="hljs-comment"># don&#x27;t serialize model with Safetensors</span>
USER_ID = <span class="hljs-string">&quot;your_huggingface_user_id&quot;</span>
REPO_ID = <span class="hljs-string">&quot;llama3-8b-int4wo-128&quot;</span>
quantized_model.push_to_hub(<span class="hljs-string">f&quot;<span class="hljs-subst">{USER_ID}</span>/llama3-8b-int4wo-128&quot;</span>, safe_serialization=<span class="hljs-literal">False</span>)
tokenizer.push_to_hub(<span class="hljs-string">f&quot;<span class="hljs-subst">{USER_ID}</span>/llama3-8b-int4wo-128&quot;</span>)`,wrap:!1}}),{c(){m(e.$$.fragment)},l(l){d(e.$$.fragment,l)},m(l,c){M(e,l,c),i=!0},p:I,i(l){i||(h(e.$$.fragment,l),i=!0)},o(l){y(e.$$.fragment,l),i=!1},d(l){b(e,l)}}}function pa(w){let e,i,l,c;return e=new R({props:{id:"serialization-examples",option:"save-locally",$$slots:{default:[oa]},$$scope:{ctx:w}}}),l=new R({props:{id:"serialization-examples",option:"push-to-huggingface-hub",$$slots:{default:[ia]},$$scope:{ctx:w}}}),{c(){m(e.$$.fragment),i=p(),m(l.$$.fragment)},l(a){d(e.$$.fragment,a),i=r(a),d(l.$$.fragment,a)},m(a,u){M(e,a,u),o(a,i,u),M(l,a,u),c=!0},p(a,u){const U={};u&2&&(U.$$scope={dirty:u,ctx:a}),e.$set(U);const Z={};u&2&&(Z.$$scope={dirty:u,ctx:a}),l.$set(Z)},i(a){c||(h(e.$$.fragment,a),h(l.$$.fragment,a),c=!0)},o(a){y(e.$$.fragment,a),y(l.$$.fragment,a),c=!1},d(a){a&&s(i),b(e,a),b(l,a)}}}function ra(w){let e,i="For best performance, you can use recommended settings by calling <code>torchao.quantization.utils.recommended_inductor_config_setter()</code>";return{c(){e=J("p"),e.innerHTML=i},l(l){e=f(l,"P",{"data-svelte-h":!0}),T(e)!=="svelte-dcid7j"&&(e.innerHTML=i)},m(l,c){o(l,e,c)},p:I,d(l){l&&s(e)}}}function ua(w){let e,i,l,c,a,u,U,Z='<a href="https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/en/quantization/torchao.ipynb" rel="nofollow"><img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab: Torchao Demo"/></a>',Qt,F,Ol='<a href="https://github.com/pytorch/ao" rel="nofollow">torchao</a> is a PyTorch architecture optimization library with support for custom high performance data types, quantization, and sparsity. It is composable with native PyTorch features such as <a href="https://pytorch.org/tutorials/intermediate/torch_compile_tutorial.html" rel="nofollow">torch.compile</a> for even faster inference and training.',Et,k,te="See the table below for additional torchao features.",Yt,v,le='<thead><tr><th>Feature</th> <th>Description</th></tr></thead> <tbody><tr><td><strong>Quantization Aware Training (QAT)</strong></td> <td>Train quantized models with minimal accuracy loss (see <a href="https://github.com/pytorch/ao/blob/main/torchao/quantization/qat/README.md" rel="nofollow">QAT README</a>)</td></tr> <tr><td><strong>Float8 Training</strong></td> <td>High-throughput training with float8 formats (see <a href="https://github.com/pytorch/torchtitan/blob/main/docs/float8.md" rel="nofollow">torchtitan</a> and <a href="https://huggingface.co/docs/accelerate/usage_guides/low_precision_training#configuring-torchao" rel="nofollow">Accelerate</a> docs)</td></tr> <tr><td><strong>Sparsity Support</strong></td> <td>Semi-structured (2:4) sparsity for faster inference (see <a href="https://pytorch.org/blog/accelerating-neural-network-training/" rel="nofollow">Accelerating Neural Network Training with Semi-Structured (2:4) Sparsity</a> blog post)</td></tr> <tr><td><strong>Optimizer Quantization</strong></td> <td>Reduce optimizer state memory with 4 and 8-bit variants of Adam</td></tr> <tr><td><strong>KV Cache Quantization</strong></td> <td>Enables long context inference with lower memory (see <a href="https://github.com/pytorch/ao/blob/main/torchao/_models/llama/README.md" rel="nofollow">KV Cache Quantization</a>)</td></tr> <tr><td><strong>Custom Kernels Support</strong></td> <td>use your own <code>torch.compile</code> compatible ops</td></tr> <tr><td><strong>FSDP2</strong></td> <td>Composable with FSDP2 for training</td></tr></tbody>',Nt,_,Ht,Q,ee='torchao supports the <a href="https://github.com/pytorch/ao/blob/main/torchao/quantization/README.md" rel="nofollow">quantization techniques</a> below.',xt,E,ae="<li>A16W8 Float8 Dynamic Quantization</li> <li>A16W8 Float8 WeightOnly Quantization</li> <li>A8W8 Int8 Dynamic Quantization</li> <li>A16W8 Int8 Weight Only Quantization</li> <li>A16W4 Int4 Weight Only Quantization</li> <li>Autoquantization</li>",St,Y,ne="torchao also supports module level configuration by specifying a dictionary from fully qualified name of module and its corresponding quantization config. This allows skip quantizing certain layers and using different quantization config for different modules.",At,N,se="Check the table below to see if your hardware is compatible.",Lt,H,oe="<thead><tr><th>Component</th> <th>Compatibility</th></tr></thead> <tbody><tr><td>CUDA Versions</td> <td>✅ cu118, cu126, cu128</td></tr> <tr><td>CPU</td> <td>✅ change <code>device_map=&quot;cpu&quot;</code> (see examples below)</td></tr></tbody>",Dt,x,ie="Install torchao from PyPi or the PyTorch index with the following commands.",Pt,B,Kt,S,pe='If your torcha version is below 0.10.0, you need to upgrade it, please refer to the <a href="#deprecation-notice">deprecation notice</a> for more details.',Ot,A,tl,L,re="TorchAO provides a variety of quantization configurations. Each configuration can be further customized with parameters such as <code>group_size</code>, <code>scheme</code>, and <code>layout</code> to optimize for specific hardware and model architectures.",ll,D,ue='For a complete list of available configurations, see the <a href="https://github.com/pytorch/ao/blob/main/torchao/quantization/quant_api.py" rel="nofollow">quantization API documentation</a>.',el,P,ce="You can manually choose the quantization types and settings or automatically select the quantization types.",al,K,me='Create a <a href="/docs/transformers/main/en/main_classes/quantization#transformers.TorchAoConfig">TorchAoConfig</a> and specify the quantization type and <code>group_size</code> of the weights to quantize (for int8 weight only and int4 weight only). Set the <code>cache_implementation</code> to <code>&quot;static&quot;</code> to automatically <a href="https://pytorch.org/tutorials/intermediate/torch_compile_tutorial.html" rel="nofollow">torch.compile</a> the forward method.',nl,O,de="We’ll show examples for recommended quantization methods based on hardwares, e.g. A100 GPU, H100 GPU, CPU.",sl,tt,ol,V,il,lt,pl,C,rl,et,ul,$,cl,at,ml,nt,dl,st,Me="With <code>AOPerModuleConfig</code> we can specify a default configuration for all layers while skipping quantization for certain layers.",Ml,ot,hl,it,yl,pt,bl,rt,Jl,ut,he='If you want to automatically choose a quantization type for quantizable layers (<code>nn.Linear</code>) you can use the <a href="https://pytorch.org/ao/stable/generated/torchao.quantization.autoquant.html#torchao.quantization.autoquant" rel="nofollow">autoquant</a> API.',fl,ct,ye="The <code>autoquant</code> API automatically chooses a quantization type by micro-benchmarking on input type and shape and compiling a single linear layer.",Tl,mt,be="Note: autoquant is for GPU only right now.",wl,dt,Je='Create a <a href="/docs/transformers/main/en/main_classes/quantization#transformers.TorchAoConfig">TorchAoConfig</a> and set to <code>&quot;autoquant&quot;</code>. Set the <code>cache_implementation</code> to <code>&quot;static&quot;</code> to automatically <a href="https://pytorch.org/tutorials/intermediate/torch_compile_tutorial.html" rel="nofollow">torch.compile</a> the forward method. Finally, call <code>finalize_autoquant</code> on the quantized model to finalize the quantization and log the input shapes.',Ul,Mt,jl,ht,Wl,yt,fe='torchao implements <a href="https://pytorch.org/docs/stable/notes/extending.html#subclassing-torch-tensor" rel="nofollow">torch.Tensor subclasses</a> for maximum flexibility in supporting new quantized torch.Tensor formats. <a href="https://huggingface.co/docs/safetensors/en/index" rel="nofollow">Safetensors</a> serialization and deserialization does not work with torchao.',Zl,bt,Te='To avoid arbitrary user code execution, torchao sets <code>weights_only=True</code> in <a href="https://pytorch.org/docs/stable/generated/torch.load.html" rel="nofollow">torch.load</a> to ensure only tensors are loaded. Any known user functions can be whitelisted with <a href="https://pytorch.org/docs/stable/notes/serialization.html#torch.serialization.add_safe_globals" rel="nofollow">add_safe_globals</a>.',gl,z,Gl,Jt,Xl,ft,we="Loading a quantized model depends on the quantization scheme. For quantization schemes, like int8 and float8, you can quantize the model on any device and also load it on any device. The example below demonstrates quantizing a model on the CPU and then loading it on CUDA.",Il,Tt,Rl,wt,Ue="For int4, the model can only be loaded on the same device it was quantized on because the layout is specific to the device. The example below demonstrates quantizing and loading a model on the CPU.",_l,Ut,Bl,jt,Vl,j,Bt,je="Starting with version 0.10.0, the string-based API for quantization configuration (e.g., <code>TorchAoConfig(&quot;int4_weight_only&quot;, group_size=128)</code>) is <strong>deprecated</strong> and will be removed in a future release.",Nl,Vt,We="Please use the new <code>AOBaseConfig</code>-based approach instead:",Hl,Wt,xl,Ct,Ze="The new API offers greater flexibility, better type safety, and access to the full range of features available in torchao.",Sl,$t,ge='<a href="#migration-guide">Migration Guide</a>',Al,zt,Ge="Here’s how to migrate from common string identifiers to their <code>AOBaseConfig</code> equivalents:",Ll,qt,Xe="<thead><tr><th>Old String API</th> <th>New <code>AOBaseConfig</code> API</th></tr></thead> <tbody><tr><td><code>&quot;int4_weight_only&quot;</code></td> <td><code>Int4WeightOnlyConfig()</code></td></tr> <tr><td><code>&quot;int8_weight_only&quot;</code></td> <td><code>Int8WeightOnlyConfig()</code></td></tr> <tr><td><code>&quot;int8_dynamic_activation_int8_weight&quot;</code></td> <td><code>Int8DynamicActivationInt8WeightConfig()</code></td></tr></tbody>",Dl,Ft,Ie="All configuration objects accept parameters for customization (e.g., <code>group_size</code>, <code>scheme</code>, <code>layout</code>).",Cl,Zt,$l,gt,Re='For a better sense of expected performance, view the <a href="https://github.com/pytorch/ao/tree/main/torchao/quantization#benchmarks" rel="nofollow">benchmarks</a> for various models with CUDA and XPU backends. You can also run the code below to benchmark a model yourself.',zl,Gt,ql,q,Fl,Xt,_e='Refer to <a href="https://github.com/pytorch/ao/tree/main/torchao/quantization#other-available-quantization-techniques" rel="nofollow">Other Available Quantization Techniques</a> for more examples and documentation.',kl,It,vl,Rt,Be='If you encounter any issues with the Transformers integration, please open an issue on the <a href="https://github.com/huggingface/transformers/issues" rel="nofollow">Transformers</a> repository. For issues directly related to torchao, please open an issue on the <a href="https://github.com/pytorch/ao/issues" rel="nofollow">torchao</a> repository.',Ql,_t,El,kt,Yl;return a=new X({props:{title:"torchao",local:"torchao",headingTag:"h1"}}),_=new Kl({props:{warning:!1,$$slots:{default:[xe]},$$scope:{ctx:w}}}),B=new vt({props:{id:"install torchao",options:["PyPi","PyTorch Index"],$$slots:{default:[Le]},$$scope:{ctx:w}}}),A=new X({props:{title:"Quantization examples",local:"quantization-examples",headingTag:"h2"}}),tt=new X({props:{title:"H100 GPU",local:"h100-gpu",headingTag:"h3"}}),V=new vt({props:{id:"examples-H100-GPU",options:["float8-dynamic-and-weight-only","int4-weight-only"],$$slots:{default:[Ke]},$$scope:{ctx:w}}}),lt=new X({props:{title:"A100 GPU",local:"a100-gpu",headingTag:"h3"}}),C=new vt({props:{id:"examples-A100-GPU",options:["int8-dynamic-and-weight-only","int4-weight-only"],$$slots:{default:[la]},$$scope:{ctx:w}}}),et=new X({props:{title:"CPU",local:"cpu",headingTag:"h3"}}),$=new vt({props:{id:"examples-CPU",options:["int8-dynamic-and-weight-only","int4-weight-only"],$$slots:{default:[sa]},$$scope:{ctx:w}}}),at=new X({props:{title:"Per Module Quantization",local:"per-module-quantization",headingTag:"h3"}}),nt=new X({props:{title:"1. Skip quantization for certain layers",local:"1-skip-quantization-for-certain-layers",headingTag:"h4"}}),ot=new g({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBBdXRvVG9rZW5pemVyJTJDJTIwVG9yY2hBb0NvbmZpZyUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIybWV0YS1sbGFtYSUyRkxsYW1hLTMuMS04Qi1JbnN0cnVjdCUyMiUwQSUwQWZyb20lMjB0b3JjaGFvLnF1YW50aXphdGlvbiUyMGltcG9ydCUyMEludDRXZWlnaHRPbmx5Q29uZmlnJTJDJTIwQU9QZXJNb2R1bGVDb25maWclMEFjb25maWclMjAlM0QlMjBJbnQ0V2VpZ2h0T25seUNvbmZpZyhncm91cF9zaXplJTNEMTI4KSUwQSUwQSUyMyUyMHNldCUyMGRlZmF1bHQlMjB0byUyMGludDQlMjAoZm9yJTIwbGluZWFycyklMkMlMjBhbmQlMjBza2lwJTIwcXVhbnRpemluZyUyMCU2MG1vZGVsLmxheWVycy4wLnNlbGZfYXR0bi5xX3Byb2olNjAlMEFxdWFudF9jb25maWclMjAlM0QlMjBBT1Blck1vZHVsZUNvbmZpZyglN0IlMjJfZGVmYXVsdCUyMiUzQSUyMGNvbmZpZyUyQyUyMCUyMm1vZGVsLmxheWVycy4wLnNlbGZfYXR0bi5xX3Byb2olMjIlM0ElMjBOb25lJTdEKSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBUb3JjaEFvQ29uZmlnKHF1YW50X3R5cGUlM0RxdWFudF9jb25maWcpJTBBcXVhbnRpemVkX21vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWcpJTBBJTIzJTIwbG1faGVhZCUyMGlzJTIwbm90JTIwcXVhbnRpemVkJTIwYW5kJTIwbW9kZWwubGF5ZXJzLjAuc2VsZl9hdHRuLnFfcHJvaiUyMGlzJTIwbm90JTIwcXVhbnRpemVkJTBBcHJpbnQoJTIycXVhbnRpemVkJTIwbW9kZWwlM0ElMjIlMkMlMjBxdWFudGl6ZWRfbW9kZWwpJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQpJTBBJTBBJTIzJTIwTWFudWFsJTIwVGVzdGluZyUwQXByb21wdCUyMCUzRCUyMCUyMkhleSUyQyUyMGFyZSUyMHlvdSUyMGNvbnNjaW91cyUzRiUyMENhbiUyMHlvdSUyMHRhbGslMjB0byUyMG1lJTNGJTIyJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKHByb21wdCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKCUyMmN1ZGElMjIpJTBBZ2VuZXJhdGVkX2lkcyUyMCUzRCUyMHF1YW50aXplZF9tb2RlbC5nZW5lcmF0ZSgqKmlucHV0cyUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMTI4KSUwQW91dHB1dF90ZXh0JTIwJTNEJTIwdG9rZW5pemVyLmJhdGNoX2RlY29kZSglMEElMjAlMjAlMjAlMjBnZW5lcmF0ZWRfaWRzJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUlMkMlMjBjbGVhbl91cF90b2tlbml6YXRpb25fc3BhY2VzJTNERmFsc2UlMEEpJTBBcHJpbnQob3V0cHV0X3RleHQp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, TorchAoConfig

model_id = <span class="hljs-string">&quot;meta-llama/Llama-3.1-8B-Instruct&quot;</span>

<span class="hljs-keyword">from</span> torchao.quantization <span class="hljs-keyword">import</span> Int4WeightOnlyConfig, AOPerModuleConfig
config = Int4WeightOnlyConfig(group_size=<span class="hljs-number">128</span>)

<span class="hljs-comment"># set default to int4 (for linears), and skip quantizing \`model.layers.0.self_attn.q_proj\`</span>
quant_config = AOPerModuleConfig({<span class="hljs-string">&quot;_default&quot;</span>: config, <span class="hljs-string">&quot;model.layers.0.self_attn.q_proj&quot;</span>: <span class="hljs-literal">None</span>})
quantization_config = TorchAoConfig(quant_type=quant_config)
quantized_model = AutoModelForCausalLM.from_pretrained(model_id, device_map=<span class="hljs-string">&quot;auto&quot;</span>, torch_dtype=torch.bfloat16, quantization_config=quantization_config)
<span class="hljs-comment"># lm_head is not quantized and model.layers.0.self_attn.q_proj is not quantized</span>
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;quantized model:&quot;</span>, quantized_model)
tokenizer = AutoTokenizer.from_pretrained(model_id)

<span class="hljs-comment"># Manual Testing</span>
prompt = <span class="hljs-string">&quot;Hey, are you conscious? Can you talk to me?&quot;</span>
inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)
generated_ids = quantized_model.generate(**inputs, max_new_tokens=<span class="hljs-number">128</span>)
output_text = tokenizer.batch_decode(
    generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>, clean_up_tokenization_spaces=<span class="hljs-literal">False</span>
)
<span class="hljs-built_in">print</span>(output_text)`,wrap:!1}}),it=new X({props:{title:"2. Quantizing different layers with different quantization configs",local:"2-quantizing-different-layers-with-different-quantization-configs",headingTag:"h4"}}),pt=new g({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBBdXRvVG9rZW5pemVyJTJDJTIwVG9yY2hBb0NvbmZpZyUwQSUwQW1vZGVsX2lkJTIwJTNEJTIwJTIyZmFjZWJvb2slMkZvcHQtMTI1bSUyMiUwQSUwQWZyb20lMjB0b3JjaGFvLnF1YW50aXphdGlvbiUyMGltcG9ydCUyMEludDRXZWlnaHRPbmx5Q29uZmlnJTJDJTIwQU9QZXJNb2R1bGVDb25maWclMkMlMjBJbnQ4RHluYW1pY0FjdGl2YXRpb25JbnQ0V2VpZ2h0Q29uZmlnJTJDJTIwSW50eFdlaWdodE9ubHlDb25maWclMkMlMjBQZXJBeGlzJTJDJTIwTWFwcGluZ1R5cGUlMEElMEF3ZWlnaHRfZHR5cGUlMjAlM0QlMjB0b3JjaC5pbnQ4JTBBZ3JhbnVsYXJpdHklMjAlM0QlMjBQZXJBeGlzKDApJTBBbWFwcGluZ190eXBlJTIwJTNEJTIwTWFwcGluZ1R5cGUuQVNZTU1FVFJJQyUwQWVtYmVkZGluZ19jb25maWclMjAlM0QlMjBJbnR4V2VpZ2h0T25seUNvbmZpZyglMEElMjAlMjAlMjAlMjB3ZWlnaHRfZHR5cGUlM0R3ZWlnaHRfZHR5cGUlMkMlMEElMjAlMjAlMjAlMjBncmFudWxhcml0eSUzRGdyYW51bGFyaXR5JTJDJTBBJTIwJTIwJTIwJTIwbWFwcGluZ190eXBlJTNEbWFwcGluZ190eXBlJTJDJTBBKSUwQWxpbmVhcl9jb25maWclMjAlM0QlMjBJbnQ4RHluYW1pY0FjdGl2YXRpb25JbnQ0V2VpZ2h0Q29uZmlnKGdyb3VwX3NpemUlM0QxMjgpJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwQU9QZXJNb2R1bGVDb25maWcoJTdCJTIyX2RlZmF1bHQlMjIlM0ElMjBsaW5lYXJfY29uZmlnJTJDJTIwJTIybW9kZWwuZGVjb2Rlci5lbWJlZF90b2tlbnMlMjIlM0ElMjBlbWJlZGRpbmdfY29uZmlnJTJDJTIwJTIybW9kZWwuZGVjb2Rlci5lbWJlZF9wb3NpdGlvbnMlMjIlM0ElMjBOb25lJTdEKSUwQSUyMyUyMHNldCUyMCU2MGluY2x1ZGVfZW1iZWRkaW5nJTYwJTIwdG8lMjBUcnVlJTIwaW4lMjBvcmRlciUyMHRvJTIwaW5jbHVkZSUyMGVtYmVkZGluZyUyMGluJTIwcXVhbnRpemF0aW9uJTBBJTIzJTIwd2hlbiUyMCU2MGluY2x1ZGVfZW1iZWRkaW5nJTYwJTIwaXMlMjBUcnVlJTJDJTIwd2UnbGwlMjByZW1vdmUlMjBpbnB1dCUyMGVtYmVkZGluZyUyMGZyb20lMjAlNjBtb2R1bGVzX25vdF90b19jb252ZXJ0JTYwJTIwYXMlMjB3ZWxsJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMFRvcmNoQW9Db25maWcocXVhbnRfdHlwZSUzRHF1YW50X2NvbmZpZyUyQyUyMGluY2x1ZGVfZW1iZWRkaW5nJTNEVHJ1ZSklMEFxdWFudGl6ZWRfbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQlMkMlMjBkZXZpY2VfbWFwJTNEJTIyY3B1JTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiUyQyUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudGl6YXRpb25fY29uZmlnKSUwQXByaW50KCUyMnF1YW50aXplZCUyMG1vZGVsJTNBJTIyJTJDJTIwcXVhbnRpemVkX21vZGVsKSUwQSUyMyUyMG1ha2UlMjBzdXJlJTIwZW1iZWRkaW5nJTIwaXMlMjBxdWFudGl6ZWQlMEFwcmludCglMjJlbWJlZF90b2tlbnMlMjB3ZWlnaHQlM0ElMjIlMkMlMjBxdWFudGl6ZWRfbW9kZWwubW9kZWwuZGVjb2Rlci5lbWJlZF90b2tlbnMud2VpZ2h0KSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKG1vZGVsX2lkKSUwQSUwQSUyMyUyME1hbnVhbCUyMFRlc3RpbmclMEFwcm9tcHQlMjAlM0QlMjAlMjJIZXklMkMlMjBhcmUlMjB5b3UlMjBjb25zY2lvdXMlM0YlMjBDYW4lMjB5b3UlMjB0YWxrJTIwdG8lMjBtZSUzRiUyMiUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplcihwcm9tcHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byglMjJjcHUlMjIpJTBBZ2VuZXJhdGVkX2lkcyUyMCUzRCUyMHF1YW50aXplZF9tb2RlbC5nZW5lcmF0ZSgqKmlucHV0cyUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMTI4JTJDJTIwY2FjaGVfaW1wbGVtZW50YXRpb24lM0QlMjJzdGF0aWMlMjIpJTBBb3V0cHV0X3RleHQlMjAlM0QlMjB0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKCUwQSUyMCUyMCUyMCUyMGdlbmVyYXRlZF9pZHMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSUyQyUyMGNsZWFuX3VwX3Rva2VuaXphdGlvbl9zcGFjZXMlM0RGYWxzZSUwQSklMEFwcmludChvdXRwdXRfdGV4dCk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, TorchAoConfig

model_id = <span class="hljs-string">&quot;facebook/opt-125m&quot;</span>

<span class="hljs-keyword">from</span> torchao.quantization <span class="hljs-keyword">import</span> Int4WeightOnlyConfig, AOPerModuleConfig, Int8DynamicActivationInt4WeightConfig, IntxWeightOnlyConfig, PerAxis, MappingType

weight_dtype = torch.int8
granularity = PerAxis(<span class="hljs-number">0</span>)
mapping_type = MappingType.ASYMMETRIC
embedding_config = IntxWeightOnlyConfig(
    weight_dtype=weight_dtype,
    granularity=granularity,
    mapping_type=mapping_type,
)
linear_config = Int8DynamicActivationInt4WeightConfig(group_size=<span class="hljs-number">128</span>)
quant_config = AOPerModuleConfig({<span class="hljs-string">&quot;_default&quot;</span>: linear_config, <span class="hljs-string">&quot;model.decoder.embed_tokens&quot;</span>: embedding_config, <span class="hljs-string">&quot;model.decoder.embed_positions&quot;</span>: <span class="hljs-literal">None</span>})
<span class="hljs-comment"># set \`include_embedding\` to True in order to include embedding in quantization</span>
<span class="hljs-comment"># when \`include_embedding\` is True, we&#x27;ll remove input embedding from \`modules_not_to_convert\` as well</span>
quantization_config = TorchAoConfig(quant_type=quant_config, include_embedding=<span class="hljs-literal">True</span>)
quantized_model = AutoModelForCausalLM.from_pretrained(model_id, device_map=<span class="hljs-string">&quot;cpu&quot;</span>, torch_dtype=torch.bfloat16, quantization_config=quantization_config)
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;quantized model:&quot;</span>, quantized_model)
<span class="hljs-comment"># make sure embedding is quantized</span>
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;embed_tokens weight:&quot;</span>, quantized_model.model.decoder.embed_tokens.weight)
tokenizer = AutoTokenizer.from_pretrained(model_id)

<span class="hljs-comment"># Manual Testing</span>
prompt = <span class="hljs-string">&quot;Hey, are you conscious? Can you talk to me?&quot;</span>
inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cpu&quot;</span>)
generated_ids = quantized_model.generate(**inputs, max_new_tokens=<span class="hljs-number">128</span>, cache_implementation=<span class="hljs-string">&quot;static&quot;</span>)
output_text = tokenizer.batch_decode(
    generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>, clean_up_tokenization_spaces=<span class="hljs-literal">False</span>
)
<span class="hljs-built_in">print</span>(output_text)`,wrap:!1}}),rt=new X({props:{title:"Autoquant",local:"autoquant",headingTag:"h3"}}),Mt=new g({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwVG9yY2hBb0NvbmZpZyUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQSUwQXF1YW50aXphdGlvbl9jb25maWclMjAlM0QlMjBUb3JjaEFvQ29uZmlnKCUyMmF1dG9xdWFudCUyMiUyQyUyMG1pbl9zcW5yJTNETm9uZSklMEFxdWFudGl6ZWRfbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIybWV0YS1sbGFtYSUyRkxsYW1hLTMuMS04Qi1JbnN0cnVjdCUyMiUyQyUwQSUyMCUyMCUyMCUyMHRvcmNoX2R0eXBlJTNEJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMEEpJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIybWV0YS1sbGFtYSUyRkxsYW1hLTMuMS04Qi1JbnN0cnVjdCUyMiklMEFpbnB1dF90ZXh0JTIwJTNEJTIwJTIyV2hhdCUyMGFyZSUyMHdlJTIwaGF2aW5nJTIwZm9yJTIwZGlubmVyJTNGJTIyJTBBaW5wdXRfaWRzJTIwJTNEJTIwdG9rZW5pemVyKGlucHV0X3RleHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byglMjJjdWRhJTIyKSUwQSUwQSUyMyUyMGF1dG8tY29tcGlsZSUyMHRoZSUyMHF1YW50aXplZCUyMG1vZGVsJTIwd2l0aCUyMCU2MGNhY2hlX2ltcGxlbWVudGF0aW9uJTNEJTIyc3RhdGljJTIyJTYwJTIwdG8lMjBnZXQlMjBzcGVlZCUyMHVwJTBBb3V0cHV0JTIwJTNEJTIwcXVhbnRpemVkX21vZGVsLmdlbmVyYXRlKCoqaW5wdXRfaWRzJTJDJTIwbWF4X25ld190b2tlbnMlM0QxMCUyQyUyMGNhY2hlX2ltcGxlbWVudGF0aW9uJTNEJTIyc3RhdGljJTIyKSUwQSUyMyUyMGV4cGxpY2l0bHklMjBjYWxsJTIwJTYwZmluYWxpemVfYXV0b3F1YW50JTYwJTIwKG1heSUyMGJlJTIwcmVmYWN0b3JlZCUyMGFuZCUyMHJlbW92ZWQlMjBpbiUyMHRoZSUyMGZ1dHVyZSklMEFxdWFudGl6ZWRfbW9kZWwuZmluYWxpemVfYXV0b3F1YW50KCklMEFwcmludCh0b2tlbml6ZXIuZGVjb2RlKG91dHB1dCU1QjAlNUQlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSkp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TorchAoConfig, AutoModelForCausalLM, AutoTokenizer

quantization_config = TorchAoConfig(<span class="hljs-string">&quot;autoquant&quot;</span>, min_sqnr=<span class="hljs-literal">None</span>)
quantized_model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-3.1-8B-Instruct&quot;</span>,
    torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
    quantization_config=quantization_config
)

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-3.1-8B-Instruct&quot;</span>)
input_text = <span class="hljs-string">&quot;What are we having for dinner?&quot;</span>
input_ids = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)

<span class="hljs-comment"># auto-compile the quantized model with \`cache_implementation=&quot;static&quot;\` to get speed up</span>
output = quantized_model.generate(**input_ids, max_new_tokens=<span class="hljs-number">10</span>, cache_implementation=<span class="hljs-string">&quot;static&quot;</span>)
<span class="hljs-comment"># explicitly call \`finalize_autoquant\` (may be refactored and removed in the future)</span>
quantized_model.finalize_autoquant()
<span class="hljs-built_in">print</span>(tokenizer.decode(output[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`,wrap:!1}}),ht=new X({props:{title:"Serialization",local:"serialization",headingTag:"h2"}}),z=new vt({props:{id:"serialization-examples",options:["save-locally","push-to-huggingface-hub"],$$slots:{default:[pa]},$$scope:{ctx:w}}}),Jt=new X({props:{title:"Loading quantized models",local:"loading-quantized-models",headingTag:"h2"}}),Tt=new g({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwVG9yY2hBb0NvbmZpZyUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQWZyb20lMjB0b3JjaGFvLnF1YW50aXphdGlvbiUyMGltcG9ydCUyMEludDhXZWlnaHRPbmx5Q29uZmlnJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwSW50OFdlaWdodE9ubHlDb25maWcoZ3JvdXBfc2l6ZSUzRDEyOCklMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwVG9yY2hBb0NvbmZpZyhxdWFudF90eXBlJTNEcXVhbnRfY29uZmlnKSUwQSUwQSUyMyUyMExvYWQlMjBhbmQlMjBxdWFudGl6ZSUyMHRoZSUyMG1vZGVsJTBBcXVhbnRpemVkX21vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMm1ldGEtbGxhbWElMkZMbGFtYS0zLjEtOEItSW5zdHJ1Y3QlMjIlMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjBkZXZpY2VfbWFwJTNEJTIyY3B1JTIyJTJDJTBBJTIwJTIwJTIwJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50aXphdGlvbl9jb25maWclMEEpJTBBJTIzJTIwc2F2ZSUyMHRoZSUyMHF1YW50aXplZCUyMG1vZGVsJTBBb3V0cHV0X2RpciUyMCUzRCUyMCUyMmxsYW1hLTMuMS04Yi10b3JjaGFvLWludDgtY3VkYSUyMiUwQXF1YW50aXplZF9tb2RlbC5zYXZlX3ByZXRyYWluZWQob3V0cHV0X2RpciUyQyUyMHNhZmVfc2VyaWFsaXphdGlvbiUzREZhbHNlKSUwQSUwQSUyMyUyMHJlbG9hZCUyMHRoZSUyMHF1YW50aXplZCUyMG1vZGVsJTBBcmVsb2FkZWRfbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUyQyUyMCUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTIwJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiUwQSklMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJtZXRhLWxsYW1hJTJGTGxhbWEtMy4xLThCLUluc3RydWN0JTIyKSUwQWlucHV0X3RleHQlMjAlM0QlMjAlMjJXaGF0JTIwYXJlJTIwd2UlMjBoYXZpbmclMjBmb3IlMjBkaW5uZXIlM0YlMjIlMEFpbnB1dF9pZHMlMjAlM0QlMjB0b2tlbml6ZXIoaW5wdXRfdGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKCUyMmN1ZGElMjIpJTBBJTBBb3V0cHV0JTIwJTNEJTIwcmVsb2FkZWRfbW9kZWwuZ2VuZXJhdGUoKippbnB1dF9pZHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDEwKSUwQXByaW50KHRva2VuaXplci5kZWNvZGUob3V0cHV0JTVCMCU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSklMEE=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TorchAoConfig, AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">from</span> torchao.quantization <span class="hljs-keyword">import</span> Int8WeightOnlyConfig

quant_config = Int8WeightOnlyConfig(group_size=<span class="hljs-number">128</span>)
quantization_config = TorchAoConfig(quant_type=quant_config)

<span class="hljs-comment"># Load and quantize the model</span>
quantized_model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-3.1-8B-Instruct&quot;</span>,
    torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>,
    device_map=<span class="hljs-string">&quot;cpu&quot;</span>,
    quantization_config=quantization_config
)
<span class="hljs-comment"># save the quantized model</span>
output_dir = <span class="hljs-string">&quot;llama-3.1-8b-torchao-int8-cuda&quot;</span>
quantized_model.save_pretrained(output_dir, safe_serialization=<span class="hljs-literal">False</span>)

<span class="hljs-comment"># reload the quantized model</span>
reloaded_model = AutoModelForCausalLM.from_pretrained(
    output_dir, 
    device_map=<span class="hljs-string">&quot;auto&quot;</span>, 
    torch_dtype=torch.bfloat16
)
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-3.1-8B-Instruct&quot;</span>)
input_text = <span class="hljs-string">&quot;What are we having for dinner?&quot;</span>
input_ids = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)

output = reloaded_model.generate(**input_ids, max_new_tokens=<span class="hljs-number">10</span>)
<span class="hljs-built_in">print</span>(tokenizer.decode(output[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))
`,wrap:!1}}),Ut=new g({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwVG9yY2hBb0NvbmZpZyUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQWZyb20lMjB0b3JjaGFvLnF1YW50aXphdGlvbiUyMGltcG9ydCUyMEludDRXZWlnaHRPbmx5Q29uZmlnJTBBZnJvbSUyMHRvcmNoYW8uZHR5cGVzJTIwaW1wb3J0JTIwSW50NENQVUxheW91dCUwQSUwQXF1YW50X2NvbmZpZyUyMCUzRCUyMEludDRXZWlnaHRPbmx5Q29uZmlnKGdyb3VwX3NpemUlM0QxMjglMkMlMjBsYXlvdXQlM0RJbnQ0Q1BVTGF5b3V0KCkpJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMFRvcmNoQW9Db25maWcocXVhbnRfdHlwZSUzRHF1YW50X2NvbmZpZyklMEElMEElMjMlMjBMb2FkJTIwYW5kJTIwcXVhbnRpemUlMjB0aGUlMjBtb2RlbCUwQXF1YW50aXplZF9tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJtZXRhLWxsYW1hJTJGTGxhbWEtMy4xLThCLUluc3RydWN0JTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0QlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRCUyMmNwdSUyMiUyQyUwQSUyMCUyMCUyMCUyMHF1YW50aXphdGlvbl9jb25maWclM0RxdWFudGl6YXRpb25fY29uZmlnJTBBKSUwQSUyMyUyMHNhdmUlMjB0aGUlMjBxdWFudGl6ZWQlMjBtb2RlbCUwQW91dHB1dF9kaXIlMjAlM0QlMjAlMjJsbGFtYS0zLjEtOGItdG9yY2hhby1pbnQ0LWNwdSUyMiUwQXF1YW50aXplZF9tb2RlbC5zYXZlX3ByZXRyYWluZWQob3V0cHV0X2RpciUyQyUyMHNhZmVfc2VyaWFsaXphdGlvbiUzREZhbHNlKSUwQSUwQSUyMyUyMHJlbG9hZCUyMHRoZSUyMHF1YW50aXplZCUyMG1vZGVsJTBBcmVsb2FkZWRfbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUyQyUyMCUwQSUyMCUyMCUyMCUyMGRldmljZV9tYXAlM0QlMjJjcHUlMjIlMkMlMjAlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTBBKSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMm1ldGEtbGxhbWElMkZMbGFtYS0zLjEtOEItSW5zdHJ1Y3QlMjIpJTBBaW5wdXRfdGV4dCUyMCUzRCUyMCUyMldoYXQlMjBhcmUlMjB3ZSUyMGhhdmluZyUyMGZvciUyMGRpbm5lciUzRiUyMiUwQWlucHV0X2lkcyUyMCUzRCUyMHRva2VuaXplcihpbnB1dF90ZXh0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiklMEElMEFvdXRwdXQlMjAlM0QlMjByZWxvYWRlZF9tb2RlbC5nZW5lcmF0ZSgqKmlucHV0X2lkcyUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMTApJTBBcHJpbnQodG9rZW5pemVyLmRlY29kZShvdXRwdXQlNUIwJTVEJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpKSUwQQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TorchAoConfig, AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">from</span> torchao.quantization <span class="hljs-keyword">import</span> Int4WeightOnlyConfig
<span class="hljs-keyword">from</span> torchao.dtypes <span class="hljs-keyword">import</span> Int4CPULayout

quant_config = Int4WeightOnlyConfig(group_size=<span class="hljs-number">128</span>, layout=Int4CPULayout())
quantization_config = TorchAoConfig(quant_type=quant_config)

<span class="hljs-comment"># Load and quantize the model</span>
quantized_model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/Llama-3.1-8B-Instruct&quot;</span>,
    torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>,
    device_map=<span class="hljs-string">&quot;cpu&quot;</span>,
    quantization_config=quantization_config
)
<span class="hljs-comment"># save the quantized model</span>
output_dir = <span class="hljs-string">&quot;llama-3.1-8b-torchao-int4-cpu&quot;</span>
quantized_model.save_pretrained(output_dir, safe_serialization=<span class="hljs-literal">False</span>)

<span class="hljs-comment"># reload the quantized model</span>
reloaded_model = AutoModelForCausalLM.from_pretrained(
    output_dir, 
    device_map=<span class="hljs-string">&quot;cpu&quot;</span>, 
    torch_dtype=torch.bfloat16
)
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-3.1-8B-Instruct&quot;</span>)
input_text = <span class="hljs-string">&quot;What are we having for dinner?&quot;</span>
input_ids = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)

output = reloaded_model.generate(**input_ids, max_new_tokens=<span class="hljs-number">10</span>)
<span class="hljs-built_in">print</span>(tokenizer.decode(output[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))
`,wrap:!1}}),jt=new X({props:{title:"⚠️ Deprecation Notice",local:"-deprecation-notice",headingTag:"h2"}}),Wt=new g({props:{code:"JTIzJTIwT2xkJTIwd2F5JTIwKGRlcHJlY2F0ZWQpJTBBcXVhbnRpemF0aW9uX2NvbmZpZyUyMCUzRCUyMFRvcmNoQW9Db25maWcoJTIyaW50NF93ZWlnaHRfb25seSUyMiUyQyUyMGdyb3VwX3NpemUlM0QxMjgpJTBBJTBBJTIzJTIwTmV3JTIwd2F5JTIwKHJlY29tbWVuZGVkKSUwQWZyb20lMjB0b3JjaGFvLnF1YW50aXphdGlvbiUyMGltcG9ydCUyMEludDRXZWlnaHRPbmx5Q29uZmlnJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwSW50NFdlaWdodE9ubHlDb25maWcoZ3JvdXBfc2l6ZSUzRDEyOCklMEFxdWFudGl6YXRpb25fY29uZmlnJTIwJTNEJTIwVG9yY2hBb0NvbmZpZyhxdWFudF90eXBlJTNEcXVhbnRfY29uZmlnKQ==",highlighted:`<span class="hljs-comment"># Old way (deprecated)</span>
quantization_config = TorchAoConfig(<span class="hljs-string">&quot;int4_weight_only&quot;</span>, group_size=<span class="hljs-number">128</span>)

<span class="hljs-comment"># New way (recommended)</span>
<span class="hljs-keyword">from</span> torchao.quantization <span class="hljs-keyword">import</span> Int4WeightOnlyConfig
quant_config = Int4WeightOnlyConfig(group_size=<span class="hljs-number">128</span>)
quantization_config = TorchAoConfig(quant_type=quant_config)`,wrap:!1}}),Zt=new X({props:{title:"Resources",local:"resources",headingTag:"h2"}}),Gt=new g({props:{code:"ZnJvbSUyMHRvcmNoLl9pbmR1Y3Rvci51dGlscyUyMGltcG9ydCUyMGRvX2JlbmNoX3VzaW5nX3Byb2ZpbGluZyUwQWZyb20lMjB0eXBpbmclMjBpbXBvcnQlMjBDYWxsYWJsZSUwQSUwQWRlZiUyMGJlbmNobWFya19mbihmdW5jJTNBJTIwQ2FsbGFibGUlMkMlMjAqYXJncyUyQyUyMCoqa3dhcmdzKSUyMC0lM0UlMjBmbG9hdCUzQSUwQSUyMCUyMCUyMCUyMCUyMiUyMiUyMlRoaW4lMjB3cmFwcGVyJTIwYXJvdW5kJTIwZG9fYmVuY2hfdXNpbmdfcHJvZmlsaW5nJTIyJTIyJTIyJTBBJTIwJTIwJTIwJTIwbm9fYXJncyUyMCUzRCUyMGxhbWJkYSUzQSUyMGZ1bmMoKmFyZ3MlMkMlMjAqKmt3YXJncyklMEElMjAlMjAlMjAlMjB0aW1lJTIwJTNEJTIwZG9fYmVuY2hfdXNpbmdfcHJvZmlsaW5nKG5vX2FyZ3MpJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwdGltZSUyMColMjAxZTMlMEElMEFNQVhfTkVXX1RPS0VOUyUyMCUzRCUyMDEwMDAlMEFwcmludCglMjJpbnQ0d28tMTI4JTIwbW9kZWwlM0ElMjIlMkMlMjBiZW5jaG1hcmtfZm4ocXVhbnRpemVkX21vZGVsLmdlbmVyYXRlJTJDJTIwKippbnB1dF9pZHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRE1BWF9ORVdfVE9LRU5TJTJDJTIwY2FjaGVfaW1wbGVtZW50YXRpb24lM0QlMjJzdGF0aWMlMjIpKSUwQSUwQWJmMTZfbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfbmFtZSUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiklMEFvdXRwdXQlMjAlM0QlMjBiZjE2X21vZGVsLmdlbmVyYXRlKCoqaW5wdXRfaWRzJTJDJTIwbWF4X25ld190b2tlbnMlM0QxMCUyQyUyMGNhY2hlX2ltcGxlbWVudGF0aW9uJTNEJTIyc3RhdGljJTIyKSUyMCUyMyUyMGF1dG8tY29tcGlsZSUwQXByaW50KCUyMmJmMTYlMjBtb2RlbCUzQSUyMiUyQyUyMGJlbmNobWFya19mbihiZjE2X21vZGVsLmdlbmVyYXRlJTJDJTIwKippbnB1dF9pZHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRE1BWF9ORVdfVE9LRU5TJTJDJTIwY2FjaGVfaW1wbGVtZW50YXRpb24lM0QlMjJzdGF0aWMlMjIpKQ==",highlighted:`<span class="hljs-keyword">from</span> torch._inductor.utils <span class="hljs-keyword">import</span> do_bench_using_profiling
<span class="hljs-keyword">from</span> typing <span class="hljs-keyword">import</span> <span class="hljs-type">Callable</span>

<span class="hljs-keyword">def</span> <span class="hljs-title function_">benchmark_fn</span>(<span class="hljs-params">func: <span class="hljs-type">Callable</span>, *args, **kwargs</span>) -&gt; <span class="hljs-built_in">float</span>:
    <span class="hljs-string">&quot;&quot;&quot;Thin wrapper around do_bench_using_profiling&quot;&quot;&quot;</span>
    no_args = <span class="hljs-keyword">lambda</span>: func(*args, **kwargs)
    time = do_bench_using_profiling(no_args)
    <span class="hljs-keyword">return</span> time * <span class="hljs-number">1e3</span>

MAX_NEW_TOKENS = <span class="hljs-number">1000</span>
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;int4wo-128 model:&quot;</span>, benchmark_fn(quantized_model.generate, **input_ids, max_new_tokens=MAX_NEW_TOKENS, cache_implementation=<span class="hljs-string">&quot;static&quot;</span>))

bf16_model = AutoModelForCausalLM.from_pretrained(model_name, device_map=<span class="hljs-string">&quot;auto&quot;</span>, torch_dtype=torch.bfloat16)
output = bf16_model.generate(**input_ids, max_new_tokens=<span class="hljs-number">10</span>, cache_implementation=<span class="hljs-string">&quot;static&quot;</span>) <span class="hljs-comment"># auto-compile</span>
<span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;bf16 model:&quot;</span>, benchmark_fn(bf16_model.generate, **input_ids, max_new_tokens=MAX_NEW_TOKENS, cache_implementation=<span class="hljs-string">&quot;static&quot;</span>))`,wrap:!1}}),q=new Kl({props:{warning:!1,$$slots:{default:[ra]},$$scope:{ctx:w}}}),It=new X({props:{title:"Issues",local:"issues",headingTag:"h2"}}),_t=new He({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/quantization/torchao.md"}}),{c(){e=J("meta"),i=p(),l=J("p"),c=p(),m(a.$$.fragment),u=p(),U=J("p"),U.innerHTML=Z,Qt=p(),F=J("p"),F.innerHTML=Ol,Et=p(),k=J("p"),k.textContent=te,Yt=p(),v=J("table"),v.innerHTML=le,Nt=p(),m(_.$$.fragment),Ht=p(),Q=J("p"),Q.innerHTML=ee,xt=p(),E=J("ul"),E.innerHTML=ae,St=p(),Y=J("p"),Y.textContent=ne,At=p(),N=J("p"),N.textContent=se,Lt=p(),H=J("table"),H.innerHTML=oe,Dt=p(),x=J("p"),x.textContent=ie,Pt=p(),m(B.$$.fragment),Kt=p(),S=J("p"),S.innerHTML=pe,Ot=p(),m(A.$$.fragment),tl=p(),L=J("p"),L.innerHTML=re,ll=p(),D=J("p"),D.innerHTML=ue,el=p(),P=J("p"),P.textContent=ce,al=p(),K=J("p"),K.innerHTML=me,nl=p(),O=J("p"),O.textContent=de,sl=p(),m(tt.$$.fragment),ol=p(),m(V.$$.fragment),il=p(),m(lt.$$.fragment),pl=p(),m(C.$$.fragment),rl=p(),m(et.$$.fragment),ul=p(),m($.$$.fragment),cl=p(),m(at.$$.fragment),ml=p(),m(nt.$$.fragment),dl=p(),st=J("p"),st.innerHTML=Me,Ml=p(),m(ot.$$.fragment),hl=p(),m(it.$$.fragment),yl=p(),m(pt.$$.fragment),bl=p(),m(rt.$$.fragment),Jl=p(),ut=J("p"),ut.innerHTML=he,fl=p(),ct=J("p"),ct.innerHTML=ye,Tl=p(),mt=J("p"),mt.textContent=be,wl=p(),dt=J("p"),dt.innerHTML=Je,Ul=p(),m(Mt.$$.fragment),jl=p(),m(ht.$$.fragment),Wl=p(),yt=J("p"),yt.innerHTML=fe,Zl=p(),bt=J("p"),bt.innerHTML=Te,gl=p(),m(z.$$.fragment),Gl=p(),m(Jt.$$.fragment),Xl=p(),ft=J("p"),ft.textContent=we,Il=p(),m(Tt.$$.fragment),Rl=p(),wt=J("p"),wt.textContent=Ue,_l=p(),m(Ut.$$.fragment),Bl=p(),m(jt.$$.fragment),Vl=p(),j=J("blockquote"),Bt=J("p"),Bt.innerHTML=je,Nl=p(),Vt=J("p"),Vt.innerHTML=We,Hl=p(),m(Wt.$$.fragment),xl=p(),Ct=J("p"),Ct.textContent=Ze,Sl=p(),$t=J("p"),$t.innerHTML=ge,Al=p(),zt=J("p"),zt.innerHTML=Ge,Ll=p(),qt=J("table"),qt.innerHTML=Xe,Dl=p(),Ft=J("p"),Ft.innerHTML=Ie,Cl=p(),m(Zt.$$.fragment),$l=p(),gt=J("p"),gt.innerHTML=Re,zl=p(),m(Gt.$$.fragment),ql=p(),m(q.$$.fragment),Fl=p(),Xt=J("p"),Xt.innerHTML=_e,kl=p(),m(It.$$.fragment),vl=p(),Rt=J("p"),Rt.innerHTML=Be,Ql=p(),m(_t.$$.fragment),El=p(),kt=J("p"),this.h()},l(t){const n=Ne("svelte-u9bgzb",document.head);e=f(n,"META",{name:!0,content:!0}),n.forEach(s),i=r(t),l=f(t,"P",{}),Pl(l).forEach(s),c=r(t),d(a.$$.fragment,t),u=r(t),U=f(t,"P",{"data-svelte-h":!0}),T(U)!=="svelte-8yu6d7"&&(U.innerHTML=Z),Qt=r(t),F=f(t,"P",{"data-svelte-h":!0}),T(F)!=="svelte-hptymg"&&(F.innerHTML=Ol),Et=r(t),k=f(t,"P",{"data-svelte-h":!0}),T(k)!=="svelte-k1mb66"&&(k.textContent=te),Yt=r(t),v=f(t,"TABLE",{"data-svelte-h":!0}),T(v)!=="svelte-1x8g49v"&&(v.innerHTML=le),Nt=r(t),d(_.$$.fragment,t),Ht=r(t),Q=f(t,"P",{"data-svelte-h":!0}),T(Q)!=="svelte-bctle4"&&(Q.innerHTML=ee),xt=r(t),E=f(t,"UL",{"data-svelte-h":!0}),T(E)!=="svelte-1oyuxco"&&(E.innerHTML=ae),St=r(t),Y=f(t,"P",{"data-svelte-h":!0}),T(Y)!=="svelte-1u913mc"&&(Y.textContent=ne),At=r(t),N=f(t,"P",{"data-svelte-h":!0}),T(N)!=="svelte-1o9cwfq"&&(N.textContent=se),Lt=r(t),H=f(t,"TABLE",{"data-svelte-h":!0}),T(H)!=="svelte-h6vlsj"&&(H.innerHTML=oe),Dt=r(t),x=f(t,"P",{"data-svelte-h":!0}),T(x)!=="svelte-1wnb12y"&&(x.textContent=ie),Pt=r(t),d(B.$$.fragment,t),Kt=r(t),S=f(t,"P",{"data-svelte-h":!0}),T(S)!=="svelte-vh1vhn"&&(S.innerHTML=pe),Ot=r(t),d(A.$$.fragment,t),tl=r(t),L=f(t,"P",{"data-svelte-h":!0}),T(L)!=="svelte-mfxtr3"&&(L.innerHTML=re),ll=r(t),D=f(t,"P",{"data-svelte-h":!0}),T(D)!=="svelte-1omxb57"&&(D.innerHTML=ue),el=r(t),P=f(t,"P",{"data-svelte-h":!0}),T(P)!=="svelte-hxxsaz"&&(P.textContent=ce),al=r(t),K=f(t,"P",{"data-svelte-h":!0}),T(K)!=="svelte-17m23y5"&&(K.innerHTML=me),nl=r(t),O=f(t,"P",{"data-svelte-h":!0}),T(O)!=="svelte-yb6i3u"&&(O.textContent=de),sl=r(t),d(tt.$$.fragment,t),ol=r(t),d(V.$$.fragment,t),il=r(t),d(lt.$$.fragment,t),pl=r(t),d(C.$$.fragment,t),rl=r(t),d(et.$$.fragment,t),ul=r(t),d($.$$.fragment,t),cl=r(t),d(at.$$.fragment,t),ml=r(t),d(nt.$$.fragment,t),dl=r(t),st=f(t,"P",{"data-svelte-h":!0}),T(st)!=="svelte-ls2wu5"&&(st.innerHTML=Me),Ml=r(t),d(ot.$$.fragment,t),hl=r(t),d(it.$$.fragment,t),yl=r(t),d(pt.$$.fragment,t),bl=r(t),d(rt.$$.fragment,t),Jl=r(t),ut=f(t,"P",{"data-svelte-h":!0}),T(ut)!=="svelte-1fr0k2r"&&(ut.innerHTML=he),fl=r(t),ct=f(t,"P",{"data-svelte-h":!0}),T(ct)!=="svelte-axxx0x"&&(ct.innerHTML=ye),Tl=r(t),mt=f(t,"P",{"data-svelte-h":!0}),T(mt)!=="svelte-19318ol"&&(mt.textContent=be),wl=r(t),dt=f(t,"P",{"data-svelte-h":!0}),T(dt)!=="svelte-21luch"&&(dt.innerHTML=Je),Ul=r(t),d(Mt.$$.fragment,t),jl=r(t),d(ht.$$.fragment,t),Wl=r(t),yt=f(t,"P",{"data-svelte-h":!0}),T(yt)!=="svelte-1g3oift"&&(yt.innerHTML=fe),Zl=r(t),bt=f(t,"P",{"data-svelte-h":!0}),T(bt)!=="svelte-5ma9bd"&&(bt.innerHTML=Te),gl=r(t),d(z.$$.fragment,t),Gl=r(t),d(Jt.$$.fragment,t),Xl=r(t),ft=f(t,"P",{"data-svelte-h":!0}),T(ft)!=="svelte-14fyln8"&&(ft.textContent=we),Il=r(t),d(Tt.$$.fragment,t),Rl=r(t),wt=f(t,"P",{"data-svelte-h":!0}),T(wt)!=="svelte-1oj3qy1"&&(wt.textContent=Ue),_l=r(t),d(Ut.$$.fragment,t),Bl=r(t),d(jt.$$.fragment,t),Vl=r(t),j=f(t,"BLOCKQUOTE",{});var W=Pl(j);Bt=f(W,"P",{"data-svelte-h":!0}),T(Bt)!=="svelte-jmxllg"&&(Bt.innerHTML=je),Nl=r(W),Vt=f(W,"P",{"data-svelte-h":!0}),T(Vt)!=="svelte-1qak5au"&&(Vt.innerHTML=We),Hl=r(W),d(Wt.$$.fragment,W),xl=r(W),Ct=f(W,"P",{"data-svelte-h":!0}),T(Ct)!=="svelte-8tiw44"&&(Ct.textContent=Ze),Sl=r(W),$t=f(W,"P",{"data-svelte-h":!0}),T($t)!=="svelte-1wt1icj"&&($t.innerHTML=ge),Al=r(W),zt=f(W,"P",{"data-svelte-h":!0}),T(zt)!=="svelte-1kyext4"&&(zt.innerHTML=Ge),Ll=r(W),qt=f(W,"TABLE",{"data-svelte-h":!0}),T(qt)!=="svelte-1w9xui3"&&(qt.innerHTML=Xe),Dl=r(W),Ft=f(W,"P",{"data-svelte-h":!0}),T(Ft)!=="svelte-9z9ctj"&&(Ft.innerHTML=Ie),W.forEach(s),Cl=r(t),d(Zt.$$.fragment,t),$l=r(t),gt=f(t,"P",{"data-svelte-h":!0}),T(gt)!=="svelte-1cgj2bs"&&(gt.innerHTML=Re),zl=r(t),d(Gt.$$.fragment,t),ql=r(t),d(q.$$.fragment,t),Fl=r(t),Xt=f(t,"P",{"data-svelte-h":!0}),T(Xt)!=="svelte-fj0t1q"&&(Xt.innerHTML=_e),kl=r(t),d(It.$$.fragment,t),vl=r(t),Rt=f(t,"P",{"data-svelte-h":!0}),T(Rt)!=="svelte-1auvp72"&&(Rt.innerHTML=Be),Ql=r(t),d(_t.$$.fragment,t),El=r(t),kt=f(t,"P",{}),Pl(kt).forEach(s),this.h()},h(){ke(e,"name","hf:doc:metadata"),ke(e,"content",ca)},m(t,n){G(document.head,e),o(t,i,n),o(t,l,n),o(t,c,n),M(a,t,n),o(t,u,n),o(t,U,n),o(t,Qt,n),o(t,F,n),o(t,Et,n),o(t,k,n),o(t,Yt,n),o(t,v,n),o(t,Nt,n),M(_,t,n),o(t,Ht,n),o(t,Q,n),o(t,xt,n),o(t,E,n),o(t,St,n),o(t,Y,n),o(t,At,n),o(t,N,n),o(t,Lt,n),o(t,H,n),o(t,Dt,n),o(t,x,n),o(t,Pt,n),M(B,t,n),o(t,Kt,n),o(t,S,n),o(t,Ot,n),M(A,t,n),o(t,tl,n),o(t,L,n),o(t,ll,n),o(t,D,n),o(t,el,n),o(t,P,n),o(t,al,n),o(t,K,n),o(t,nl,n),o(t,O,n),o(t,sl,n),M(tt,t,n),o(t,ol,n),M(V,t,n),o(t,il,n),M(lt,t,n),o(t,pl,n),M(C,t,n),o(t,rl,n),M(et,t,n),o(t,ul,n),M($,t,n),o(t,cl,n),M(at,t,n),o(t,ml,n),M(nt,t,n),o(t,dl,n),o(t,st,n),o(t,Ml,n),M(ot,t,n),o(t,hl,n),M(it,t,n),o(t,yl,n),M(pt,t,n),o(t,bl,n),M(rt,t,n),o(t,Jl,n),o(t,ut,n),o(t,fl,n),o(t,ct,n),o(t,Tl,n),o(t,mt,n),o(t,wl,n),o(t,dt,n),o(t,Ul,n),M(Mt,t,n),o(t,jl,n),M(ht,t,n),o(t,Wl,n),o(t,yt,n),o(t,Zl,n),o(t,bt,n),o(t,gl,n),M(z,t,n),o(t,Gl,n),M(Jt,t,n),o(t,Xl,n),o(t,ft,n),o(t,Il,n),M(Tt,t,n),o(t,Rl,n),o(t,wt,n),o(t,_l,n),M(Ut,t,n),o(t,Bl,n),M(jt,t,n),o(t,Vl,n),o(t,j,n),G(j,Bt),G(j,Nl),G(j,Vt),G(j,Hl),M(Wt,j,null),G(j,xl),G(j,Ct),G(j,Sl),G(j,$t),G(j,Al),G(j,zt),G(j,Ll),G(j,qt),G(j,Dl),G(j,Ft),o(t,Cl,n),M(Zt,t,n),o(t,$l,n),o(t,gt,n),o(t,zl,n),M(Gt,t,n),o(t,ql,n),M(q,t,n),o(t,Fl,n),o(t,Xt,n),o(t,kl,n),M(It,t,n),o(t,vl,n),o(t,Rt,n),o(t,Ql,n),M(_t,t,n),o(t,El,n),o(t,kt,n),Yl=!0},p(t,[n]){const W={};n&2&&(W.$$scope={dirty:n,ctx:t}),_.$set(W);const Ve={};n&2&&(Ve.$$scope={dirty:n,ctx:t}),B.$set(Ve);const Ce={};n&2&&(Ce.$$scope={dirty:n,ctx:t}),V.$set(Ce);const $e={};n&2&&($e.$$scope={dirty:n,ctx:t}),C.$set($e);const ze={};n&2&&(ze.$$scope={dirty:n,ctx:t}),$.$set(ze);const qe={};n&2&&(qe.$$scope={dirty:n,ctx:t}),z.$set(qe);const Fe={};n&2&&(Fe.$$scope={dirty:n,ctx:t}),q.$set(Fe)},i(t){Yl||(h(a.$$.fragment,t),h(_.$$.fragment,t),h(B.$$.fragment,t),h(A.$$.fragment,t),h(tt.$$.fragment,t),h(V.$$.fragment,t),h(lt.$$.fragment,t),h(C.$$.fragment,t),h(et.$$.fragment,t),h($.$$.fragment,t),h(at.$$.fragment,t),h(nt.$$.fragment,t),h(ot.$$.fragment,t),h(it.$$.fragment,t),h(pt.$$.fragment,t),h(rt.$$.fragment,t),h(Mt.$$.fragment,t),h(ht.$$.fragment,t),h(z.$$.fragment,t),h(Jt.$$.fragment,t),h(Tt.$$.fragment,t),h(Ut.$$.fragment,t),h(jt.$$.fragment,t),h(Wt.$$.fragment,t),h(Zt.$$.fragment,t),h(Gt.$$.fragment,t),h(q.$$.fragment,t),h(It.$$.fragment,t),h(_t.$$.fragment,t),Yl=!0)},o(t){y(a.$$.fragment,t),y(_.$$.fragment,t),y(B.$$.fragment,t),y(A.$$.fragment,t),y(tt.$$.fragment,t),y(V.$$.fragment,t),y(lt.$$.fragment,t),y(C.$$.fragment,t),y(et.$$.fragment,t),y($.$$.fragment,t),y(at.$$.fragment,t),y(nt.$$.fragment,t),y(ot.$$.fragment,t),y(it.$$.fragment,t),y(pt.$$.fragment,t),y(rt.$$.fragment,t),y(Mt.$$.fragment,t),y(ht.$$.fragment,t),y(z.$$.fragment,t),y(Jt.$$.fragment,t),y(Tt.$$.fragment,t),y(Ut.$$.fragment,t),y(jt.$$.fragment,t),y(Wt.$$.fragment,t),y(Zt.$$.fragment,t),y(Gt.$$.fragment,t),y(q.$$.fragment,t),y(It.$$.fragment,t),y(_t.$$.fragment,t),Yl=!1},d(t){t&&(s(i),s(l),s(c),s(u),s(U),s(Qt),s(F),s(Et),s(k),s(Yt),s(v),s(Nt),s(Ht),s(Q),s(xt),s(E),s(St),s(Y),s(At),s(N),s(Lt),s(H),s(Dt),s(x),s(Pt),s(Kt),s(S),s(Ot),s(tl),s(L),s(ll),s(D),s(el),s(P),s(al),s(K),s(nl),s(O),s(sl),s(ol),s(il),s(pl),s(rl),s(ul),s(cl),s(ml),s(dl),s(st),s(Ml),s(hl),s(yl),s(bl),s(Jl),s(ut),s(fl),s(ct),s(Tl),s(mt),s(wl),s(dt),s(Ul),s(jl),s(Wl),s(yt),s(Zl),s(bt),s(gl),s(Gl),s(Xl),s(ft),s(Il),s(Rl),s(wt),s(_l),s(Bl),s(Vl),s(j),s(Cl),s($l),s(gt),s(zl),s(ql),s(Fl),s(Xt),s(kl),s(vl),s(Rt),s(Ql),s(El),s(kt)),s(e),b(a,t),b(_,t),b(B,t),b(A,t),b(tt,t),b(V,t),b(lt,t),b(C,t),b(et,t),b($,t),b(at,t),b(nt,t),b(ot,t),b(it,t),b(pt,t),b(rt,t),b(Mt,t),b(ht,t),b(z,t),b(Jt,t),b(Tt,t),b(Ut,t),b(jt,t),b(Wt),b(Zt,t),b(Gt,t),b(q,t),b(It,t),b(_t,t)}}}const ca='{"title":"torchao","local":"torchao","sections":[{"title":"Quantization examples","local":"quantization-examples","sections":[{"title":"H100 GPU","local":"h100-gpu","sections":[],"depth":3},{"title":"A100 GPU","local":"a100-gpu","sections":[],"depth":3},{"title":"CPU","local":"cpu","sections":[],"depth":3},{"title":"Per Module Quantization","local":"per-module-quantization","sections":[{"title":"1. Skip quantization for certain layers","local":"1-skip-quantization-for-certain-layers","sections":[],"depth":4},{"title":"2. Quantizing different layers with different quantization configs","local":"2-quantizing-different-layers-with-different-quantization-configs","sections":[],"depth":4}],"depth":3},{"title":"Autoquant","local":"autoquant","sections":[],"depth":3}],"depth":2},{"title":"Serialization","local":"serialization","sections":[],"depth":2},{"title":"Loading quantized models","local":"loading-quantized-models","sections":[],"depth":2},{"title":"⚠️ Deprecation Notice","local":"-deprecation-notice","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2},{"title":"Issues","local":"issues","sections":[],"depth":2}],"depth":1}';function ma(w){return Qe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ta extends Ee{constructor(e){super(),Ye(this,e,ma,ua,ve,{})}}export{Ta as component};
