import{s as tl,z as ll,o as sl,n as $}from"../chunks/scheduler.18a86fab.js";import{S as al,i as nl,g as u,s as p,r as J,A as ol,h,f as a,c,j as wt,u as T,x as f,k as ve,y as il,a as n,v as U,d as b,t as w,w as j}from"../chunks/index.98837b22.js";import{T as D}from"../chunks/Tip.77304350.js";import{C as I}from"../chunks/CodeBlock.8d0c2e8a.js";import{H as Y}from"../chunks/Heading.7e7e0c5b.js";import{H as jt,a as Be}from"../chunks/HfOption.6641485e.js";import{E as rl}from"../chunks/index.499f366c.js";function pl(g){let t,M='Try out <a href="https://hf.co/docs/text-generation-inference" rel="nofollow">Text Generation Inference (TGI)</a>, a Hugging Face library dedicated to deploying and serving highly optimized LLMs for inference.';return{c(){t=u("p"),t.innerHTML=M},l(s){t=h(s,"P",{"data-svelte-h":!0}),f(t)!=="svelte-c1q0e5"&&(t.innerHTML=M)},m(s,r){n(s,t,r)},p:$,d(s){s&&a(t)}}}function cl(g){let t,M='Follow this <a href="https://github.com/huggingface/transformers/issues/28981" rel="nofollow">issue</a> to track which models (Llama, Gemma, Mistral, etc.) support a static kv-cache and torch.compile.';return{c(){t=u("p"),t.innerHTML=M},l(s){t=h(s,"P",{"data-svelte-h":!0}),f(t)!=="svelte-58h2fm"&&(t.innerHTML=M)},m(s,r){n(s,t,r)},p:$,d(s){s&&a(t)}}}function Ml(g){let t,M='Regardless of how you use the static kv-cache and torch.compile, left-pad your inputs with <a href="https://hf.co/docs/transformers/main_classes/tokenizer#transformers.PreTrainedTokenizer.__call__.pad_to_multiple_of" rel="nofollow">pad_to_multiple_of</a> to a limited set of values to avoid shape-related recompilations.';return{c(){t=u("p"),t.innerHTML=M},l(s){t=h(s,"P",{"data-svelte-h":!0}),f(t)!=="svelte-uz572w"&&(t.innerHTML=M)},m(s,r){n(s,t,r)},p:$,d(s){s&&a(t)}}}function ml(g){let t,M='<li>Set the <a href="https://hf.co/docs/transformers/main_classes/text_generation#transformers.GenerationConfig.cache_implementation" rel="nofollow">cache_implementation</a> to <code>&quot;static&quot;</code> in a models <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationConfig">GenerationConfig</a>.</li> <li>Call <a href="./perf_torch_compile">torch.compile</a> to compile the forward pass with the static kv-cache.</li>',s,r,o,i,m='Under the hood, <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a> attempts to reuse the same cache object to avoid recompilation at each call, which is critical to get the most out of <a href="./perf_torch_compile">torch.compile</a>. Be aware of the following to avoid triggering recompilation or if generation is slower than expected.',Z,k,v="<li>If the batch size changes or the maximum output length increases between calls, the cache is reinitialized and recompiled.</li> <li>The first several calls of the compiled function are slower because it is being compiled.</li>",_;return r=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQWltcG9ydCUyMHRvcmNoJTBBaW1wb3J0JTIwb3MlMEFvcy5lbnZpcm9uJTVCJTIyVE9LRU5JWkVSU19QQVJBTExFTElTTSUyMiU1RCUyMCUzRCUyMCUyMmZhbHNlJTIyJTIwJTIwJTIzJTIwVG8lMjBwcmV2ZW50JTIwbG9uZyUyMHdhcm5pbmdzJTIwJTNBKSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmdlbW1hLTJiJTIyKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmdlbW1hLTJiJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0QlMjJhdXRvJTIyJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIpJTBBJTBBbW9kZWwuZ2VuZXJhdGlvbl9jb25maWcuY2FjaGVfaW1wbGVtZW50YXRpb24lMjAlM0QlMjAlMjJzdGF0aWMlMjIlMEElMEFtb2RlbC5mb3J3YXJkJTIwJTNEJTIwdG9yY2guY29tcGlsZShtb2RlbC5mb3J3YXJkJTJDJTIwbW9kZSUzRCUyMnJlZHVjZS1vdmVyaGVhZCUyMiUyQyUyMGZ1bGxncmFwaCUzRFRydWUpJTBBaW5wdXRfdGV4dCUyMCUzRCUyMCUyMlRoZSUyMHRoZW9yeSUyMG9mJTIwc3BlY2lhbCUyMHJlbGF0aXZpdHklMjBzdGF0ZXMlMjAlMjIlMEFpbnB1dF9pZHMlMjAlM0QlMjB0b2tlbml6ZXIoaW5wdXRfdGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKG1vZGVsLmRldmljZS50eXBlKSUwQSUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0X2lkcyklMEFwcmludCh0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKG91dHB1dHMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSkpJTBBJTVCJ1RoZSUyMHRoZW9yeSUyMG9mJTIwc3BlY2lhbCUyMHJlbGF0aXZpdHklMjBzdGF0ZXMlMjAxLiUyMFRoZSUyMHNwZWVkJTIwb2YlMjBsaWdodCUyMGlzJTIwY29uc3RhbnQlMjBpbiUyMGFsbCUyMGluZXJ0aWFsJTIwcmVmZXJlbmNlJyU1RA==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> os
os.environ[<span class="hljs-string">&quot;TOKENIZERS_PARALLELISM&quot;</span>] = <span class="hljs-string">&quot;false&quot;</span>  <span class="hljs-comment"># To prevent long warnings :)</span>

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;google/gemma-2b&quot;</span>)
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;google/gemma-2b&quot;</span>, torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)

model.generation_config.cache_implementation = <span class="hljs-string">&quot;static&quot;</span>

model.forward = torch.<span class="hljs-built_in">compile</span>(model.forward, mode=<span class="hljs-string">&quot;reduce-overhead&quot;</span>, fullgraph=<span class="hljs-literal">True</span>)
input_text = <span class="hljs-string">&quot;The theory of special relativity states &quot;</span>
input_ids = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device.<span class="hljs-built_in">type</span>)

outputs = model.generate(**input_ids)
<span class="hljs-built_in">print</span>(tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>))
[<span class="hljs-string">&#x27;The theory of special relativity states 1. The speed of light is constant in all inertial reference&#x27;</span>]`,wrap:!1}}),{c(){t=u("ol"),t.innerHTML=M,s=p(),J(r.$$.fragment),o=p(),i=u("p"),i.innerHTML=m,Z=p(),k=u("ol"),k.innerHTML=v},l(d){t=h(d,"OL",{"data-svelte-h":!0}),f(t)!=="svelte-k7c087"&&(t.innerHTML=M),s=c(d),T(r.$$.fragment,d),o=c(d),i=h(d,"P",{"data-svelte-h":!0}),f(i)!=="svelte-1nxft6v"&&(i.innerHTML=m),Z=c(d),k=h(d,"OL",{"data-svelte-h":!0}),f(k)!=="svelte-13b9uyo"&&(k.innerHTML=v)},m(d,G){n(d,t,G),n(d,s,G),U(r,d,G),n(d,o,G),n(d,i,G),n(d,Z,G),n(d,k,G),_=!0},p:$,i(d){_||(b(r.$$.fragment,d),_=!0)},o(d){w(r.$$.fragment,d),_=!1},d(d){d&&(a(t),a(s),a(o),a(i),a(Z),a(k)),j(r,d)}}}function dl(g){let t,M='To reuse <a href="/docs/transformers/main/en/internal/generation_utils#transformers.StaticCache">StaticCache</a> on a new prompt, use <a href="/docs/transformers/main/en/internal/generation_utils#transformers.StaticCache.reset">reset()</a> to reset the cache contents between calls.';return{c(){t=u("p"),t.innerHTML=M},l(s){t=h(s,"P",{"data-svelte-h":!0}),f(t)!=="svelte-1j560i1"&&(t.innerHTML=M)},m(s,r){n(s,t,r)},p:$,d(s){s&&a(t)}}}function yl(g){let t,M='Directly initialize a <a href="/docs/transformers/main/en/internal/generation_utils#transformers.StaticCache">StaticCache</a> object and pass it to the <code>past_key_values</code> parameter in <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a>. The <a href="/docs/transformers/main/en/internal/generation_utils#transformers.StaticCache">StaticCache</a> keeps the cache contents, so you can pass it to a new <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a> call to continue generation, similar to a dynamic cache.',s,r,o,i,m,Z,k='Another option for using <a href="/docs/transformers/main/en/internal/generation_utils#transformers.StaticCache">StaticCache</a> is to pass it to a models forward pass using the same <code>past_key_values</code> argument. This allows you to write your own custom decoding function to decode the next token given the current token, position, and cache position of previously generated tokens.',v,_,d,G,Ce='To enable static kv-cache and <a href="./perf_torch_compile">torch.compile</a> with <a href="/docs/transformers/main/en/internal/generation_utils#transformers.StaticCache">StaticCache</a>, follow the steps below.',z,C,P='<li>Initialize <a href="/docs/transformers/main/en/internal/generation_utils#transformers.StaticCache">StaticCache</a> before using the model for inference to configure parameters like the maximum batch size and sequence length.</li> <li>Call <a href="./perf_torch_compile">torch.compile</a> on the model to compile the forward pass with the static kv-cache.</li> <li>se SDPBackend.MATH in the <a href="https://pytorch.org/docs/stable/generated/torch.nn.attention.sdpa_kernel.html" rel="nofollow">torch.nn.attention.sdpa_kernel</a> context manager to enable the native PyTorch C++ implementation of scaled dot product attention to speed up inference even more.</li>',X,W,V;return r=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUyQyUyMFN0YXRpY0NhY2hlJTBBaW1wb3J0JTIwdG9yY2glMEFpbXBvcnQlMjBvcyUwQW9zLmVudmlyb24lNUIlMjJUT0tFTklaRVJTX1BBUkFMTEVMSVNNJTIyJTVEJTIwJTNEJTIwJTIyZmFsc2UlMjIlMjAlMjAlMjMlMjBUbyUyMHByZXZlbnQlMjBsb25nJTIwd2FybmluZ3MlMjAlM0EpJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyZ29vZ2xlJTJGZ2VtbWEtMmIlMjIpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyZ29vZ2xlJTJGZ2VtbWEtMmIlMjIlMkMlMjB0b3JjaF9kdHlwZSUzRCUyMmF1dG8lMjIlMkMlMjBkZXZpY2VfbWFwJTNEJTIyYXV0byUyMiklMEElMEFtb2RlbC5mb3J3YXJkJTIwJTNEJTIwdG9yY2guY29tcGlsZShtb2RlbC5mb3J3YXJkJTJDJTIwbW9kZSUzRCUyMnJlZHVjZS1vdmVyaGVhZCUyMiUyQyUyMGZ1bGxncmFwaCUzRFRydWUpJTBBaW5wdXRfdGV4dCUyMCUzRCUyMCUyMlRoZSUyMHRoZW9yeSUyMG9mJTIwc3BlY2lhbCUyMHJlbGF0aXZpdHklMjBzdGF0ZXMlMjAlMjIlMEFpbnB1dF9pZHMlMjAlM0QlMjB0b2tlbml6ZXIoaW5wdXRfdGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKG1vZGVsLmRldmljZS50eXBlKSUwQXByb21wdF9sZW5ndGglMjAlM0QlMjBpbnB1dF9pZHMuaW5wdXRfaWRzLnNoYXBlJTVCMSU1RCUwQW1vZGVsLmdlbmVyYXRpb25fY29uZmlnLm1heF9uZXdfdG9rZW5zJTIwJTNEJTIwMTYlMEElMEFwYXN0X2tleV92YWx1ZXMlMjAlM0QlMjBTdGF0aWNDYWNoZSglMEElMjAlMjAlMjAlMjBjb25maWclM0Rtb2RlbC5jb25maWclMkMlMEElMjAlMjAlMjAlMjBtYXhfYmF0Y2hfc2l6ZSUzRDElMkMlMEElMjAlMjAlMjAlMjAlMjMlMjBJZiUyMHlvdSUyMHBsYW4lMjB0byUyMHJldXNlJTIwdGhlJTIwY2FjaGUlMkMlMjBtYWtlJTIwc3VyZSUyMHRoZSUyMGNhY2hlJTIwbGVuZ3RoJTIwaXMlMjBsYXJnZSUyMGVub3VnaCUyMGZvciUyMGFsbCUyMGNhc2VzJTBBJTIwJTIwJTIwJTIwbWF4X2NhY2hlX2xlbiUzRHByb21wdF9sZW5ndGglMkIobW9kZWwuZ2VuZXJhdGlvbl9jb25maWcubWF4X25ld190b2tlbnMqMiklMkMlMEElMjAlMjAlMjAlMjBkZXZpY2UlM0Rtb2RlbC5kZXZpY2UlMkMlMEElMjAlMjAlMjAlMjBkdHlwZSUzRG1vZGVsLmR0eXBlJTBBKSUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0X2lkcyUyQyUyMHBhc3Rfa2V5X3ZhbHVlcyUzRHBhc3Rfa2V5X3ZhbHVlcyklMEFwcmludCh0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKG91dHB1dHMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSkpJTBBJTVCJ1RoZSUyMHRoZW9yeSUyMG9mJTIwc3BlY2lhbCUyMHJlbGF0aXZpdHklMjBzdGF0ZXMlMjAxLiUyMFRoZSUyMHNwZWVkJTIwb2YlMjBsaWdodCUyMGlzJTIwY29uc3RhbnQlMjBpbiUyMGFsbCUyMGluZXJ0aWFsJTIwcmVmZXJlbmNlJTIwZnJhbWVzLiUyMDInJTVEJTBBJTBBJTIzJTIwcGFzcyUyMGluJTIwdGhlJTIwZ2VuZXJhdGVkJTIwdGV4dCUyMGFuZCUyMHRoZSUyMHNhbWUlMjBjYWNoZSUyMG9iamVjdCUyMHRvJTIwY29udGludWUlMjBnZW5lcmF0aW9uJTIwZnJvbSUyMHdoZXJlJTIwaXQlMjBsZWZ0JTIwb2ZmLiUyME9wdGlvbmFsbHklMkMlMjBpbiUyMGElMEElMjMlMjBtdWx0aS10dXJuJTIwY29udmVyc2F0aW9uJTJDJTIwYXBwZW5kJTIwdGhlJTIwbmV3JTIwdXNlciUyMGlucHV0JTIwdG8lMjB0aGUlMjBnZW5lcmF0ZWQlMjB0ZXh0LiUwQW5ld19pbnB1dF9pZHMlMjAlM0QlMjBvdXRwdXRzJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKG5ld19pbnB1dF9pZHMlMkMlMjBwYXN0X2tleV92YWx1ZXMlM0RwYXN0X2tleV92YWx1ZXMpJTBBcHJpbnQodG9rZW5pemVyLmJhdGNoX2RlY29kZShvdXRwdXRzJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpKSUwQSU1QidUaGUlMjB0aGVvcnklMjBvZiUyMHNwZWNpYWwlMjByZWxhdGl2aXR5JTIwc3RhdGVzJTIwMS4lMjBUaGUlMjBzcGVlZCUyMG9mJTIwbGlnaHQlMjBpcyUyMGNvbnN0YW50JTIwaW4lMjBhbGwlMjBpbmVydGlhbCUyMHJlZmVyZW5jZSUyMGZyYW1lcy4lMjAyLiUyMFRoZSUyMHNwZWVkJTIwb2YlMjBsaWdodCUyMGlzJTIwY29uc3RhbnQlMjBpbiUyMGFsbCUyMGluZXJ0aWFsJTIwcmVmZXJlbmNlJTIwZnJhbWVzLiUyMDMuJyU1RA==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM, StaticCache
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> os
os.environ[<span class="hljs-string">&quot;TOKENIZERS_PARALLELISM&quot;</span>] = <span class="hljs-string">&quot;false&quot;</span>  <span class="hljs-comment"># To prevent long warnings :)</span>

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;google/gemma-2b&quot;</span>)
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;google/gemma-2b&quot;</span>, torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)

model.forward = torch.<span class="hljs-built_in">compile</span>(model.forward, mode=<span class="hljs-string">&quot;reduce-overhead&quot;</span>, fullgraph=<span class="hljs-literal">True</span>)
input_text = <span class="hljs-string">&quot;The theory of special relativity states &quot;</span>
input_ids = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device.<span class="hljs-built_in">type</span>)
prompt_length = input_ids.input_ids.shape[<span class="hljs-number">1</span>]
model.generation_config.max_new_tokens = <span class="hljs-number">16</span>

past_key_values = StaticCache(
    config=model.config,
    max_batch_size=<span class="hljs-number">1</span>,
    <span class="hljs-comment"># If you plan to reuse the cache, make sure the cache length is large enough for all cases</span>
    max_cache_len=prompt_length+(model.generation_config.max_new_tokens*<span class="hljs-number">2</span>),
    device=model.device,
    dtype=model.dtype
)
outputs = model.generate(**input_ids, past_key_values=past_key_values)
<span class="hljs-built_in">print</span>(tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>))
[<span class="hljs-string">&#x27;The theory of special relativity states 1. The speed of light is constant in all inertial reference frames. 2&#x27;</span>]

<span class="hljs-comment"># pass in the generated text and the same cache object to continue generation from where it left off. Optionally, in a</span>
<span class="hljs-comment"># multi-turn conversation, append the new user input to the generated text.</span>
new_input_ids = outputs
outputs = model.generate(new_input_ids, past_key_values=past_key_values)
<span class="hljs-built_in">print</span>(tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>))
[<span class="hljs-string">&#x27;The theory of special relativity states 1. The speed of light is constant in all inertial reference frames. 2. The speed of light is constant in all inertial reference frames. 3.&#x27;</span>]`,wrap:!1}}),i=new D({props:{warning:!1,$$slots:{default:[dl]},$$scope:{ctx:g}}}),_=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMExsYW1hVG9rZW5pemVyJTJDJTIwTGxhbWFGb3JDYXVzYWxMTSUyQyUyMFN0YXRpY0NhY2hlJTJDJTIwbG9nZ2luZyUwQWZyb20lMjB0cmFuc2Zvcm1lcnMudGVzdGluZ191dGlscyUyMGltcG9ydCUyMENhcHR1cmVMb2dnZXIlMEFpbXBvcnQlMjB0b3JjaCUwQWZyb20lMjBhY2NlbGVyYXRlLnRlc3RfdXRpbHMudGVzdGluZyUyMGltcG9ydCUyMGdldF9iYWNrZW5kJTBBJTBBcHJvbXB0cyUyMCUzRCUyMCU1QiUwQSUyMCUyMCUyMCUyMCUyMlNpbXBseSUyMHB1dCUyQyUyMHRoZSUyMHRoZW9yeSUyMG9mJTIwcmVsYXRpdml0eSUyMHN0YXRlcyUyMHRoYXQlMjAlMjIlMkMlMEElMjAlMjAlMjAlMjAlMjJNeSUyMGZhdm9yaXRlJTIwYWxsJTIwdGltZSUyMGZhdm9yaXRlJTIwY29uZGltZW50JTIwaXMlMjBrZXRjaHVwLiUyMiUyQyUwQSU1RCUwQSUwQU5VTV9UT0tFTlNfVE9fR0VORVJBVEUlMjAlM0QlMjA0MCUwQXRvcmNoX2RldmljZSUyQyUyMF8lMkMlMjBfJTIwJTNEJTIwZ2V0X2JhY2tlbmQoKSUyMCUyMyUyMGF1dG9tYXRpY2FsbHklMjBkZXRlY3RzJTIwdGhlJTIwdW5kZXJseWluZyUyMGRldmljZSUyMHR5cGUlMjAoQ1VEQSUyQyUyMENQVSUyQyUyMFhQVSUyQyUyME1QUyUyQyUyMGV0Yy4pJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwTGxhbWFUb2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMm1ldGEtbGxhbWElMkZMbGFtYS0yLTdiLWhmJTIyJTJDJTIwcGFkX3Rva2VuJTNEJTIyJTNDJTJGcyUzRSUyMiUyQyUyMHBhZGRpbmdfc2lkZSUzRCUyMnJpZ2h0JTIyKSUwQW1vZGVsJTIwJTNEJTIwTGxhbWFGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIybWV0YS1sbGFtYSUyRkxsYW1hLTItN2ItaGYlMjIlMkMlMjBkZXZpY2VfbWFwJTNEJTIyc2VxdWVudGlhbCUyMiklMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIocHJvbXB0cyUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIlMkMlMjBwYWRkaW5nJTNEVHJ1ZSkudG8obW9kZWwuZGV2aWNlKSUwQSUwQWRlZiUyMGRlY29kZV9vbmVfdG9rZW5zKG1vZGVsJTJDJTIwY3VyX3Rva2VuJTJDJTIwaW5wdXRfcG9zJTJDJTIwY2FjaGVfcG9zaXRpb24lMkMlMjBwYXN0X2tleV92YWx1ZXMpJTNBJTBBJTIwJTIwJTIwJTIwbG9naXRzJTIwJTNEJTIwbW9kZWwoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY3VyX3Rva2VuJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcG9zaXRpb25faWRzJTNEaW5wdXRfcG9zJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY2FjaGVfcG9zaXRpb24lM0RjYWNoZV9wb3NpdGlvbiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHBhc3Rfa2V5X3ZhbHVlcyUzRHBhc3Rfa2V5X3ZhbHVlcyUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybl9kaWN0JTNERmFsc2UlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB1c2VfY2FjaGUlM0RUcnVlJTBBJTIwJTIwJTIwJTIwKSU1QjAlNUQlMEElMjAlMjAlMjAlMjBuZXdfdG9rZW4lMjAlM0QlMjB0b3JjaC5hcmdtYXgobG9naXRzJTVCJTNBJTJDJTIwLTElNUQlMkMlMjBkaW0lM0QtMSklNUIlM0ElMkMlMjBOb25lJTVEJTBBJTIwJTIwJTIwJTIwcmV0dXJuJTIwbmV3X3Rva2Vu",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> LlamaTokenizer, LlamaForCausalLM, StaticCache, logging
<span class="hljs-keyword">from</span> transformers.testing_utils <span class="hljs-keyword">import</span> CaptureLogger
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> accelerate.test_utils.testing <span class="hljs-keyword">import</span> get_backend

prompts = [
    <span class="hljs-string">&quot;Simply put, the theory of relativity states that &quot;</span>,
    <span class="hljs-string">&quot;My favorite all time favorite condiment is ketchup.&quot;</span>,
]

NUM_TOKENS_TO_GENERATE = <span class="hljs-number">40</span>
torch_device, _, _ = get_backend() <span class="hljs-comment"># automatically detects the underlying device type (CUDA, CPU, XPU, MPS, etc.)</span>

tokenizer = LlamaTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-7b-hf&quot;</span>, pad_token=<span class="hljs-string">&quot;&lt;/s&gt;&quot;</span>, padding_side=<span class="hljs-string">&quot;right&quot;</span>)
model = LlamaForCausalLM.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-7b-hf&quot;</span>, device_map=<span class="hljs-string">&quot;sequential&quot;</span>)
inputs = tokenizer(prompts, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>, padding=<span class="hljs-literal">True</span>).to(model.device)

<span class="hljs-keyword">def</span> <span class="hljs-title function_">decode_one_tokens</span>(<span class="hljs-params">model, cur_token, input_pos, cache_position, past_key_values</span>):
    logits = model(
        cur_token,
        position_ids=input_pos,
        cache_position=cache_position,
        past_key_values=past_key_values,
        return_dict=<span class="hljs-literal">False</span>,
        use_cache=<span class="hljs-literal">True</span>
    )[<span class="hljs-number">0</span>]
    new_token = torch.argmax(logits[:, -<span class="hljs-number">1</span>], dim=-<span class="hljs-number">1</span>)[:, <span class="hljs-literal">None</span>]
    <span class="hljs-keyword">return</span> new_token`,wrap:!1}}),W=new I({props:{code:"ZnJvbSUyMHRvcmNoLm5uLmF0dGVudGlvbiUyMGltcG9ydCUyMFNEUEJhY2tlbmQlMkMlMjBzZHBhX2tlcm5lbCUwQSUwQWJhdGNoX3NpemUlMkMlMjBzZXFfbGVuZ3RoJTIwJTNEJTIwaW5wdXRzJTVCJTIyaW5wdXRfaWRzJTIyJTVELnNoYXBlJTBBd2l0aCUyMHRvcmNoLm5vX2dyYWQoKSUzQSUwQSUyMCUyMCUyMCUyMHBhc3Rfa2V5X3ZhbHVlcyUyMCUzRCUyMFN0YXRpY0NhY2hlKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGNvbmZpZyUzRG1vZGVsLmNvbmZpZyUyQyUyMG1heF9iYXRjaF9zaXplJTNEMiUyQyUyMG1heF9jYWNoZV9sZW4lM0Q0MDk2JTJDJTIwZGV2aWNlJTNEdG9yY2hfZGV2aWNlJTJDJTIwZHR5cGUlM0Rtb2RlbC5kdHlwZSUwQSUyMCUyMCUyMCUyMCklMEElMjAlMjAlMjAlMjBjYWNoZV9wb3NpdGlvbiUyMCUzRCUyMHRvcmNoLmFyYW5nZShzZXFfbGVuZ3RoJTJDJTIwZGV2aWNlJTNEdG9yY2hfZGV2aWNlKSUwQSUyMCUyMCUyMCUyMGdlbmVyYXRlZF9pZHMlMjAlM0QlMjB0b3JjaC56ZXJvcyglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBiYXRjaF9zaXplJTJDJTIwc2VxX2xlbmd0aCUyMCUyQiUyME5VTV9UT0tFTlNfVE9fR0VORVJBVEUlMjAlMkIlMjAxJTJDJTIwZHR5cGUlM0R0b3JjaC5pbnQlMkMlMjBkZXZpY2UlM0R0b3JjaF9kZXZpY2UlMEElMjAlMjAlMjAlMjApJTBBJTIwJTIwJTIwJTIwZ2VuZXJhdGVkX2lkcyU1QiUzQSUyQyUyMGNhY2hlX3Bvc2l0aW9uJTVEJTIwJTNEJTIwaW5wdXRzJTVCJTIyaW5wdXRfaWRzJTIyJTVELnRvKHRvcmNoX2RldmljZSkudG8odG9yY2guaW50KSUwQSUwQSUyMCUyMCUyMCUyMGxvZ2l0cyUyMCUzRCUyMG1vZGVsKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCoqaW5wdXRzJTJDJTIwY2FjaGVfcG9zaXRpb24lM0RjYWNoZV9wb3NpdGlvbiUyQyUyMHBhc3Rfa2V5X3ZhbHVlcyUzRHBhc3Rfa2V5X3ZhbHVlcyUyQ3JldHVybl9kaWN0JTNERmFsc2UlMkMlMjB1c2VfY2FjaGUlM0RUcnVlJTBBJTIwJTIwJTIwJTIwKSU1QjAlNUQlMEElMjAlMjAlMjAlMjBuZXh0X3Rva2VuJTIwJTNEJTIwdG9yY2guYXJnbWF4KGxvZ2l0cyU1QiUzQSUyQyUyMC0xJTVEJTJDJTIwZGltJTNELTEpJTVCJTNBJTJDJTIwTm9uZSU1RCUwQSUyMCUyMCUyMCUyMGdlbmVyYXRlZF9pZHMlNUIlM0ElMkMlMjBzZXFfbGVuZ3RoJTVEJTIwJTNEJTIwbmV4dF90b2tlbiU1QiUzQSUyQyUyMDAlNUQlMEElMEElMjAlMjAlMjAlMjBkZWNvZGVfb25lX3Rva2VucyUyMCUzRCUyMHRvcmNoLmNvbXBpbGUoZGVjb2RlX29uZV90b2tlbnMlMkMlMjBtb2RlJTNEJTIycmVkdWNlLW92ZXJoZWFkJTIyJTJDJTIwZnVsbGdyYXBoJTNEVHJ1ZSklMEElMjAlMjAlMjAlMjBjYWNoZV9wb3NpdGlvbiUyMCUzRCUyMHRvcmNoLnRlbnNvciglNUJzZXFfbGVuZ3RoJTIwJTJCJTIwMSU1RCUyQyUyMGRldmljZSUzRHRvcmNoX2RldmljZSklMEElMjAlMjAlMjAlMjBmb3IlMjBfJTIwaW4lMjByYW5nZSgxJTJDJTIwTlVNX1RPS0VOU19UT19HRU5FUkFURSklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3aXRoJTIwc2RwYV9rZXJuZWwoU0RQQmFja2VuZC5NQVRIKSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG5leHRfdG9rZW4lMjAlM0QlMjBkZWNvZGVfb25lX3Rva2Vucyhtb2RlbCUyQyUyMG5leHRfdG9rZW4uY2xvbmUoKSUyQyUyME5vbmUlMkMlMjBjYWNoZV9wb3NpdGlvbiUyQyUyMHBhc3Rfa2V5X3ZhbHVlcyklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBnZW5lcmF0ZWRfaWRzJTVCJTNBJTJDJTIwY2FjaGVfcG9zaXRpb24lNUQlMjAlM0QlMjBuZXh0X3Rva2VuLmludCgpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY2FjaGVfcG9zaXRpb24lMjAlMkIlM0QlMjAxJTBBJTBBdGV4dCUyMCUzRCUyMHRva2VuaXplci5iYXRjaF9kZWNvZGUoZ2VuZXJhdGVkX2lkcyUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSUwQXRleHQlMEElNUInU2ltcGx5JTIwcHV0JTJDJTIwdGhlJTIwdGhlb3J5JTIwb2YlMjByZWxhdGl2aXR5JTIwc3RhdGVzJTIwdGhhdCUyMDEpJTIwdGhlJTIwc3BlZWQlMjBvZiUyMGxpZ2h0JTIwaXMlMjBjb25zdGFudCUyQyUyMDIpJTIwdGhlJTIwc3BlZWQlMjBvZiUyMGxpZ2h0JTIwaXMlMjB0aGUlMjBzYW1lJTIwZm9yJTIwYWxsJTIwb2JzZXJ2ZXJzJTJDJTIwYW5kJTIwMyklMjB0aGUlMjBsYXdzJTIwb2YlMjBwaHlzaWNzJTIwYXJlJTIwdGhlJTIwc2FtZSUyMGZvciUyMGFsbCUyMG9ic2VydmVycy4nJTJDJTBBJTIwJ015JTIwZmF2b3JpdGUlMjBhbGwlMjB0aW1lJTIwZmF2b3JpdGUlMjBjb25kaW1lbnQlMjBpcyUyMGtldGNodXAuJTIwSSUyMGxvdmUlMjBpdCUyMG9uJTIwZXZlcnl0aGluZy4lMjBJJTIwbG92ZSUyMGl0JTIwb24lMjBteSUyMGVnZ3MlMkMlMjBteSUyMGZyaWVzJTJDJTIwbXklMjBjaGlja2VuJTJDJTIwbXklMjBidXJnZXJzJTJDJTIwbXklMjBob3QlMjBkb2dzJTJDJTIwbXklMjBzYW5kd2ljaGVzJTJDJTIwbXklMjBzYWxhZHMlMkMlMjBteSUyMHAnJTVE",highlighted:`<span class="hljs-keyword">from</span> torch.nn.attention <span class="hljs-keyword">import</span> SDPBackend, sdpa_kernel

batch_size, seq_length = inputs[<span class="hljs-string">&quot;input_ids&quot;</span>].shape
<span class="hljs-keyword">with</span> torch.no_grad():
    past_key_values = StaticCache(
        config=model.config, max_batch_size=<span class="hljs-number">2</span>, max_cache_len=<span class="hljs-number">4096</span>, device=torch_device, dtype=model.dtype
    )
    cache_position = torch.arange(seq_length, device=torch_device)
    generated_ids = torch.zeros(
        batch_size, seq_length + NUM_TOKENS_TO_GENERATE + <span class="hljs-number">1</span>, dtype=torch.<span class="hljs-built_in">int</span>, device=torch_device
    )
    generated_ids[:, cache_position] = inputs[<span class="hljs-string">&quot;input_ids&quot;</span>].to(torch_device).to(torch.<span class="hljs-built_in">int</span>)

    logits = model(
        **inputs, cache_position=cache_position, past_key_values=past_key_values,return_dict=<span class="hljs-literal">False</span>, use_cache=<span class="hljs-literal">True</span>
    )[<span class="hljs-number">0</span>]
    next_token = torch.argmax(logits[:, -<span class="hljs-number">1</span>], dim=-<span class="hljs-number">1</span>)[:, <span class="hljs-literal">None</span>]
    generated_ids[:, seq_length] = next_token[:, <span class="hljs-number">0</span>]

    decode_one_tokens = torch.<span class="hljs-built_in">compile</span>(decode_one_tokens, mode=<span class="hljs-string">&quot;reduce-overhead&quot;</span>, fullgraph=<span class="hljs-literal">True</span>)
    cache_position = torch.tensor([seq_length + <span class="hljs-number">1</span>], device=torch_device)
    <span class="hljs-keyword">for</span> _ <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">1</span>, NUM_TOKENS_TO_GENERATE):
        <span class="hljs-keyword">with</span> sdpa_kernel(SDPBackend.MATH):
            next_token = decode_one_tokens(model, next_token.clone(), <span class="hljs-literal">None</span>, cache_position, past_key_values)
            generated_ids[:, cache_position] = next_token.<span class="hljs-built_in">int</span>()
        cache_position += <span class="hljs-number">1</span>

text = tokenizer.batch_decode(generated_ids, skip_special_tokens=<span class="hljs-literal">True</span>)
text
[<span class="hljs-string">&#x27;Simply put, the theory of relativity states that 1) the speed of light is constant, 2) the speed of light is the same for all observers, and 3) the laws of physics are the same for all observers.&#x27;</span>,
 <span class="hljs-string">&#x27;My favorite all time favorite condiment is ketchup. I love it on everything. I love it on my eggs, my fries, my chicken, my burgers, my hot dogs, my sandwiches, my salads, my p&#x27;</span>]`,wrap:!1}}),{c(){t=u("p"),t.innerHTML=M,s=p(),J(r.$$.fragment),o=p(),J(i.$$.fragment),m=p(),Z=u("p"),Z.innerHTML=k,v=p(),J(_.$$.fragment),d=p(),G=u("p"),G.innerHTML=Ce,z=p(),C=u("ol"),C.innerHTML=P,X=p(),J(W.$$.fragment)},l(y){t=h(y,"P",{"data-svelte-h":!0}),f(t)!=="svelte-3zw9rq"&&(t.innerHTML=M),s=c(y),T(r.$$.fragment,y),o=c(y),T(i.$$.fragment,y),m=c(y),Z=h(y,"P",{"data-svelte-h":!0}),f(Z)!=="svelte-1gz44wn"&&(Z.innerHTML=k),v=c(y),T(_.$$.fragment,y),d=c(y),G=h(y,"P",{"data-svelte-h":!0}),f(G)!=="svelte-1g8sj7q"&&(G.innerHTML=Ce),z=c(y),C=h(y,"OL",{"data-svelte-h":!0}),f(C)!=="svelte-iqu6q8"&&(C.innerHTML=P),X=c(y),T(W.$$.fragment,y)},m(y,B){n(y,t,B),n(y,s,B),U(r,y,B),n(y,o,B),U(i,y,B),n(y,m,B),n(y,Z,B),n(y,v,B),U(_,y,B),n(y,d,B),n(y,G,B),n(y,z,B),n(y,C,B),n(y,X,B),U(W,y,B),V=!0},p(y,B){const R={};B&2&&(R.$$scope={dirty:B,ctx:y}),i.$set(R)},i(y){V||(b(r.$$.fragment,y),b(i.$$.fragment,y),b(_.$$.fragment,y),b(W.$$.fragment,y),V=!0)},o(y){w(r.$$.fragment,y),w(i.$$.fragment,y),w(_.$$.fragment,y),w(W.$$.fragment,y),V=!1},d(y){y&&(a(t),a(s),a(o),a(m),a(Z),a(v),a(d),a(G),a(z),a(C),a(X)),j(r,y),j(i,y),j(_,y),j(W,y)}}}function ul(g){let t,M='Compiling the entire <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a> function also compiles the input preparation logit processor operations, and more, in addition to the forward pass. With this approach, you don’t need to initialize <a href="/docs/transformers/main/en/internal/generation_utils#transformers.StaticCache">StaticCache</a> or set the <a href="https://hf.co/docs/transformers/main_classes/text_generation#transformers.GenerationConfig.cache_implementation" rel="nofollow">cache_implementation</a> parameter.',s,r,o,i,m="This usage pattern is more appropriate for unique hardware or use cases, but there are several drawbacks to consider.",Z,k,v='<li>Compilation is much slower.</li> <li>Parameters must be configured through <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationConfig">GenerationConfig</a>.</li> <li>Many warnings and exceptions are suppressed. We recommend testing the uncompiled model first.</li> <li>Many features are unavailable at the moment. For example, generation does not stop if an <code>EOS</code> token is selected.</li>',_;return r=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQWltcG9ydCUyMHRvcmNoJTBBaW1wb3J0JTIwb3MlMEFvcy5lbnZpcm9uJTVCJTIyVE9LRU5JWkVSU19QQVJBTExFTElTTSUyMiU1RCUyMCUzRCUyMCUyMmZhbHNlJTIyJTIwJTIwJTIzJTIwVG8lMjBwcmV2ZW50JTIwbG9uZyUyMHdhcm5pbmdzJTIwJTNBKSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmdlbW1hLTJiJTIyKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmdlbW1hLTJiJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0QlMjJhdXRvJTIyJTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIpJTBBJTBBbW9kZWwuZ2VuZXJhdGUlMjAlM0QlMjB0b3JjaC5jb21waWxlKG1vZGVsLmdlbmVyYXRlJTJDJTIwbW9kZSUzRCUyMnJlZHVjZS1vdmVyaGVhZCUyMiUyQyUyMGZ1bGxncmFwaCUzRFRydWUpJTBBaW5wdXRfdGV4dCUyMCUzRCUyMCUyMlRoZSUyMHRoZW9yeSUyMG9mJTIwc3BlY2lhbCUyMHJlbGF0aXZpdHklMjBzdGF0ZXMlMjAlMjIlMEFpbnB1dF9pZHMlMjAlM0QlMjB0b2tlbml6ZXIoaW5wdXRfdGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIpLnRvKG1vZGVsLmRldmljZS50eXBlKSUwQSUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0X2lkcyklMEFwcmludCh0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKG91dHB1dHMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSkpJTBBJTVCJ1RoZSUyMHRoZW9yeSUyMG9mJTIwc3BlY2lhbCUyMHJlbGF0aXZpdHklMjBzdGF0ZXMlMjAxLiUyMFRoZSUyMHNwZWVkJTIwb2YlMjBsaWdodCUyMGlzJTIwY29uc3RhbnQlMjBpbiUyMGFsbCUyMGluZXJ0aWFsJTIwcmVmZXJlbmNlJyU1RA==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> os
os.environ[<span class="hljs-string">&quot;TOKENIZERS_PARALLELISM&quot;</span>] = <span class="hljs-string">&quot;false&quot;</span>  <span class="hljs-comment"># To prevent long warnings :)</span>

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;google/gemma-2b&quot;</span>)
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;google/gemma-2b&quot;</span>, torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>, device_map=<span class="hljs-string">&quot;auto&quot;</span>)

model.generate = torch.<span class="hljs-built_in">compile</span>(model.generate, mode=<span class="hljs-string">&quot;reduce-overhead&quot;</span>, fullgraph=<span class="hljs-literal">True</span>)
input_text = <span class="hljs-string">&quot;The theory of special relativity states &quot;</span>
input_ids = tokenizer(input_text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device.<span class="hljs-built_in">type</span>)

outputs = model.generate(**input_ids)
<span class="hljs-built_in">print</span>(tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>))
[<span class="hljs-string">&#x27;The theory of special relativity states 1. The speed of light is constant in all inertial reference&#x27;</span>]`,wrap:!1}}),{c(){t=u("p"),t.innerHTML=M,s=p(),J(r.$$.fragment),o=p(),i=u("p"),i.textContent=m,Z=p(),k=u("ol"),k.innerHTML=v},l(d){t=h(d,"P",{"data-svelte-h":!0}),f(t)!=="svelte-1tio1av"&&(t.innerHTML=M),s=c(d),T(r.$$.fragment,d),o=c(d),i=h(d,"P",{"data-svelte-h":!0}),f(i)!=="svelte-3z6n1l"&&(i.textContent=m),Z=c(d),k=h(d,"OL",{"data-svelte-h":!0}),f(k)!=="svelte-djx6ec"&&(k.innerHTML=v)},m(d,G){n(d,t,G),n(d,s,G),U(r,d,G),n(d,o,G),n(d,i,G),n(d,Z,G),n(d,k,G),_=!0},p:$,i(d){_||(b(r.$$.fragment,d),_=!0)},o(d){w(r.$$.fragment,d),_=!1},d(d){d&&(a(t),a(s),a(o),a(i),a(Z),a(k)),j(r,d)}}}function hl(g){let t,M,s,r,o,i;return t=new Be({props:{id:"static-kv",option:"1. cache_implementation",$$slots:{default:[ml]},$$scope:{ctx:g}}}),s=new Be({props:{id:"static-kv",option:"2. StaticCache",$$slots:{default:[yl]},$$scope:{ctx:g}}}),o=new Be({props:{id:"static-kv",option:"3. compile entire generate function",$$slots:{default:[ul]},$$scope:{ctx:g}}}),{c(){J(t.$$.fragment),M=p(),J(s.$$.fragment),r=p(),J(o.$$.fragment)},l(m){T(t.$$.fragment,m),M=c(m),T(s.$$.fragment,m),r=c(m),T(o.$$.fragment,m)},m(m,Z){U(t,m,Z),n(m,M,Z),U(s,m,Z),n(m,r,Z),U(o,m,Z),i=!0},p(m,Z){const k={};Z&2&&(k.$$scope={dirty:Z,ctx:m}),t.$set(k);const v={};Z&2&&(v.$$scope={dirty:Z,ctx:m}),s.$set(v);const _={};Z&2&&(_.$$scope={dirty:Z,ctx:m}),o.$set(_)},i(m){i||(b(t.$$.fragment,m),b(s.$$.fragment,m),b(o.$$.fragment,m),i=!0)},o(m){w(t.$$.fragment,m),w(s.$$.fragment,m),w(o.$$.fragment,m),i=!1},d(m){m&&(a(M),a(r)),j(t,m),j(s,m),j(o,m)}}}function fl(g){let t,M='For a more in-depth explanation, take a look at the <a href="https://hf.co/blog/assisted-generation" rel="nofollow">Assisted Generation: a new direction toward low-latency text generation</a> blog post!';return{c(){t=u("p"),t.innerHTML=M},l(s){t=h(s,"P",{"data-svelte-h":!0}),f(t)!=="svelte-112w74b"&&(t.innerHTML=M)},m(s,r){n(s,t,r)},p:$,d(s){s&&a(t)}}}function Jl(g){let t,M="Speculative decoding is only supported for the greedy search and sampling decoding strategies, and it doesn’t support batched inputs.";return{c(){t=u("p"),t.textContent=M},l(s){t=h(s,"P",{"data-svelte-h":!0}),f(t)!=="svelte-1wfld0h"&&(t.textContent=M)},m(s,r){n(s,t,r)},p:$,d(s){s&&a(t)}}}function Tl(g){let t,M;return t=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQWltcG9ydCUyMHRvcmNoJTBBZnJvbSUyMGFjY2VsZXJhdGUudGVzdF91dGlscy50ZXN0aW5nJTIwaW1wb3J0JTIwZ2V0X2JhY2tlbmQlMEElMEFkZXZpY2UlMkMlMjBfJTJDJTIwXyUyMCUzRCUyMGdldF9iYWNrZW5kKCklMjAlMjMlMjBhdXRvbWF0aWNhbGx5JTIwZGV0ZWN0cyUyMHRoZSUyMHVuZGVybHlpbmclMjBkZXZpY2UlMjB0eXBlJTIwKENVREElMkMlMjBDUFUlMkMlMjBYUFUlMkMlMjBNUFMlMkMlMjBldGMuKSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmZhY2Vib29rJTJGb3B0LTEuM2IlMjIpJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKCUyMkVpbnN0ZWluJ3MlMjB0aGVvcnklMjBvZiUyMHJlbGF0aXZpdHklMjBzdGF0ZXMlMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhkZXZpY2UpJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZvcHQtMS4zYiUyMiUyQyUyMHRvcmNoX2R0eXBlJTNEJTIyYXV0byUyMikudG8oZGV2aWNlKSUwQWFzc2lzdGFudF9tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRm9wdC0xMjVtJTIyKS50byhkZXZpY2UpJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzJTJDJTIwYXNzaXN0YW50X21vZGVsJTNEYXNzaXN0YW50X21vZGVsKSUwQXRva2VuaXplci5iYXRjaF9kZWNvZGUob3V0cHV0cyUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSUwQSU1QiUyMkVpbnN0ZWluJ3MlMjB0aGVvcnklMjBvZiUyMHJlbGF0aXZpdHklMjBzdGF0ZXMlMjB0aGF0JTIwdGhlJTIwc3BlZWQlMjBvZiUyMGxpZ2h0JTIwaXMlMjBjb25zdGFudC4lMjAlMjAlMjAlMjAlMjIlNUQ=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> accelerate.test_utils.testing <span class="hljs-keyword">import</span> get_backend

device, _, _ = get_backend() <span class="hljs-comment"># automatically detects the underlying device type (CUDA, CPU, XPU, MPS, etc.)</span>

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;facebook/opt-1.3b&quot;</span>)
inputs = tokenizer(<span class="hljs-string">&quot;Einstein&#x27;s theory of relativity states&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(device)

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-1.3b&quot;</span>, torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>).to(device)
assistant_model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-125m&quot;</span>).to(device)
outputs = model.generate(**inputs, assistant_model=assistant_model)
tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>)
[<span class="hljs-string">&quot;Einstein&#x27;s theory of relativity states that the speed of light is constant.    &quot;</span>]`,wrap:!1}}),{c(){J(t.$$.fragment)},l(s){T(t.$$.fragment,s)},m(s,r){U(t,s,r),M=!0},p:$,i(s){M||(b(t.$$.fragment,s),M=!0)},o(s){w(t.$$.fragment,s),M=!1},d(s){j(t,s)}}}function Ul(g){let t,M='For speculative sampling decoding, add the <a href="https://hf.co/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationConfig.do_sample" rel="nofollow">do_sample</a> and <a href="https://hf.co/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationConfig.temperature" rel="nofollow">temperature</a> parameters to <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a>.',s,r,o;return r=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQWltcG9ydCUyMHRvcmNoJTBBZnJvbSUyMGFjY2VsZXJhdGUudGVzdF91dGlscy50ZXN0aW5nJTIwaW1wb3J0JTIwZ2V0X2JhY2tlbmQlMEElMEFkZXZpY2UlMkMlMjBfJTJDJTIwXyUyMCUzRCUyMGdldF9iYWNrZW5kKCklMjAlMjMlMjBhdXRvbWF0aWNhbGx5JTIwZGV0ZWN0cyUyMHRoZSUyMHVuZGVybHlpbmclMjBkZXZpY2UlMjB0eXBlJTIwKENVREElMkMlMjBDUFUlMkMlMjBYUFUlMkMlMjBNUFMlMkMlMjBldGMuKSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmZhY2Vib29rJTJGb3B0LTEuM2IlMjIpJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKCUyMkVpbnN0ZWluJ3MlMjB0aGVvcnklMjBvZiUyMHJlbGF0aXZpdHklMjBzdGF0ZXMlMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhkZXZpY2UpJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZvcHQtMS4zYiUyMiUyQyUyMHRvcmNoX2R0eXBlJTNEJTIyYXV0byUyMikudG8oZGV2aWNlKSUwQWFzc2lzdGFudF9tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRm9wdC0xMjVtJTIyKS50byhkZXZpY2UpJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzJTJDJTIwYXNzaXN0YW50X21vZGVsJTNEYXNzaXN0YW50X21vZGVsJTJDJTIwZG9fc2FtcGxlJTNEVHJ1ZSUyQyUyMHRlbXBlcmF0dXJlJTNEMC43KSUwQXByaW50KHRva2VuaXplci5iYXRjaF9kZWNvZGUob3V0cHV0cyUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSklMEElNUIlMjJFaW5zdGVpbidzJTIwdGhlb3J5JTIwb2YlMjByZWxhdGl2aXR5JTIwc3RhdGVzJTIwdGhhdCUyMG1vdGlvbiUyMGluJTIwdGhlJTIwdW5pdmVyc2UlMjBpcyUyMG5vdCUyMGElMjBzdHJhaWdodCUyMGxpbmUuJTVDbiUyMiU1RA==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> accelerate.test_utils.testing <span class="hljs-keyword">import</span> get_backend

device, _, _ = get_backend() <span class="hljs-comment"># automatically detects the underlying device type (CUDA, CPU, XPU, MPS, etc.)</span>

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;facebook/opt-1.3b&quot;</span>)
inputs = tokenizer(<span class="hljs-string">&quot;Einstein&#x27;s theory of relativity states&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(device)

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-1.3b&quot;</span>, torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>).to(device)
assistant_model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-125m&quot;</span>).to(device)
outputs = model.generate(**inputs, assistant_model=assistant_model, do_sample=<span class="hljs-literal">True</span>, temperature=<span class="hljs-number">0.7</span>)
<span class="hljs-built_in">print</span>(tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>))
[<span class="hljs-string">&quot;Einstein&#x27;s theory of relativity states that motion in the universe is not a straight line.\\n&quot;</span>]`,wrap:!1}}),{c(){t=u("p"),t.innerHTML=M,s=p(),J(r.$$.fragment)},l(i){t=h(i,"P",{"data-svelte-h":!0}),f(t)!=="svelte-nx7l91"&&(t.innerHTML=M),s=c(i),T(r.$$.fragment,i)},m(i,m){n(i,t,m),n(i,s,m),U(r,i,m),o=!0},p:$,i(i){o||(b(r.$$.fragment,i),o=!0)},o(i){w(r.$$.fragment,i),o=!1},d(i){i&&(a(t),a(s)),j(r,i)}}}function bl(g){let t,M,s,r;return t=new Be({props:{id:"spec-decoding",option:"greedy search",$$slots:{default:[Tl]},$$scope:{ctx:g}}}),s=new Be({props:{id:"spec-decoding",option:"sampling",$$slots:{default:[Ul]},$$scope:{ctx:g}}}),{c(){J(t.$$.fragment),M=p(),J(s.$$.fragment)},l(o){T(t.$$.fragment,o),M=c(o),T(s.$$.fragment,o)},m(o,i){U(t,o,i),n(o,M,i),U(s,o,i),r=!0},p(o,i){const m={};i&2&&(m.$$scope={dirty:i,ctx:o}),t.$set(m);const Z={};i&2&&(Z.$$scope={dirty:i,ctx:o}),s.$set(Z)},i(o){r||(b(t.$$.fragment,o),b(s.$$.fragment,o),r=!0)},o(o){w(t.$$.fragment,o),w(s.$$.fragment,o),r=!1},d(o){o&&a(M),j(t,o),j(s,o)}}}function wl(g){let t,M;return t=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQWltcG9ydCUyMHRvcmNoJTBBZnJvbSUyMGFjY2VsZXJhdGUudGVzdF91dGlscy50ZXN0aW5nJTIwaW1wb3J0JTIwZ2V0X2JhY2tlbmQlMEElMEFkZXZpY2UlMkMlMjBfJTJDJTIwXyUyMCUzRCUyMGdldF9iYWNrZW5kKCklMjAlMjMlMjBhdXRvbWF0aWNhbGx5JTIwZGV0ZWN0cyUyMHRoZSUyMHVuZGVybHlpbmclMjBkZXZpY2UlMjB0eXBlJTIwKENVREElMkMlMjBDUFUlMkMlMjBYUFUlMkMlMjBNUFMlMkMlMjBldGMuKSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmZhY2Vib29rJTJGb3B0LTEuM2IlMjIpJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKCUyMlRoZSUyMHNlY29uZCUyMGxhdyUyMG9mJTIwdGhlcm1vZHluYW1pY3MlMjBzdGF0ZXMlMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhkZXZpY2UpJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZvcHQtMS4zYiUyMiUyQyUyMHRvcmNoX2R0eXBlJTNEJTIyYXV0byUyMikudG8oZGV2aWNlKSUwQWFzc2lzdGFudF9tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJmYWNlYm9vayUyRm9wdC0xMjVtJTIyKS50byhkZXZpY2UpJTBBb3V0cHV0cyUyMCUzRCUyMG1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzJTJDJTIwcHJvbXB0X2xvb2t1cF9udW1fdG9rZW5zJTNEMyklMEFwcmludCh0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKG91dHB1dHMlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSkpJTBBJTVCJ1RoZSUyMHNlY29uZCUyMGxhdyUyMG9mJTIwdGhlcm1vZHluYW1pY3MlMjBzdGF0ZXMlMjB0aGF0JTIwZW50cm9weSUyMGluY3JlYXNlcyUyMHdpdGglMjB0ZW1wZXJhdHVyZS4lMjAlMjAlMjAlMjAlMjAlMjAnJTVE",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> accelerate.test_utils.testing <span class="hljs-keyword">import</span> get_backend

device, _, _ = get_backend() <span class="hljs-comment"># automatically detects the underlying device type (CUDA, CPU, XPU, MPS, etc.)</span>

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;facebook/opt-1.3b&quot;</span>)
inputs = tokenizer(<span class="hljs-string">&quot;The second law of thermodynamics states&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(device)

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-1.3b&quot;</span>, torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>).to(device)
assistant_model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-125m&quot;</span>).to(device)
outputs = model.generate(**inputs, prompt_lookup_num_tokens=<span class="hljs-number">3</span>)
<span class="hljs-built_in">print</span>(tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>))
[<span class="hljs-string">&#x27;The second law of thermodynamics states that entropy increases with temperature.      &#x27;</span>]`,wrap:!1}}),{c(){J(t.$$.fragment)},l(s){T(t.$$.fragment,s)},m(s,r){U(t,s,r),M=!0},p:$,i(s){M||(b(t.$$.fragment,s),M=!0)},o(s){w(t.$$.fragment,s),M=!1},d(s){j(t,s)}}}function jl(g){let t,M='For prompt lookup decoding with sampling, add the <a href="https://hf.co/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationConfig.do_sample" rel="nofollow">do_sample</a> and <a href="https://hf.co/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationConfig.temperature" rel="nofollow">temperature</a> parameters to <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a>.',s,r,o;return r=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUwQWltcG9ydCUyMHRvcmNoJTBBZnJvbSUyMGFjY2VsZXJhdGUudGVzdF91dGlscy50ZXN0aW5nJTIwaW1wb3J0JTIwZ2V0X2JhY2tlbmQlMEElMEFkZXZpY2UlMkMlMjBfJTJDJTIwXyUyMCUzRCUyMGdldF9iYWNrZW5kKCklMjAlMjMlMjBhdXRvbWF0aWNhbGx5JTIwZGV0ZWN0cyUyMHRoZSUyMHVuZGVybHlpbmclMjBkZXZpY2UlMjB0eXBlJTIwKENVREElMkMlMjBDUFUlMkMlMjBYUFUlMkMlMjBNUFMlMkMlMjBldGMuKSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmZhY2Vib29rJTJGb3B0LTEuM2IlMjIpJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKCUyMlRoZSUyMHNlY29uZCUyMGxhdyUyMG9mJTIwdGhlcm1vZHluYW1pY3MlMjBzdGF0ZXMlMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhkZXZpY2UpJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTIyZmFjZWJvb2slMkZvcHQtMS4zYiUyMiUyQyUyMHRvcmNoX2R0eXBlJTNEJTIyYXV0byUyMikudG8oZGV2aWNlKSUwQW91dHB1dHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0cyUyQyUyMHByb21wdF9sb29rdXBfbnVtX3Rva2VucyUzRDMlMkMlMjBkb19zYW1wbGUlM0RUcnVlJTJDJTIwdGVtcGVyYXR1cmUlM0QwLjcpJTBBcHJpbnQodG9rZW5pemVyLmJhdGNoX2RlY29kZShvdXRwdXRzJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpKSUwQSU1QiUyMlRoZSUyMHNlY29uZCUyMGxhdyUyMG9mJTIwdGhlcm1vZHluYW1pY3MlMjBzdGF0ZXMlMjB0aGF0JTIwZW5lcmd5JTIwY2Fubm90JTIwYmUlMjBjcmVhdGVkJTIwbm9yJTIwZGVzdHJveWVkLiUyMEl0J3MlMjBub3QlMjBhJTIyJTVE",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> accelerate.test_utils.testing <span class="hljs-keyword">import</span> get_backend

device, _, _ = get_backend() <span class="hljs-comment"># automatically detects the underlying device type (CUDA, CPU, XPU, MPS, etc.)</span>

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;facebook/opt-1.3b&quot;</span>)
inputs = tokenizer(<span class="hljs-string">&quot;The second law of thermodynamics states&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(device)

model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;facebook/opt-1.3b&quot;</span>, torch_dtype=<span class="hljs-string">&quot;auto&quot;</span>).to(device)
outputs = model.generate(**inputs, prompt_lookup_num_tokens=<span class="hljs-number">3</span>, do_sample=<span class="hljs-literal">True</span>, temperature=<span class="hljs-number">0.7</span>)
<span class="hljs-built_in">print</span>(tokenizer.batch_decode(outputs, skip_special_tokens=<span class="hljs-literal">True</span>))
[<span class="hljs-string">&quot;The second law of thermodynamics states that energy cannot be created nor destroyed. It&#x27;s not a&quot;</span>]`,wrap:!1}}),{c(){t=u("p"),t.innerHTML=M,s=p(),J(r.$$.fragment)},l(i){t=h(i,"P",{"data-svelte-h":!0}),f(t)!=="svelte-1uj7i8u"&&(t.innerHTML=M),s=c(i),T(r.$$.fragment,i)},m(i,m){n(i,t,m),n(i,s,m),U(r,i,m),o=!0},p:$,i(i){o||(b(r.$$.fragment,i),o=!0)},o(i){w(r.$$.fragment,i),o=!1},d(i){i&&(a(t),a(s)),j(r,i)}}}function Zl(g){let t,M,s,r;return t=new Be({props:{id:"pld",option:"greedy decoding",$$slots:{default:[wl]},$$scope:{ctx:g}}}),s=new Be({props:{id:"pld",option:"sampling",$$slots:{default:[jl]},$$scope:{ctx:g}}}),{c(){J(t.$$.fragment),M=p(),J(s.$$.fragment)},l(o){T(t.$$.fragment,o),M=c(o),T(s.$$.fragment,o)},m(o,i){U(t,o,i),n(o,M,i),U(s,o,i),r=!0},p(o,i){const m={};i&2&&(m.$$scope={dirty:i,ctx:o}),t.$set(m);const Z={};i&2&&(Z.$$scope={dirty:i,ctx:o}),s.$set(Z)},i(o){r||(b(t.$$.fragment,o),b(s.$$.fragment,o),r=!0)},o(o){w(t.$$.fragment,o),w(s.$$.fragment,o),r=!1},d(o){o&&a(M),j(t,o),j(s,o)}}}function gl(g){let t,M="SDPA automaticallysupports FlashAttention-2 as long as you have the latest PyTorch version installed.";return{c(){t=u("p"),t.textContent=M},l(s){t=h(s,"P",{"data-svelte-h":!0}),f(t)!=="svelte-19scp3k"&&(t.textContent=M)},m(s,r){n(s,t,r)},p:$,d(s){s&&a(t)}}}function Gl(g){let t,M='There are many quantization libraries (see the <a href="./quantization">Quantization</a> guide for more details) available, such as Quanto, AQLM, VPTQ, AWQ, and AutoGPTQ. Feel free to try them out and see which one works best for your use case. We also recommend reading the <a href="https://hf.co/blog/overview-quantization-transformers" rel="nofollow">Overview of natively supported quantization schemes in 🤗 Transformers</a> blog post which compares AutoGPTQ and bitsandbytes.';return{c(){t=u("p"),t.innerHTML=M},l(s){t=h(s,"P",{"data-svelte-h":!0}),f(t)!=="svelte-p1oqjt"&&(t.innerHTML=M)},m(s,r){n(s,t,r)},p:$,d(s){s&&a(t)}}}function kl(g){let t,M,s,r,o,i,m,Z='Inference with large language models (LLMs) can be challenging because they have to store and handle billions of parameters. To load a 70B parameter <a href="https://hf.co/meta-llama/Llama-2-70b-hf" rel="nofollow">Llama 2</a> model, it requires 256GB of memory for full precision weights and 128GB of memory for half-precision weights. The most powerful GPUs today - the A100 and H100 - only have 80GB of memory.',k,v,_="On top of the memory requirements, inference is slow because LLMs are called repeatedly to generate the next token. The input sequence increases as generation progresses, which takes longer and longer to process.",d,G,Ce="This guide will show you how to optimize LLM inference to accelerate generation and reduce memory usage.",z,C,P,X,W,V,y="LLMs compute key-value (kv) values for each input token, and it performs the same kv computation each time because the generated output becomes part of the input. However, performing the same kv computation every time is not very efficient.",B,R,Zt='A <em>kv-cache</em> stores the past keys and values instead of recomputing them each time. As a result, the kv-cache is dynamic and it grows with each generation step which prevents you from taking advantage of <a href="./perf_torch_compile">torch.compile</a>, a powerful optimization method that fuses PyTorch code into optimized kernels.',Ie,K,gt='The <em>static kv-cache</em> solves this issue by pre-allocating the kv-cache size to a maximum value, so you can combine it with <a href="./perf_torch_compile">torch.compile</a> for up to a 4x speed up. Your speed up may vary depending on the model size (larger models have a smaller speed up) and hardware.',Xe,H,We,O,Gt="Depending on your task, there are several ways you can use the static kv-cache.",Ve,ee,kt='<li>For basic use cases, set <a href="https://hf.co/docs/transformers/main_classes/text_generation#transformers.GenerationConfig.cache_implementation" rel="nofollow">cache_implementation</a> to <code>&quot;static&quot;</code> (recommended).</li> <li>For multi-turn generation or a custom generation loop, initialize and handle <a href="/docs/transformers/main/en/internal/generation_utils#transformers.StaticCache">StaticCache</a> directly.</li> <li>For more unique hardware or use cases, it may be better to compile the entire <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a> function into a single graph.</li>',Re,N,Fe,S,ze,te,Ye,le,_t="Decoding can also be optimized to accelerate generation. You can use a lightweight assistant model to generate candidate tokens faster than the LLM itself or you can use a variant of this decoding strategy that works especially well for input-grounded tasks.",He,se,Ne,E,Se,ae,Bt="For each input token, the model weights are loaded each time during the forward pass, which is slow and cumbersome when a model has billions of parameters. Speculative decoding alleviates this slowdown by using a second smaller and faster assistant model to generate candidate tokens that are verified by the larger model in a single forward pass. If the verified tokens are correct, the LLM essentially gets them for “free” without having to generate them itself. There is no degradation in accuracy because the verification forward pass ensures the same outputs are generated as if the LLM had generated them on its own.",Ee,ne,vt="To get the largest speed up, the assistant model should be a lot smaller than the LLM so that it can generate tokens quickly. The assistant and LLM model must also share the same tokenizer to avoid re-encoding and decoding tokens.",xe,x,Qe,oe,Ct='Enable speculative decoding by loading an assistant model and passing it to <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a>.',Ae,Q,qe,ie,Le,re,$t="Prompt lookup decoding is a variant of speculative decoding that is also compatible with greedy search and sampling. Prompt lookup works especially well for input-grounded tasks - such as summarization - where there is often overlapping words between the prompt and output. These overlapping n-grams are used as the LLM candidate tokens.",De,pe,It='To enable prompt lookup decoding, specify the number of tokens that should be overlapping in the <a href="https://hf.co/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationConfig.prompt_lookup_num_tokens" rel="nofollow">prompt_lookup_num_tokens</a> parameter. Then pass this parameter to <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a>.',Pe,A,Ke,ce,Oe,Me,Xt="A known issue with transformer models is that the self-attention mechanism grows quadratically in compute and memory with the number of input tokens. This limitation is only magnified in LLMs which handles much longer sequences. To address this, try FlashAttention2 or PyTorch’s scaled dot product attention (SDPA), which are more memory efficient attention implementations.",et,me,tt,de,Wt='FlashAttention and <a href="./perf_infer_gpu_one#flashattention-2">FlashAttention-2</a> break up the attention computation into smaller chunks and reduces the number of intermediate read/write operations to the GPU memory to speed up inference. FlashAttention-2 improves on the original FlashAttention algorithm by also parallelizing over sequence length dimension and better partitioning work on the hardware to reduce synchronization and communication overhead.',lt,ye,Vt='To use FlashAttention-2, set <a href="https://hf.co/docs/transformers/main/en/main_classes/text_generation#transformers.PreTrainedModel.from_pretrained.attn_implementation" rel="nofollow">attn_implementation</a> to <code>&quot;flash_attention_2&quot;</code> in <a href="/docs/transformers/main/en/main_classes/model#transformers.PreTrainedModel.from_pretrained">from_pretrained()</a>.',st,ue,at,he,nt,fe,Rt="Scaled dot product attention (SDPA) is automatically enabled in PyTorch 2.0 and it supports FlashAttention, xFormers, and PyTorch’s C++ implementation. SDPA chooses the most performant attention algorithm if you’re using a CUDA backend. For other backends, SDPA defaults to the PyTorch C++ implementation.",ot,q,it,Je,Ft='Use the <a href="https://pytorch.org/docs/stable/generated/torch.nn.attention.sdpa_kernel.html" rel="nofollow">torch.nn.attention.sdpa_kernel</a> context manager to explicitly enable or disable any of the four attention algorithms. For example, use <code>SDPBackend.FLASH_ATTENTION</code> to enable FlashAttention.',rt,Te,pt,Ue,ct,be,zt="Quantization reduces the size of model weights by storing them in a lower precision. This translates to lower memory usage and makes loading LLMs for inference more accessible if you’re constrained by GPU memory.",Mt,we,Yt="If you aren’t limited by your GPU, you don’t necessarily need to quantize your model because it can increase latency slightly (except for AWQ and fused AWQ modules) due to the extra step required to quantize and dequantize the weights.",mt,L,dt,je,Ht='Use the Model Memory Calculator below to estimate and compare how much memory is required to load a model. For example, try estimating the memory required to load <a href="https://hf.co/mistralai/Mistral-7B-v0.1" rel="nofollow">Mistral-7B-v0.1</a>.',yt,F,Nt,ut,Ze,St='To load a model in half-precision, set the <a href="https://hf.co/docs/transformers/main/en/main_classes/text_generation#transformers.PreTrainedModel.from_pretrained.torch_dtype" rel="nofollow">torch_dtype</a> parameter in <a href="/docs/transformers/main/en/model_doc/auto#transformers.AutoModel.from_pretrained">from_pretrained()</a> to <code>torch.bfloat16</code>. This requires 13.74GB of memory.',ht,ge,ft,Ge,Et='To load a quantized model (8-bit or 4-bit), try <a href="https://hf.co/docs/bitsandbytes" rel="nofollow">bitsandbytes</a> and set the <a href="https://hf.co/docs/transformers/main/en/main_classes/text_generation#transformers.BitsAndBytesConfig.load_in_4bit" rel="nofollow">load_in_4bit</a> or <a href="https://hf.co/docs/transformers/main/en/main_classes/text_generation#transformers.BitsAndBytesConfig.load_in_8bit" rel="nofollow">load_in_8bit</a> parameters to <code>True</code>. Loading the model in 8-bits only requires 6.87 GB of memory.',Jt,ke,Tt,_e,Ut,$e,bt;return o=new Y({props:{title:"Optimizing inference",local:"optimizing-inference",headingTag:"h1"}}),C=new D({props:{warning:!1,$$slots:{default:[pl]},$$scope:{ctx:g}}}),X=new Y({props:{title:"Static kv-cache and torch.compile",local:"static-kv-cache-and-torchcompile",headingTag:"h2"}}),H=new D({props:{warning:!0,$$slots:{default:[cl]},$$scope:{ctx:g}}}),N=new D({props:{warning:!1,$$slots:{default:[Ml]},$$scope:{ctx:g}}}),S=new jt({props:{id:"static-kv",options:["1. cache_implementation","2. StaticCache","3. compile entire generate function"],$$slots:{default:[hl]},$$scope:{ctx:g}}}),te=new Y({props:{title:"Decoding strategies",local:"decoding-strategies",headingTag:"h2"}}),se=new Y({props:{title:"Speculative decoding",local:"speculative-decoding",headingTag:"h3"}}),E=new D({props:{warning:!1,$$slots:{default:[fl]},$$scope:{ctx:g}}}),x=new D({props:{warning:!0,$$slots:{default:[Jl]},$$scope:{ctx:g}}}),Q=new jt({props:{id:"spec-decoding",options:["greedy search","sampling"],$$slots:{default:[bl]},$$scope:{ctx:g}}}),ie=new Y({props:{title:"Prompt lookup decoding",local:"prompt-lookup-decoding",headingTag:"h3"}}),A=new jt({props:{id:"pld",options:["greedy decoding","sampling"],$$slots:{default:[Zl]},$$scope:{ctx:g}}}),ce=new Y({props:{title:"Attention",local:"attention",headingTag:"h2"}}),me=new Y({props:{title:"FlashAttention-2",local:"flashattention-2",headingTag:"h3"}}),ue=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyZ29vZ2xlJTJGZ2VtbWEtMmIlMjIlMkMlMEElMjAlMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEcXVhbnRfY29uZmlnJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiUyQyUwQSUyMCUyMCUyMCUyMGF0dG5faW1wbGVtZW50YXRpb24lM0QlMjJmbGFzaF9hdHRlbnRpb25fMiUyMiUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, BitsAndBytesConfig

quant_config = BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)
model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;google/gemma-2b&quot;</span>,
    quantization_config=quant_config,
    torch_dtype=torch.bfloat16,
    attn_implementation=<span class="hljs-string">&quot;flash_attention_2&quot;</span>,
)`,wrap:!1}}),he=new Y({props:{title:"PyTorch scaled dot product attention",local:"pytorch-scaled-dot-product-attention",headingTag:"h3"}}),q=new D({props:{warning:!1,$$slots:{default:[gl]},$$scope:{ctx:g}}}),Te=new I({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdG9yY2gubm4uYXR0ZW50aW9uJTIwaW1wb3J0JTIwU0RQQmFja2VuZCUyQyUyMHNkcGFfa2VybmVsJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIyZ29vZ2xlJTJGZ2VtbWEtMmIlMjIlMkMlMEElMjAlMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTJDJTBBKSUwQSUwQXdpdGglMjBzZHBhX2tlcm5lbChTRFBCYWNrZW5kLkZMQVNIX0FUVEVOVElPTiklM0ElMEElMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> torch.nn.attention <span class="hljs-keyword">import</span> SDPBackend, sdpa_kernel
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM

model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;google/gemma-2b&quot;</span>,
    torch_dtype=torch.bfloat16,
)

<span class="hljs-keyword">with</span> sdpa_kernel(SDPBackend.FLASH_ATTENTION):
    outputs = model.generate(**inputs)`,wrap:!1}}),Ue=new Y({props:{title:"Quantization",local:"quantization",headingTag:"h2"}}),L=new D({props:{warning:!1,$$slots:{default:[Gl]},$$scope:{ctx:g}}}),ge=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQWltcG9ydCUyMHRvcmNoJTBBJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIybWlzdHJhbGFpJTJGTWlzdHJhbC03Qi12MC4xJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5iZmxvYXQxNiUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTJDJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM
<span class="hljs-keyword">import</span> torch

model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;mistralai/Mistral-7B-v0.1&quot;</span>, torch_dtype=torch.bfloat16, device_map=<span class="hljs-string">&quot;auto&quot;</span>,
)`,wrap:!1}}),ke=new I({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUyQyUyMEJpdHNBbmRCeXRlc0NvbmZpZyUwQWltcG9ydCUyMHRvcmNoJTBBJTBBcXVhbnRfY29uZmlnJTIwJTNEJTIwQml0c0FuZEJ5dGVzQ29uZmlnKGxvYWRfaW5fOGJpdCUzRFRydWUpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIwJTIybWlzdHJhbGFpJTJGTWlzdHJhbC03Qi12MC4xJTIyJTJDJTIwcXVhbnRpemF0aW9uX2NvbmZpZyUzRHF1YW50X2NvbmZpZyUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyJTBBKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
<span class="hljs-keyword">import</span> torch

quant_config = BitsAndBytesConfig(load_in_8bit=<span class="hljs-literal">True</span>)
model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;mistralai/Mistral-7B-v0.1&quot;</span>, quantization_config=quant_config, device_map=<span class="hljs-string">&quot;auto&quot;</span>
)`,wrap:!1}}),_e=new rl({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/llm_optims.md"}}),{c(){t=u("meta"),M=p(),s=u("p"),r=p(),J(o.$$.fragment),i=p(),m=u("p"),m.innerHTML=Z,k=p(),v=u("p"),v.textContent=_,d=p(),G=u("p"),G.textContent=Ce,z=p(),J(C.$$.fragment),P=p(),J(X.$$.fragment),W=p(),V=u("p"),V.textContent=y,B=p(),R=u("p"),R.innerHTML=Zt,Ie=p(),K=u("p"),K.innerHTML=gt,Xe=p(),J(H.$$.fragment),We=p(),O=u("p"),O.textContent=Gt,Ve=p(),ee=u("ol"),ee.innerHTML=kt,Re=p(),J(N.$$.fragment),Fe=p(),J(S.$$.fragment),ze=p(),J(te.$$.fragment),Ye=p(),le=u("p"),le.textContent=_t,He=p(),J(se.$$.fragment),Ne=p(),J(E.$$.fragment),Se=p(),ae=u("p"),ae.textContent=Bt,Ee=p(),ne=u("p"),ne.textContent=vt,xe=p(),J(x.$$.fragment),Qe=p(),oe=u("p"),oe.innerHTML=Ct,Ae=p(),J(Q.$$.fragment),qe=p(),J(ie.$$.fragment),Le=p(),re=u("p"),re.textContent=$t,De=p(),pe=u("p"),pe.innerHTML=It,Pe=p(),J(A.$$.fragment),Ke=p(),J(ce.$$.fragment),Oe=p(),Me=u("p"),Me.textContent=Xt,et=p(),J(me.$$.fragment),tt=p(),de=u("p"),de.innerHTML=Wt,lt=p(),ye=u("p"),ye.innerHTML=Vt,st=p(),J(ue.$$.fragment),at=p(),J(he.$$.fragment),nt=p(),fe=u("p"),fe.textContent=Rt,ot=p(),J(q.$$.fragment),it=p(),Je=u("p"),Je.innerHTML=Ft,rt=p(),J(Te.$$.fragment),pt=p(),J(Ue.$$.fragment),ct=p(),be=u("p"),be.textContent=zt,Mt=p(),we=u("p"),we.textContent=Yt,mt=p(),J(L.$$.fragment),dt=p(),je=u("p"),je.innerHTML=Ht,yt=p(),F=u("iframe"),ut=p(),Ze=u("p"),Ze.innerHTML=St,ht=p(),J(ge.$$.fragment),ft=p(),Ge=u("p"),Ge.innerHTML=Et,Jt=p(),J(ke.$$.fragment),Tt=p(),J(_e.$$.fragment),Ut=p(),$e=u("p"),this.h()},l(e){const l=ol("svelte-u9bgzb",document.head);t=h(l,"META",{name:!0,content:!0}),l.forEach(a),M=c(e),s=h(e,"P",{}),wt(s).forEach(a),r=c(e),T(o.$$.fragment,e),i=c(e),m=h(e,"P",{"data-svelte-h":!0}),f(m)!=="svelte-1f9oh7n"&&(m.innerHTML=Z),k=c(e),v=h(e,"P",{"data-svelte-h":!0}),f(v)!=="svelte-1olgvni"&&(v.textContent=_),d=c(e),G=h(e,"P",{"data-svelte-h":!0}),f(G)!=="svelte-16ypsvh"&&(G.textContent=Ce),z=c(e),T(C.$$.fragment,e),P=c(e),T(X.$$.fragment,e),W=c(e),V=h(e,"P",{"data-svelte-h":!0}),f(V)!=="svelte-1a9ib4g"&&(V.textContent=y),B=c(e),R=h(e,"P",{"data-svelte-h":!0}),f(R)!=="svelte-7nmn0g"&&(R.innerHTML=Zt),Ie=c(e),K=h(e,"P",{"data-svelte-h":!0}),f(K)!=="svelte-1abkxhz"&&(K.innerHTML=gt),Xe=c(e),T(H.$$.fragment,e),We=c(e),O=h(e,"P",{"data-svelte-h":!0}),f(O)!=="svelte-13l87jg"&&(O.textContent=Gt),Ve=c(e),ee=h(e,"OL",{"data-svelte-h":!0}),f(ee)!=="svelte-lc6gnj"&&(ee.innerHTML=kt),Re=c(e),T(N.$$.fragment,e),Fe=c(e),T(S.$$.fragment,e),ze=c(e),T(te.$$.fragment,e),Ye=c(e),le=h(e,"P",{"data-svelte-h":!0}),f(le)!=="svelte-a4gxio"&&(le.textContent=_t),He=c(e),T(se.$$.fragment,e),Ne=c(e),T(E.$$.fragment,e),Se=c(e),ae=h(e,"P",{"data-svelte-h":!0}),f(ae)!=="svelte-nv0ghv"&&(ae.textContent=Bt),Ee=c(e),ne=h(e,"P",{"data-svelte-h":!0}),f(ne)!=="svelte-1nm1j4o"&&(ne.textContent=vt),xe=c(e),T(x.$$.fragment,e),Qe=c(e),oe=h(e,"P",{"data-svelte-h":!0}),f(oe)!=="svelte-1yxjr5n"&&(oe.innerHTML=Ct),Ae=c(e),T(Q.$$.fragment,e),qe=c(e),T(ie.$$.fragment,e),Le=c(e),re=h(e,"P",{"data-svelte-h":!0}),f(re)!=="svelte-123yl06"&&(re.textContent=$t),De=c(e),pe=h(e,"P",{"data-svelte-h":!0}),f(pe)!=="svelte-53xl5d"&&(pe.innerHTML=It),Pe=c(e),T(A.$$.fragment,e),Ke=c(e),T(ce.$$.fragment,e),Oe=c(e),Me=h(e,"P",{"data-svelte-h":!0}),f(Me)!=="svelte-119vxvi"&&(Me.textContent=Xt),et=c(e),T(me.$$.fragment,e),tt=c(e),de=h(e,"P",{"data-svelte-h":!0}),f(de)!=="svelte-1noxdna"&&(de.innerHTML=Wt),lt=c(e),ye=h(e,"P",{"data-svelte-h":!0}),f(ye)!=="svelte-1d73c2c"&&(ye.innerHTML=Vt),st=c(e),T(ue.$$.fragment,e),at=c(e),T(he.$$.fragment,e),nt=c(e),fe=h(e,"P",{"data-svelte-h":!0}),f(fe)!=="svelte-16n93px"&&(fe.textContent=Rt),ot=c(e),T(q.$$.fragment,e),it=c(e),Je=h(e,"P",{"data-svelte-h":!0}),f(Je)!=="svelte-xcpcjc"&&(Je.innerHTML=Ft),rt=c(e),T(Te.$$.fragment,e),pt=c(e),T(Ue.$$.fragment,e),ct=c(e),be=h(e,"P",{"data-svelte-h":!0}),f(be)!=="svelte-1h1vcbj"&&(be.textContent=zt),Mt=c(e),we=h(e,"P",{"data-svelte-h":!0}),f(we)!=="svelte-bsjn3b"&&(we.textContent=Yt),mt=c(e),T(L.$$.fragment,e),dt=c(e),je=h(e,"P",{"data-svelte-h":!0}),f(je)!=="svelte-14tgx8l"&&(je.innerHTML=Ht),yt=c(e),F=h(e,"IFRAME",{src:!0,frameborder:!0,width:!0,height:!0}),wt(F).forEach(a),ut=c(e),Ze=h(e,"P",{"data-svelte-h":!0}),f(Ze)!=="svelte-whgkfb"&&(Ze.innerHTML=St),ht=c(e),T(ge.$$.fragment,e),ft=c(e),Ge=h(e,"P",{"data-svelte-h":!0}),f(Ge)!=="svelte-1f44nt3"&&(Ge.innerHTML=Et),Jt=c(e),T(ke.$$.fragment,e),Tt=c(e),T(_e.$$.fragment,e),Ut=c(e),$e=h(e,"P",{}),wt($e).forEach(a),this.h()},h(){ve(t,"name","hf:doc:metadata"),ve(t,"content",_l),ll(F.src,Nt="https://hf-accelerate-model-memory-usage.hf.space")||ve(F,"src",Nt),ve(F,"frameborder","0"),ve(F,"width","850"),ve(F,"height","450")},m(e,l){il(document.head,t),n(e,M,l),n(e,s,l),n(e,r,l),U(o,e,l),n(e,i,l),n(e,m,l),n(e,k,l),n(e,v,l),n(e,d,l),n(e,G,l),n(e,z,l),U(C,e,l),n(e,P,l),U(X,e,l),n(e,W,l),n(e,V,l),n(e,B,l),n(e,R,l),n(e,Ie,l),n(e,K,l),n(e,Xe,l),U(H,e,l),n(e,We,l),n(e,O,l),n(e,Ve,l),n(e,ee,l),n(e,Re,l),U(N,e,l),n(e,Fe,l),U(S,e,l),n(e,ze,l),U(te,e,l),n(e,Ye,l),n(e,le,l),n(e,He,l),U(se,e,l),n(e,Ne,l),U(E,e,l),n(e,Se,l),n(e,ae,l),n(e,Ee,l),n(e,ne,l),n(e,xe,l),U(x,e,l),n(e,Qe,l),n(e,oe,l),n(e,Ae,l),U(Q,e,l),n(e,qe,l),U(ie,e,l),n(e,Le,l),n(e,re,l),n(e,De,l),n(e,pe,l),n(e,Pe,l),U(A,e,l),n(e,Ke,l),U(ce,e,l),n(e,Oe,l),n(e,Me,l),n(e,et,l),U(me,e,l),n(e,tt,l),n(e,de,l),n(e,lt,l),n(e,ye,l),n(e,st,l),U(ue,e,l),n(e,at,l),U(he,e,l),n(e,nt,l),n(e,fe,l),n(e,ot,l),U(q,e,l),n(e,it,l),n(e,Je,l),n(e,rt,l),U(Te,e,l),n(e,pt,l),U(Ue,e,l),n(e,ct,l),n(e,be,l),n(e,Mt,l),n(e,we,l),n(e,mt,l),U(L,e,l),n(e,dt,l),n(e,je,l),n(e,yt,l),n(e,F,l),n(e,ut,l),n(e,Ze,l),n(e,ht,l),U(ge,e,l),n(e,ft,l),n(e,Ge,l),n(e,Jt,l),U(ke,e,l),n(e,Tt,l),U(_e,e,l),n(e,Ut,l),n(e,$e,l),bt=!0},p(e,[l]){const xt={};l&2&&(xt.$$scope={dirty:l,ctx:e}),C.$set(xt);const Qt={};l&2&&(Qt.$$scope={dirty:l,ctx:e}),H.$set(Qt);const At={};l&2&&(At.$$scope={dirty:l,ctx:e}),N.$set(At);const qt={};l&2&&(qt.$$scope={dirty:l,ctx:e}),S.$set(qt);const Lt={};l&2&&(Lt.$$scope={dirty:l,ctx:e}),E.$set(Lt);const Dt={};l&2&&(Dt.$$scope={dirty:l,ctx:e}),x.$set(Dt);const Pt={};l&2&&(Pt.$$scope={dirty:l,ctx:e}),Q.$set(Pt);const Kt={};l&2&&(Kt.$$scope={dirty:l,ctx:e}),A.$set(Kt);const Ot={};l&2&&(Ot.$$scope={dirty:l,ctx:e}),q.$set(Ot);const el={};l&2&&(el.$$scope={dirty:l,ctx:e}),L.$set(el)},i(e){bt||(b(o.$$.fragment,e),b(C.$$.fragment,e),b(X.$$.fragment,e),b(H.$$.fragment,e),b(N.$$.fragment,e),b(S.$$.fragment,e),b(te.$$.fragment,e),b(se.$$.fragment,e),b(E.$$.fragment,e),b(x.$$.fragment,e),b(Q.$$.fragment,e),b(ie.$$.fragment,e),b(A.$$.fragment,e),b(ce.$$.fragment,e),b(me.$$.fragment,e),b(ue.$$.fragment,e),b(he.$$.fragment,e),b(q.$$.fragment,e),b(Te.$$.fragment,e),b(Ue.$$.fragment,e),b(L.$$.fragment,e),b(ge.$$.fragment,e),b(ke.$$.fragment,e),b(_e.$$.fragment,e),bt=!0)},o(e){w(o.$$.fragment,e),w(C.$$.fragment,e),w(X.$$.fragment,e),w(H.$$.fragment,e),w(N.$$.fragment,e),w(S.$$.fragment,e),w(te.$$.fragment,e),w(se.$$.fragment,e),w(E.$$.fragment,e),w(x.$$.fragment,e),w(Q.$$.fragment,e),w(ie.$$.fragment,e),w(A.$$.fragment,e),w(ce.$$.fragment,e),w(me.$$.fragment,e),w(ue.$$.fragment,e),w(he.$$.fragment,e),w(q.$$.fragment,e),w(Te.$$.fragment,e),w(Ue.$$.fragment,e),w(L.$$.fragment,e),w(ge.$$.fragment,e),w(ke.$$.fragment,e),w(_e.$$.fragment,e),bt=!1},d(e){e&&(a(M),a(s),a(r),a(i),a(m),a(k),a(v),a(d),a(G),a(z),a(P),a(W),a(V),a(B),a(R),a(Ie),a(K),a(Xe),a(We),a(O),a(Ve),a(ee),a(Re),a(Fe),a(ze),a(Ye),a(le),a(He),a(Ne),a(Se),a(ae),a(Ee),a(ne),a(xe),a(Qe),a(oe),a(Ae),a(qe),a(Le),a(re),a(De),a(pe),a(Pe),a(Ke),a(Oe),a(Me),a(et),a(tt),a(de),a(lt),a(ye),a(st),a(at),a(nt),a(fe),a(ot),a(it),a(Je),a(rt),a(pt),a(ct),a(be),a(Mt),a(we),a(mt),a(dt),a(je),a(yt),a(F),a(ut),a(Ze),a(ht),a(ft),a(Ge),a(Jt),a(Tt),a(Ut),a($e)),a(t),j(o,e),j(C,e),j(X,e),j(H,e),j(N,e),j(S,e),j(te,e),j(se,e),j(E,e),j(x,e),j(Q,e),j(ie,e),j(A,e),j(ce,e),j(me,e),j(ue,e),j(he,e),j(q,e),j(Te,e),j(Ue,e),j(L,e),j(ge,e),j(ke,e),j(_e,e)}}}const _l='{"title":"Optimizing inference","local":"optimizing-inference","sections":[{"title":"Static kv-cache and torch.compile","local":"static-kv-cache-and-torchcompile","sections":[],"depth":2},{"title":"Decoding strategies","local":"decoding-strategies","sections":[{"title":"Speculative decoding","local":"speculative-decoding","sections":[],"depth":3},{"title":"Prompt lookup decoding","local":"prompt-lookup-decoding","sections":[],"depth":3}],"depth":2},{"title":"Attention","local":"attention","sections":[{"title":"FlashAttention-2","local":"flashattention-2","sections":[],"depth":3},{"title":"PyTorch scaled dot product attention","local":"pytorch-scaled-dot-product-attention","sections":[],"depth":3}],"depth":2},{"title":"Quantization","local":"quantization","sections":[],"depth":2}],"depth":1}';function Bl(g){return sl(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Rl extends al{constructor(t){super(),nl(this,t,Bl,kl,tl,{})}}export{Rl as component};
