import{s as Ka,o as Pa,n as $e}from"../chunks/scheduler.18a86fab.js";import{S as Oa,i as es,g as r,s as l,r as d,A as ts,h as o,f as a,c as n,j as La,u as M,x as c,k as Aa,y as as,a as s,v as u,d as y,t as f,w as J}from"../chunks/index.98837b22.js";import{T as la}from"../chunks/Tip.77304350.js";import{C as k}from"../chunks/CodeBlock.8d0c2e8a.js";import{H as U}from"../chunks/Heading.7e7e0c5b.js";import{H as ss,a as Da}from"../chunks/HfOption.6641485e.js";import{E as ls}from"../chunks/index.499f366c.js";function ns(j){let i,T='You may notice a small degradation in generation throughput compared to <a href="/docs/transformers/main/en/internal/generation_utils#transformers.DynamicCache">DynamicCache</a> depending on your model and generation choices (context size, number of generated tokens, number of beams, etc.).';return{c(){i=r("p"),i.innerHTML=T},l(m){i=o(m,"P",{"data-svelte-h":!0}),c(i)!=="svelte-1j8sz51"&&(i.innerHTML=T)},m(m,b){s(m,i,b)},p:$e,d(m){m&&a(i)}}}function is(j){let i,T="Quantizing the cache can harm latency if the context length is short and there is enough GPU memory available for generation without enabling cache quantization. Try to find a balance between memory efficiency and latency.";return{c(){i=r("p"),i.textContent=T},l(m){i=o(m,"P",{"data-svelte-h":!0}),c(i)!=="svelte-1n6vxi"&&(i.textContent=T)},m(m,b){s(m,i,b)},p:$e,d(m){m&&a(i)}}}function rs(j){let i,T='For <a href="/docs/transformers/main/en/internal/generation_utils#transformers.HQQQuantizedCache">HQQQuantizedCache</a>, we recommend setting the <code>axis-key</code> and <code>axis-value</code> parameters to <code>1</code>.',m,b,h;return b=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUyQyUyMEhRUVF1YW50aXplZENhY2hlJTJDJTIwUXVhbnRpemVkQ2FjaGVDb25maWclMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJtZXRhLWxsYW1hJTJGTGxhbWEtMi03Yi1jaGF0LWhmJTIyKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMm1ldGEtbGxhbWElMkZMbGFtYS0yLTdiLWNoYXQtaGYlMjIlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYpLnRvKCUyMmN1ZGElM0EwJTIyKSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplciglMjJJJTIwbGlrZSUyMHJvY2slMjBtdXNpYyUyMGJlY2F1c2UlMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhtb2RlbC5kZXZpY2UpJTBBJTBBb3V0JTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBkb19zYW1wbGUlM0RGYWxzZSUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMjAlMkMlMjBjYWNoZV9pbXBsZW1lbnRhdGlvbiUzRCUyMnF1YW50aXplZCUyMiUyQyUyMGNhY2hlX2NvbmZpZyUzRCU3QiUyMmF4aXMta2V5JTIyJTNBJTIwMSUyQyUyMCUyMmF4aXMtdmFsdWUlMjIlM0ElMjAxJTJDJTIwJTIyYmFja2VuZCUyMiUzQSUyMCUyMmhxcSUyMiU3RCklMEFwcmludCh0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKG91dCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSU1QjAlNUQpJTBBSSUyMGxpa2UlMjByb2NrJTIwbXVzaWMlMjBiZWNhdXNlJTIwaXQncyUyMGxvdWQlMjBhbmQlMjBlbmVyZ2V0aWMuJTIwSXQncyUyMGElMjBncmVhdCUyMHdheSUyMHRvJTIwZXhwcmVzcyUyMG15c2VsZiUyMGFuZCUyMHJlbA==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM, HQQQuantizedCache, QuantizedCacheConfig

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-7b-chat-hf&quot;</span>)
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-7b-chat-hf&quot;</span>, torch_dtype=torch.float16).to(<span class="hljs-string">&quot;cuda:0&quot;</span>)
inputs = tokenizer(<span class="hljs-string">&quot;I like rock music because&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)

out = model.generate(**inputs, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">20</span>, cache_implementation=<span class="hljs-string">&quot;quantized&quot;</span>, cache_config={<span class="hljs-string">&quot;axis-key&quot;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&quot;axis-value&quot;</span>: <span class="hljs-number">1</span>, <span class="hljs-string">&quot;backend&quot;</span>: <span class="hljs-string">&quot;hqq&quot;</span>})
<span class="hljs-built_in">print</span>(tokenizer.batch_decode(out, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>])
I like rock music because it<span class="hljs-string">&#x27;s loud and energetic. It&#x27;</span>s a great way to express myself <span class="hljs-keyword">and</span> rel`,wrap:!1}}),{c(){i=r("p"),i.innerHTML=T,m=l(),d(b.$$.fragment)},l(p){i=o(p,"P",{"data-svelte-h":!0}),c(i)!=="svelte-128p31e"&&(i.innerHTML=T),m=n(p),M(b.$$.fragment,p)},m(p,w){s(p,i,w),s(p,m,w),u(b,p,w),h=!0},p:$e,i(p){h||(y(b.$$.fragment,p),h=!0)},o(p){f(b.$$.fragment,p),h=!1},d(p){p&&(a(i),a(m)),J(b,p)}}}function os(j){let i,T='For <a href="/docs/transformers/main/en/internal/generation_utils#transformers.QuantoQuantizedCache">QuantoQuantizedCache</a>, we recommend setting the <code>axis-key</code> and <code>axis-value</code> parameters to <code>0</code>.',m,b,h;return b=new k({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUyQyUyMFF1YW50b1F1YW50aXplZENhY2hlJTJDJTIwUXVhbnRpemVkQ2FjaGVDb25maWclMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJtZXRhLWxsYW1hJTJGTGxhbWEtMi03Yi1jaGF0LWhmJTIyKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMm1ldGEtbGxhbWElMkZMbGFtYS0yLTdiLWNoYXQtaGYlMjIlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYpLnRvKCUyMmN1ZGElM0EwJTIyKSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplciglMjJJJTIwbGlrZSUyMHJvY2slMjBtdXNpYyUyMGJlY2F1c2UlMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhtb2RlbC5kZXZpY2UpJTBBJTBBb3V0JTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBkb19zYW1wbGUlM0RGYWxzZSUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMjAlMkMlMjBjYWNoZV9pbXBsZW1lbnRhdGlvbiUzRCUyMnF1YW50aXplZCUyMiUyQyUyMGNhY2hlX2NvbmZpZyUzRCU3QiUyMm5iaXRzJTIyJTNBJTIwNCUyQyUyMCUyMmF4aXMta2V5JTIyJTNBJTIwMCUyQyUyMCUyMmF4aXMtdmFsdWUlMjIlM0ElMjAwJTJDJTIwJTIyYmFja2VuZCUyMiUzQSUyMCUyMnF1YW50byUyMiU3RCklMEFwcmludCh0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKG91dCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSU1QjAlNUQpJTBBSSUyMGxpa2UlMjByb2NrJTIwbXVzaWMlMjBiZWNhdXNlJTIwaXQncyUyMGxvdWQlMjBhbmQlMjBlbmVyZ2V0aWMuJTIwSXQncyUyMGElMjBncmVhdCUyMHdheSUyMHRvJTIwZXhwcmVzcyUyMG15c2VsZiUyMGFuZCUyMHJlbA==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM, QuantoQuantizedCache, QuantizedCacheConfig

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-7b-chat-hf&quot;</span>)
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-7b-chat-hf&quot;</span>, torch_dtype=torch.float16).to(<span class="hljs-string">&quot;cuda:0&quot;</span>)
inputs = tokenizer(<span class="hljs-string">&quot;I like rock music because&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)

out = model.generate(**inputs, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">20</span>, cache_implementation=<span class="hljs-string">&quot;quantized&quot;</span>, cache_config={<span class="hljs-string">&quot;nbits&quot;</span>: <span class="hljs-number">4</span>, <span class="hljs-string">&quot;axis-key&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;axis-value&quot;</span>: <span class="hljs-number">0</span>, <span class="hljs-string">&quot;backend&quot;</span>: <span class="hljs-string">&quot;quanto&quot;</span>})
<span class="hljs-built_in">print</span>(tokenizer.batch_decode(out, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>])
I like rock music because it<span class="hljs-string">&#x27;s loud and energetic. It&#x27;</span>s a great way to express myself <span class="hljs-keyword">and</span> rel`,wrap:!1}}),{c(){i=r("p"),i.innerHTML=T,m=l(),d(b.$$.fragment)},l(p){i=o(p,"P",{"data-svelte-h":!0}),c(i)!=="svelte-tu48rh"&&(i.innerHTML=T),m=n(p),M(b.$$.fragment,p)},m(p,w){s(p,i,w),s(p,m,w),u(b,p,w),h=!0},p:$e,i(p){h||(y(b.$$.fragment,p),h=!0)},o(p){f(b.$$.fragment,p),h=!1},d(p){p&&(a(i),a(m)),J(b,p)}}}function cs(j){let i,T,m,b;return i=new Da({props:{id:"quantized-cache",option:"HQQQuantizedCache",$$slots:{default:[rs]},$$scope:{ctx:j}}}),m=new Da({props:{id:"quantized-cache",option:"Quanto",$$slots:{default:[os]},$$scope:{ctx:j}}}),{c(){d(i.$$.fragment),T=l(),d(m.$$.fragment)},l(h){M(i.$$.fragment,h),T=n(h),M(m.$$.fragment,h)},m(h,p){u(i,h,p),s(h,T,p),u(m,h,p),b=!0},p(h,p){const w={};p&2&&(w.$$scope={dirty:p,ctx:h}),i.$set(w);const ze={};p&2&&(ze.$$scope={dirty:p,ctx:h}),m.$set(ze)},i(h){b||(y(i.$$.fragment,h),y(m.$$.fragment,h),b=!0)},o(h){f(i.$$.fragment,h),f(m.$$.fragment,h),b=!1},d(h){h&&a(T),J(i,h),J(m,h)}}}function ps(j){let i,T='The <a href="/docs/transformers/main/en/internal/generation_utils#transformers.EncoderDecoderCache">EncoderDecoderCache</a> currently only supports <a href="./model_doc/whisper">Whisper</a>.';return{c(){i=r("p"),i.innerHTML=T},l(m){i=o(m,"P",{"data-svelte-h":!0}),c(i)!=="svelte-1s3twwe"&&(i.innerHTML=T)},m(m,b){s(m,i,b)},p:$e,d(m){m&&a(i)}}}function ms(j){let i,T,m,b,h,p,w,ze="The key-value (KV) vectors are used to calculate attention scores. For autoregressive models, KV scores are calculated <em>every</em> time because the model predicts one token at a time. Each prediction depends on the previous tokens, which means the model performs the same computations each time.",He,I,na='A KV <em>cache</em> stores these calculations so they can be reused without recomputing them. Efficient caching is crucial for optimizing model performance because it reduces computation time and improves response rates. Refer to the <a href="./cache_explanation">Caching</a> doc for a more detailed explanation about how a cache works.',Fe,G,ia='Transformers offers several <a href="/docs/transformers/main/en/internal/generation_utils#transformers.Cache">Cache</a> classes that implement different caching mechanisms. Some of these <a href="/docs/transformers/main/en/internal/generation_utils#transformers.Cache">Cache</a> classes are optimized to save memory while others are designed to maximize generation speed. Refer to the table below to compare cache types and use it to help you select the best cache for your use case.',xe,_,ra="<thead><tr><th>Cache Type</th> <th>Memory Efficient  </th> <th>Supports torch.compile()</th> <th>Initialization Recommended</th> <th>Latency</th> <th>Long Context Generation</th></tr></thead> <tbody><tr><td>Dynamic Cache</td> <td>No</td> <td>No</td> <td>No</td> <td>Mid</td> <td>No</td></tr> <tr><td>Static Cache</td> <td>No</td> <td>Yes</td> <td>Yes</td> <td>High</td> <td>No</td></tr> <tr><td>Offloaded Cache</td> <td>Yes</td> <td>No</td> <td>No</td> <td>Low</td> <td>Yes</td></tr> <tr><td>Offloaded Static Cache</td> <td>No</td> <td>Yes</td> <td>Yes</td> <td>High</td> <td>Yes</td></tr> <tr><td>Quantized Cache</td> <td>Yes</td> <td>No</td> <td>No</td> <td>Low</td> <td>Yes</td></tr> <tr><td>Sliding Window Cache</td> <td>No</td> <td>Yes</td> <td>Yes</td> <td>High</td> <td>No</td></tr> <tr><td>Sink Cache</td> <td>Yes</td> <td>No</td> <td>Yes</td> <td>Mid</td> <td>Yes</td></tr></tbody>",Ee,X,oa='This guide introduces you to the different <a href="/docs/transformers/main/en/internal/generation_utils#transformers.Cache">Cache</a> classes and shows you how to use them for generation.',qe,B,Se,v,ca='The <a href="/docs/transformers/main/en/internal/generation_utils#transformers.DynamicCache">DynamicCache</a> is the default cache class for most models. It allows the cache size to grow dynamically in order to store an increasing number of keys and values as generation progresses.',Le,N,pa='Disable the cache by configuring <code>use_cache=False</code> in <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a>.',Ae,R,De,V,ma='Cache classes can also be initialized first before calling and passing it to the models <a href="https://hf.co/docs/transformers/internal/generation_utils#transformers.generation.GenerateDecoderOnlyOutput.past_key_values" rel="nofollow">past_key_values</a> parameter. This cache initialization strategy is only recommended for some cache types.',Ke,Q,ha='In most other cases, it’s easier to define the cache strategy in the <a href="https://hf.co/docs/transformers/main_classes/text_generation#transformers.GenerationConfig.cache_implementation" rel="nofollow">cache_implementation</a> parameter.',Pe,z,Oe,Y,et,$,da='The KV cache can occupy a significant portion of memory and become a <a href="https://hf.co/blog/llama31#inference-memory-requirements" rel="nofollow">bottleneck</a> for long-context generation. Memory efficient caches focus on trading off speed for reduced memory usage. This is especially important for large language models (LLMs) and if your hardware is memory constrained.',tt,H,at,F,Ma='The <a href="/docs/transformers/main/en/internal/generation_utils#transformers.OffloadedCache">OffloadedCache</a> saves GPU memory by moving the KV cache for most model layers to the CPU. Only the current layer cache is maintained on the GPU during a models <code>forward</code> iteration over the layers. <a href="/docs/transformers/main/en/internal/generation_utils#transformers.OffloadedCache">OffloadedCache</a> asynchronously prefetches the next layer cache and sends the previous layer cache back to the CPU.',st,x,ua='This cache strategy always generates the same result as <a href="/docs/transformers/main/en/internal/generation_utils#transformers.DynamicCache">DynamicCache</a> and works as a drop-in replacement or fallback. You may want to use <a href="/docs/transformers/main/en/internal/generation_utils#transformers.OffloadedCache">OffloadedCache</a> if you have a GPU and you’re getting out-of-memory (OOM) errors.',lt,C,nt,E,ya='Enable <a href="/docs/transformers/main/en/internal/generation_utils#transformers.OffloadedCache">OffloadedCache</a> by configuring <code>cache_implementation=&quot;offloaded&quot;</code> in either <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationConfig">GenerationConfig</a> or <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a>.',it,q,rt,S,fa='The example below shows how you can fallback on <a href="/docs/transformers/main/en/internal/generation_utils#transformers.OffloadedCache">OffloadedCache</a> if you run out of memory.',ot,L,ct,A,pt,D,Ja='The <a href="/docs/transformers/main/en/internal/generation_utils#transformers.QuantizedCache">QuantizedCache</a> reduces memory requirements by quantizing the KV values to a lower precision. <a href="/docs/transformers/main/en/internal/generation_utils#transformers.QuantizedCache">QuantizedCache</a> currently supports two quantization backends.',mt,K,ba='<li><a href="/docs/transformers/main/en/internal/generation_utils#transformers.HQQQuantizedCache">HQQQuantizedCache</a> supports int2, int4, and int8 datatypes.</li> <li><a href="/docs/transformers/main/en/internal/generation_utils#transformers.QuantoQuantizedCache">QuantoQuantizedCache</a> supports int2 and int4 datatypes. This is the default quantization backend.</li>',ht,g,dt,P,Ta='Enable <a href="/docs/transformers/main/en/internal/generation_utils#transformers.QuantizedCache">QuantizedCache</a> by configuring <code>cache_implementation=&quot;quantized&quot;</code> in <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationConfig">GenerationConfig</a>, and indicate the quantization backend in <a href="/docs/transformers/main/en/internal/generation_utils#transformers.QuantizedCacheConfig">QuantizedCacheConfig</a>. Any additional quantization related parameters should also be passed either as a dict or an instance of <a href="/docs/transformers/main/en/internal/generation_utils#transformers.QuantizedCacheConfig">QuantizedCacheConfig</a>. You should use the default values for these additional parameters unless you’re running out-of-memory. In that case, consider decreasing the residual length.',Mt,Z,ut,O,yt,ee,wa='<a href="/docs/transformers/main/en/internal/generation_utils#transformers.SinkCache">SinkCache</a> is capable of generating very long sequences (“infinite length” according to the paper) by only retaining a few initial tokens from the sequence. These are called the <em>sink tokens</em> because they account for a significant portion of the attention scores during generation. Subsequent tokens are discarded on a sliding windowed basis, and only the latest <code>window_size</code> tokens are kept. This means most of the previous knowledge is discarded.',ft,te,Ua="The sink tokens allow a model to maintain stable performance even when it’s dealing with very long text sequences.",Jt,ae,ja='Enable <a href="/docs/transformers/main/en/internal/generation_utils#transformers.SinkCache">SinkCache</a> by initializing it first with the <a href="https://hf.co/docs/transformers/main/en/internal/generation_utils#transformers.SinkCache.window_length" rel="nofollow">window_length</a> and <a href="https://hf.co/docs/transformers/main/en/internal/generation_utils#transformers.SinkCache.num_sink_tokens" rel="nofollow">num_sink_tokens</a> parameters before passing it to <a href="https://hf.co/docs/transformers/internal/generation_utils#transformers.generation.GenerateDecoderOnlyOutput.past_key_values" rel="nofollow">past_key_values</a> in <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a>.',bt,se,Tt,le,wt,ne,ka='The default <a href="/docs/transformers/main/en/internal/generation_utils#transformers.DynamicCache">DynamicCache</a> prevents you from taking advantage of just-in-time (JIT) optimizations because the cache size isn’t fixed. JIT optimizations enable you to maximize latency at the expense of memory usage. All of the following cache types are compatible with JIT optimizations like <a href="./llm_optims#static-kv-cache-and-torchcompile">torch.compile</a> to accelerate generation.',Ut,ie,jt,re,Ca='A <a href="/docs/transformers/main/en/internal/generation_utils#transformers.StaticCache">StaticCache</a> pre-allocates a specific maximum cache size for the kv pairs. You can generate up to the maximum cache size without needing to modify it.',kt,oe,ga='Enable <a href="/docs/transformers/main/en/internal/generation_utils#transformers.StaticCache">StaticCache</a> by configuring <code>cache_implementation=&quot;static&quot;</code> in <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a>.',Ct,ce,gt,pe,Zt,me,Za='The <a href="/docs/transformers/main/en/internal/generation_utils#transformers.OffloadedStaticCache">OffloadedStaticCache</a> is very similar to the <a href="#offloaded-cache">OffloadedCache</a> except the cache size is set to a maximum cache size. Otherwise, <a href="/docs/transformers/main/en/internal/generation_utils#transformers.OffloadedStaticCache">OffloadedStaticCache</a> only keeps the current layer cache on the GPU and the rest are moved to the CPU.',Wt,he,Wa='Enable <a href="/docs/transformers/main/en/internal/generation_utils#transformers.OffloadedStaticCache">OffloadedStaticCache</a> by configuring <code>cache_implementation=&quot;offloaded_static&quot;</code> in <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a>.',It,de,Gt,Me,Ia="Cache offloading requires a CUDA GPU.",_t,ue,Xt,ye,Ga='<a href="/docs/transformers/main/en/internal/generation_utils#transformers.SlidingWindowCache">SlidingWindowCache</a> implements a sliding window over the previous kv pairs, and only keeps the last <code>sliding_window</code> tokens. This cache type is designed to only work with models that support <em>sliding window attention</em>, such as <a href="./model_doc/mistral">Mistral</a>. Older kv states are discarded and replaced by new kv states.',Bt,fe,_a='Enable <a href="/docs/transformers/main/en/internal/generation_utils#transformers.SlidingWindowCache">SlidingWindowCache</a> by configuring <code>cache_implementation=&quot;sliding_window&quot;</code> in <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a>.',vt,Je,Nt,be,Rt,Te,Xa='Some model types, like encoder-decoder models or <a href="./model_doc/gemma2">Gemma2</a> and <a href="./model_doc/mamba">Mamba</a>, have dedicated cache classes.',Vt,we,Qt,Ue,Ba='<a href="/docs/transformers/main/en/internal/generation_utils#transformers.EncoderDecoderCache">EncoderDecoderCache</a> is designed for encoder-decoder models. It manages both the self-attention and cross-attention caches to ensure storage and retrieval of previous kv pairs. It is possible to individually set a different cache type for the encoder and decoder.',zt,je,va='This cache type doesn’t require any setup. It can be used when calling <a href="/docs/transformers/main/en/main_classes/text_generation#transformers.GenerationMixin.generate">generate()</a> or a models <code>forward</code> method.',Yt,W,$t,ke,Ht,Ce,Na="Some models have a unique way of storing past kv pairs or states that is not compatible with any other cache classes.",Ft,ge,Ra='<a href="./model_doc/gemma2">Gemma2</a> requires <a href="/docs/transformers/main/en/internal/generation_utils#transformers.HybridCache">HybridCache</a>, which uses a combination of <a href="/docs/transformers/main/en/internal/generation_utils#transformers.SlidingWindowCache">SlidingWindowCache</a> for sliding window attention and <a href="/docs/transformers/main/en/internal/generation_utils#transformers.StaticCache">StaticCache</a> for global attention under the hood.',xt,Ze,Va='<a href="./model_doc/mamba">Mamba</a> requires <a href="/docs/transformers/main/en/internal/generation_utils#transformers.MambaCache">MambaCache</a> because the model doesn’t have an attention mechanism or kv states.',Et,We,qt,Ie,Qa="A cache can also work in iterative generation settings where there is back-and-forth interaction with a model (chatbots). Like regular generation, iterative generation with a cache allows a model to efficiently handle ongoing conversations without recomputing the entire context at each step.",St,Ge,za='For iterative generation with a cache, start by initializing an empty cache class and then you can feed in your new prompts. Keep track of dialogue history with a <a href="./chat_templating">chat template</a>.',Lt,_e,Ya='If you’re using <a href="/docs/transformers/main/en/internal/generation_utils#transformers.SinkCache">SinkCache</a>, the inputs need to be truncated to the maximum length because <a href="/docs/transformers/main/en/internal/generation_utils#transformers.SinkCache">SinkCache</a> can generate text that exceeds its maximum window size. However, the first input shouldn’t exceed the maximum cache length.',At,Xe,$a="The example below demonstrates how to use a cache for iterative generation.",Dt,Be,Kt,ve,Pt,Ne,Ha='In some situations, you may want to fill a <a href="/docs/transformers/main/en/internal/generation_utils#transformers.Cache">Cache</a> with kv pairs for a certain prefix prompt and reuse it to generate different sequences.',Ot,Re,Fa='The example below initializes a <a href="/docs/transformers/main/en/internal/generation_utils#transformers.StaticCache">StaticCache</a>, and then caches an initial prompt. Now you can generate several sequences from the prefilled prompt.',ea,Ve,ta,Qe,aa,Ye,sa;return h=new U({props:{title:"KV cache strategies",local:"kv-cache-strategies",headingTag:"h1"}}),B=new U({props:{title:"Default cache",local:"default-cache",headingTag:"h2"}}),R=new k({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIybWV0YS1sbGFtYSUyRkxsYW1hLTItN2ItY2hhdC1oZiUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJtZXRhLWxsYW1hJTJGTGxhbWEtMi03Yi1jaGF0LWhmJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2KS50byglMjJjdWRhJTNBMCUyMiklMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoJTIySSUyMGxpa2UlMjByb2NrJTIwbXVzaWMlMjBiZWNhdXNlJTIyJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8obW9kZWwuZGV2aWNlKSUwQSUwQW1vZGVsLmdlbmVyYXRlKCoqaW5wdXRzJTJDJTIwZG9fc2FtcGxlJTNERmFsc2UlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDIwJTJDJTIwdXNlX2NhY2hlJTNERmFsc2Up",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-7b-chat-hf&quot;</span>)
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-7b-chat-hf&quot;</span>, torch_dtype=torch.float16).to(<span class="hljs-string">&quot;cuda:0&quot;</span>)
inputs = tokenizer(<span class="hljs-string">&quot;I like rock music because&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)

model.generate(**inputs, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">20</span>, use_cache=<span class="hljs-literal">False</span>)`,wrap:!1}}),z=new k({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwRHluYW1pY0NhY2hlJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIybWV0YS1sbGFtYSUyRkxsYW1hLTItN2ItY2hhdC1oZiUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJtZXRhLWxsYW1hJTJGTGxhbWEtMi03Yi1jaGF0LWhmJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2KS50byglMjJjdWRhJTNBMCUyMiklMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoJTIySSUyMGxpa2UlMjByb2NrJTIwbXVzaWMlMjBiZWNhdXNlJTIyJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8obW9kZWwuZGV2aWNlKSUwQSUwQXBhc3Rfa2V5X3ZhbHVlcyUyMCUzRCUyMER5bmFtaWNDYWNoZSgpJTBBb3V0JTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBkb19zYW1wbGUlM0RGYWxzZSUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMjAlMkMlMjBwYXN0X2tleV92YWx1ZXMlM0RwYXN0X2tleV92YWx1ZXMp",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM, DynamicCache

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-7b-chat-hf&quot;</span>)
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-7b-chat-hf&quot;</span>, torch_dtype=torch.float16).to(<span class="hljs-string">&quot;cuda:0&quot;</span>)
inputs = tokenizer(<span class="hljs-string">&quot;I like rock music because&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)

past_key_values = DynamicCache()
out = model.generate(**inputs, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">20</span>, past_key_values=past_key_values)`,wrap:!1}}),Y=new U({props:{title:"Memory efficient caches",local:"memory-efficient-caches",headingTag:"h2"}}),H=new U({props:{title:"Offloaded cache",local:"offloaded-cache",headingTag:"h3"}}),C=new la({props:{warning:!0,$$slots:{default:[ns]},$$scope:{ctx:j}}}),q=new k({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBY2twdCUyMCUzRCUyMCUyMm1pY3Jvc29mdCUyRlBoaS0zLW1pbmktNGstaW5zdHJ1Y3QlMjIlMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChja3B0KSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKGNrcHQlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmZsb2F0MTYpLnRvKCUyMmN1ZGElM0EwJTIyKSUwQWlucHV0cyUyMCUzRCUyMHRva2VuaXplciglMjJGdW4lMjBmYWN0JTNBJTIwVGhlJTIwc2hvcnRlc3QlMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhtb2RlbC5kZXZpY2UpJTBBJTBBb3V0JTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBkb19zYW1wbGUlM0RGYWxzZSUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMjMlMkMlMjBjYWNoZV9pbXBsZW1lbnRhdGlvbiUzRCUyMm9mZmxvYWRlZCUyMiklMEFwcmludCh0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKG91dCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSU1QjAlNUQpJTBBRnVuJTIwZmFjdCUzQSUyMFRoZSUyMHNob3J0ZXN0JTIwd2FyJTIwaW4lMjBoaXN0b3J5JTIwd2FzJTIwYmV0d2VlbiUyMEJyaXRhaW4lMjBhbmQlMjBaYW56aWJhciUyMG9uJTIwQXVndXN0JTIwMjclMkMlMjAxODk2Lg==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

ckpt = <span class="hljs-string">&quot;microsoft/Phi-3-mini-4k-instruct&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(ckpt)
model = AutoModelForCausalLM.from_pretrained(ckpt, torch_dtype=torch.float16).to(<span class="hljs-string">&quot;cuda:0&quot;</span>)
inputs = tokenizer(<span class="hljs-string">&quot;Fun fact: The shortest&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)

out = model.generate(**inputs, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">23</span>, cache_implementation=<span class="hljs-string">&quot;offloaded&quot;</span>)
<span class="hljs-built_in">print</span>(tokenizer.batch_decode(out, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>])
Fun fact: The shortest war <span class="hljs-keyword">in</span> history was between Britain <span class="hljs-keyword">and</span> Zanzibar on August <span class="hljs-number">27</span>, <span class="hljs-number">1896.</span>`,wrap:!1}}),L=new k({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBZGVmJTIwcmVzaWxpZW50X2dlbmVyYXRlKG1vZGVsJTJDJTIwKmFyZ3MlMkMlMjAqKmt3YXJncyklM0ElMEElMjAlMjAlMjAlMjBvb20lMjAlM0QlMjBGYWxzZSUwQSUyMCUyMCUyMCUyMHRyeSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJldHVybiUyMG1vZGVsLmdlbmVyYXRlKCphcmdzJTJDJTIwKiprd2FyZ3MpJTBBJTIwJTIwJTIwJTIwZXhjZXB0JTIwdG9yY2guY3VkYS5PdXRPZk1lbW9yeUVycm9yJTIwYXMlMjBlJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcHJpbnQoZSklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmludCglMjJyZXRyeWluZyUyMHdpdGglMjBjYWNoZV9pbXBsZW1lbnRhdGlvbiUzRCdvZmZsb2FkZWQnJTIyKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG9vbSUyMCUzRCUyMFRydWUlMEElMjAlMjAlMjAlMjBpZiUyMG9vbSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHRvcmNoLmN1ZGEuZW1wdHlfY2FjaGUoKSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGt3YXJncyU1QiUyMmNhY2hlX2ltcGxlbWVudGF0aW9uJTIyJTVEJTIwJTNEJTIwJTIyb2ZmbG9hZGVkJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcmV0dXJuJTIwbW9kZWwuZ2VuZXJhdGUoKmFyZ3MlMkMlMjAqKmt3YXJncyklMEElMEFja3B0JTIwJTNEJTIwJTIybWljcm9zb2Z0JTJGUGhpLTMtbWluaS00ay1pbnN0cnVjdCUyMiUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKGNrcHQpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoY2twdCUyQyUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNikudG8oJTIyY3VkYSUzQTAlMjIpJTBBcHJvbXB0JTIwJTNEJTIwJTVCJTIyb2theSUyMCUyMioxMDAwJTIwJTJCJTIwJTIyRnVuJTIwZmFjdCUzQSUyMFRoZSUyMG1vc3QlMjIlNUQlMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIocHJvbXB0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8obW9kZWwuZGV2aWNlKSUwQWJlYW1zJTIwJTNEJTIwJTdCJTIwJTIybnVtX2JlYW1zJTIyJTNBJTIwNDAlMkMlMjAlMjJudW1fYmVhbV9ncm91cHMlMjIlM0ElMjA0MCUyQyUyMCUyMm51bV9yZXR1cm5fc2VxdWVuY2VzJTIyJTNBJTIwNDAlMkMlMjAlMjJkaXZlcnNpdHlfcGVuYWx0eSUyMiUzQSUyMDEuMCUyQyUyMCUyMm1heF9uZXdfdG9rZW5zJTIyJTNBJTIwMjMlMkMlMjAlMjJlYXJseV9zdG9wcGluZyUyMiUzQSUyMFRydWUlMkMlMjAlN0QlMEFvdXQlMjAlM0QlMjByZXNpbGllbnRfZ2VuZXJhdGUobW9kZWwlMkMlMjAqKmlucHV0cyUyQyUyMCoqYmVhbXMpJTBBcmVzcG9uc2VzJTIwJTNEJTIwdG9rZW5pemVyLmJhdGNoX2RlY29kZShvdXQlNUIlM0ElMkMtMjglM0ElNUQlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

<span class="hljs-keyword">def</span> <span class="hljs-title function_">resilient_generate</span>(<span class="hljs-params">model, *args, **kwargs</span>):
    oom = <span class="hljs-literal">False</span>
    <span class="hljs-keyword">try</span>:
        <span class="hljs-keyword">return</span> model.generate(*args, **kwargs)
    <span class="hljs-keyword">except</span> torch.cuda.OutOfMemoryError <span class="hljs-keyword">as</span> e:
        <span class="hljs-built_in">print</span>(e)
        <span class="hljs-built_in">print</span>(<span class="hljs-string">&quot;retrying with cache_implementation=&#x27;offloaded&#x27;&quot;</span>)
        oom = <span class="hljs-literal">True</span>
    <span class="hljs-keyword">if</span> oom:
        torch.cuda.empty_cache()
        kwargs[<span class="hljs-string">&quot;cache_implementation&quot;</span>] = <span class="hljs-string">&quot;offloaded&quot;</span>
        <span class="hljs-keyword">return</span> model.generate(*args, **kwargs)

ckpt = <span class="hljs-string">&quot;microsoft/Phi-3-mini-4k-instruct&quot;</span>
tokenizer = AutoTokenizer.from_pretrained(ckpt)
model = AutoModelForCausalLM.from_pretrained(ckpt, torch_dtype=torch.float16).to(<span class="hljs-string">&quot;cuda:0&quot;</span>)
prompt = [<span class="hljs-string">&quot;okay &quot;</span>*<span class="hljs-number">1000</span> + <span class="hljs-string">&quot;Fun fact: The most&quot;</span>]
inputs = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)
beams = { <span class="hljs-string">&quot;num_beams&quot;</span>: <span class="hljs-number">40</span>, <span class="hljs-string">&quot;num_beam_groups&quot;</span>: <span class="hljs-number">40</span>, <span class="hljs-string">&quot;num_return_sequences&quot;</span>: <span class="hljs-number">40</span>, <span class="hljs-string">&quot;diversity_penalty&quot;</span>: <span class="hljs-number">1.0</span>, <span class="hljs-string">&quot;max_new_tokens&quot;</span>: <span class="hljs-number">23</span>, <span class="hljs-string">&quot;early_stopping&quot;</span>: <span class="hljs-literal">True</span>, }
out = resilient_generate(model, **inputs, **beams)
responses = tokenizer.batch_decode(out[:,-<span class="hljs-number">28</span>:], skip_special_tokens=<span class="hljs-literal">True</span>)`,wrap:!1}}),A=new U({props:{title:"Quantized cache",local:"quantized-cache",headingTag:"h3"}}),g=new la({props:{warning:!0,$$slots:{default:[is]},$$scope:{ctx:j}}}),Z=new ss({props:{id:"quantized-cache",options:["HQQQuantizedCache","Quanto"],$$slots:{default:[cs]},$$scope:{ctx:j}}}),O=new U({props:{title:"Sink cache",local:"sink-cache",headingTag:"h3"}}),se=new k({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwU2lua0NhY2hlJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIybWV0YS1sbGFtYSUyRkxsYW1hLTItN2ItY2hhdC1oZiUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJtZXRhLWxsYW1hJTJGTGxhbWEtMi03Yi1jaGF0LWhmJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2KS50byglMjJjdWRhJTNBMCUyMiklMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoJTIyVGhpcyUyMGlzJTIwYSUyMGxvbmclMjBzdG9yeSUyMGFib3V0JTIwdW5pY29ybnMlMkMlMjBmYWlyaWVzJTIwYW5kJTIwbWFnaWMuJTIyJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8obW9kZWwuZGV2aWNlKSUwQSUwQXBhc3Rfa2V5X3ZhbHVlcyUyMCUzRCUyMFNpbmtDYWNoZSh3aW5kb3dfbGVuZ3RoJTNEMjU2JTJDJTIwbnVtX3NpbmtfdG9rZW5zJTNENCklMEFvdXQlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0cyUyQyUyMGRvX3NhbXBsZSUzREZhbHNlJTJDJTIwbWF4X25ld190b2tlbnMlM0QzMCUyQyUyMHBhc3Rfa2V5X3ZhbHVlcyUzRHBhc3Rfa2V5X3ZhbHVlcyklMEF0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKG91dCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSU1QjAlNUQlMEElMjJUaGlzJTIwaXMlMjBhJTIwbG9uZyUyMHN0b3J5JTIwYWJvdXQlMjB1bmljb3JucyUyQyUyMGZhaXJpZXMlMjBhbmQlMjBtYWdpYy4lMjBJdCUyMGlzJTIwYSUyMGZhbnRhc3klMjB3b3JsZCUyMHdoZXJlJTIwdW5pY29ybnMlMjBhbmQlMjBmYWlyaWVzJTIwbGl2ZSUyMHRvZ2V0aGVyJTIwaW4lMjBoYXJtb255LiUyMFRoZSUyMHN0b3J5JTIwZm9sbG93cyUyMGElMjB5b3VuZyUyMGdpcmwlMjBuYW1lZCUyMExpbHklMjI=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM, SinkCache

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-7b-chat-hf&quot;</span>)
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-7b-chat-hf&quot;</span>, torch_dtype=torch.float16).to(<span class="hljs-string">&quot;cuda:0&quot;</span>)
inputs = tokenizer(<span class="hljs-string">&quot;This is a long story about unicorns, fairies and magic.&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)

past_key_values = SinkCache(window_length=<span class="hljs-number">256</span>, num_sink_tokens=<span class="hljs-number">4</span>)
out = model.generate(**inputs, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">30</span>, past_key_values=past_key_values)
tokenizer.batch_decode(out, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
<span class="hljs-string">&quot;This is a long story about unicorns, fairies and magic. It is a fantasy world where unicorns and fairies live together in harmony. The story follows a young girl named Lily&quot;</span>`,wrap:!1}}),le=new U({props:{title:"Speed optimized caches",local:"speed-optimized-caches",headingTag:"h2"}}),ie=new U({props:{title:"Static cache",local:"static-cache",headingTag:"h3"}}),ce=new k({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIybWV0YS1sbGFtYSUyRkxsYW1hLTItN2ItY2hhdC1oZiUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJtZXRhLWxsYW1hJTJGTGxhbWEtMi03Yi1jaGF0LWhmJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIpJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKCUyMkhlbGxvJTJDJTIwbXklMjBuYW1lJTIwaXMlMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhtb2RlbC5kZXZpY2UpJTBBJTBBb3V0JTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBkb19zYW1wbGUlM0RGYWxzZSUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMjAlMkMlMjBjYWNoZV9pbXBsZW1lbnRhdGlvbiUzRCUyMnN0YXRpYyUyMiklMEF0b2tlbml6ZXIuYmF0Y2hfZGVjb2RlKG91dCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSU1QjAlNUQlMEElMjJIZWxsbyUyQyUyMG15JTIwbmFtZSUyMGlzJTIwJTVCWW91ciUyME5hbWUlNUQlMkMlMjBhbmQlMjBJJTIwYW0lMjBhJTIwJTVCWW91ciUyMFByb2Zlc3Npb24lNUQlMjB3aXRoJTIwJTVCTnVtYmVyJTIwb2YlMjBZZWFycyU1RCUyMG9mJTIy",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-7b-chat-hf&quot;</span>)
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-7b-chat-hf&quot;</span>, torch_dtype=torch.float16, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
inputs = tokenizer(<span class="hljs-string">&quot;Hello, my name is&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)

out = model.generate(**inputs, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">20</span>, cache_implementation=<span class="hljs-string">&quot;static&quot;</span>)
tokenizer.batch_decode(out, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
<span class="hljs-string">&quot;Hello, my name is [Your Name], and I am a [Your Profession] with [Number of Years] of&quot;</span>`,wrap:!1}}),pe=new U({props:{title:"Offloaded static cache",local:"offloaded-static-cache",headingTag:"h3"}}),de=new k({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIybWV0YS1sbGFtYSUyRkxsYW1hLTItN2ItY2hhdC1oZiUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJtZXRhLWxsYW1hJTJGTGxhbWEtMi03Yi1jaGF0LWhmJTIyJTJDJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIpJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKCUyMkhlbGxvJTJDJTIwbXklMjBuYW1lJTIwaXMlMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhtb2RlbC5kZXZpY2UpJTBBJTBBb3V0JTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBkb19zYW1wbGUlM0RGYWxzZSUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMjAlMkMlMjBjYWNoZV9pbXBsZW1lbnRhdGlvbiUzRCUyMm9mZmxvYWRlZF9zdGF0aWMlMjIpJTBBdG9rZW5pemVyLmJhdGNoX2RlY29kZShvdXQlMkMlMjBza2lwX3NwZWNpYWxfdG9rZW5zJTNEVHJ1ZSklNUIwJTVEJTBBJTIySGVsbG8lMkMlMjBteSUyMG5hbWUlMjBpcyUyMCU1QllvdXIlMjBOYW1lJTVEJTJDJTIwYW5kJTIwSSUyMGFtJTIwYSUyMCU1QllvdXIlMjBQcm9mZXNzaW9uJTVEJTIwd2l0aCUyMCU1Qk51bWJlciUyMG9mJTIwWWVhcnMlNUQlMjBvZiUyMg==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-7b-chat-hf&quot;</span>)
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;meta-llama/Llama-2-7b-chat-hf&quot;</span>, torch_dtype=torch.float16, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
inputs = tokenizer(<span class="hljs-string">&quot;Hello, my name is&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)

out = model.generate(**inputs, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">20</span>, cache_implementation=<span class="hljs-string">&quot;offloaded_static&quot;</span>)
tokenizer.batch_decode(out, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
<span class="hljs-string">&quot;Hello, my name is [Your Name], and I am a [Your Profession] with [Number of Years] of&quot;</span>`,wrap:!1}}),ue=new U({props:{title:"Sliding window cache",local:"sliding-window-cache",headingTag:"h3"}}),Je=new k({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQyUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwU2lua0NhY2hlJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIybWlzdHJhbGFpJTJGTWlzdHJhbC03Qi12MC4xJTIyKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMm1pc3RyYWxhaSUyRk1pc3RyYWwtN0ItdjAuMSUyMiUyQyUyMHRvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNikudG8oJTIyY3VkYSUzQTAlMjIpJTBBaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKCUyMlllc3RlcmRheSUyMEklMjB3YXMlMjBvbiUyMGElMjByb2NrJTIwY29uY2VydCUyMGFuZC4lMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byhtb2RlbC5kZXZpY2UpJTBBJTBBb3V0JTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBkb19zYW1wbGUlM0RGYWxzZSUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMzAlMkMlMjBjYWNoZV9pbXBsZW1lbnRhdGlvbiUzRCUyMnNsaWRpbmdfd2luZG93JTIyKSUwQXRva2VuaXplci5iYXRjaF9kZWNvZGUob3V0JTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpJTVCMCU1RA==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM, SinkCache

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;mistralai/Mistral-7B-v0.1&quot;</span>)
model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;mistralai/Mistral-7B-v0.1&quot;</span>, torch_dtype=torch.float16).to(<span class="hljs-string">&quot;cuda:0&quot;</span>)
inputs = tokenizer(<span class="hljs-string">&quot;Yesterday I was on a rock concert and.&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)

out = model.generate(**inputs, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">30</span>, cache_implementation=<span class="hljs-string">&quot;sliding_window&quot;</span>)
tokenizer.batch_decode(out, skip_special_tokens=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]`,wrap:!1}}),be=new U({props:{title:"Model caches",local:"model-caches",headingTag:"h2"}}),we=new U({props:{title:"Encoder-decoder cache",local:"encoder-decoder-cache",headingTag:"h3"}}),W=new la({props:{warning:!1,$$slots:{default:[ps]},$$scope:{ctx:j}}}),ke=new U({props:{title:"Model-specific caches",local:"model-specific-caches",headingTag:"h3"}}),We=new U({props:{title:"Iterative generation",local:"iterative-generation",headingTag:"h2"}}),Be=new k({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b1Rva2VuaXplciUyQ0F1dG9Nb2RlbEZvckNhdXNhbExNJTBBZnJvbSUyMHRyYW5zZm9ybWVycy5jYWNoZV91dGlscyUyMGltcG9ydCUyMCglMEElMjAlMjAlMjAlMjBEeW5hbWljQ2FjaGUlMkMlMEElMjAlMjAlMjAlMjBTaW5rQ2FjaGUlMkMlMEElMjAlMjAlMjAlMjBTdGF0aWNDYWNoZSUyQyUwQSUyMCUyMCUyMCUyMFNsaWRpbmdXaW5kb3dDYWNoZSUyQyUwQSUyMCUyMCUyMCUyMFF1YW50b1F1YW50aXplZENhY2hlJTJDJTBBJTIwJTIwJTIwJTIwUXVhbnRpemVkQ2FjaGVDb25maWclMkMlMEEpJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJtZXRhLWxsYW1hJTJGTGxhbWEtMi03Yi1jaGF0LWhmJTIyJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTJDJTIwZGV2aWNlX21hcCUzRCdhdXRvJyklMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZChtb2RlbF9pZCklMEElMEF1c2VyX3Byb21wdHMlMjAlM0QlMjAlNUIlMjJIZWxsbyUyQyUyMHdoYXQncyUyMHlvdXIlMjBuYW1lJTNGJTIyJTJDJTIwJTIyQnR3JTJDJTIweWVzdGVyZGF5JTIwSSUyMHdhcyUyMG9uJTIwYSUyMHJvY2slMjBjb25jZXJ0LiUyMiU1RCUwQSUwQXBhc3Rfa2V5X3ZhbHVlcyUyMCUzRCUyMER5bmFtaWNDYWNoZSgpJTBBbWF4X2NhY2hlX2xlbmd0aCUyMCUzRCUyMHBhc3Rfa2V5X3ZhbHVlcy5nZXRfbWF4X2xlbmd0aCgpJTBBJTBBbWVzc2FnZXMlMjAlM0QlMjAlNUIlNUQlMEFmb3IlMjBwcm9tcHQlMjBpbiUyMHVzZXJfcHJvbXB0cyUzQSUwQSUyMCUyMCUyMCUyMG1lc3NhZ2VzLmFwcGVuZCglN0IlMjJyb2xlJTIyJTNBJTIwJTIydXNlciUyMiUyQyUyMCUyMmNvbnRlbnQlMjIlM0ElMjBwcm9tcHQlN0QpJTBBJTIwJTIwJTIwJTIwaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyLmFwcGx5X2NoYXRfdGVtcGxhdGUobWVzc2FnZXMlMkMlMjBhZGRfZ2VuZXJhdGlvbl9wcm9tcHQlM0RUcnVlJTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMiUyQyUyMHJldHVybl9kaWN0JTNEVHJ1ZSkudG8obW9kZWwuZGV2aWNlKSUwQSUyMCUyMCUyMCUyMGlmJTIwaXNpbnN0YW5jZShwYXN0X2tleV92YWx1ZXMlMkMlMjBTaW5rQ2FjaGUpJTNBJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwaW5wdXRzJTIwJTNEJTIwJTdCayUzQSUyMHYlNUIlM0ElMkMlMjAtbWF4X2NhY2hlX2xlbmd0aCUzQSU1RCUyMGZvciUyMGslMkMlMjB2JTIwaW4lMjBpbnB1dHMuaXRlbXMoKSU3RCUwQSUyMCUyMCUyMCUyMGlucHV0X2xlbmd0aCUyMCUzRCUyMGlucHV0cyU1QiUyMmlucHV0X2lkcyUyMiU1RC5zaGFwZSU1QjElNUQlMEElMjAlMjAlMjAlMjBvdXRwdXRzJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBkb19zYW1wbGUlM0RGYWxzZSUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMjU2JTJDJTIwcGFzdF9rZXlfdmFsdWVzJTNEcGFzdF9rZXlfdmFsdWVzKSUwQSUyMCUyMCUyMCUyMGNvbXBsZXRpb24lMjAlM0QlMjB0b2tlbml6ZXIuZGVjb2RlKG91dHB1dHMlNUIwJTJDJTIwaW5wdXRfbGVuZ3RoJTNBJTIwJTVEJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpJTBBJTIwJTIwJTIwJTIwbWVzc2FnZXMuYXBwZW5kKCU3QiUyMnJvbGUlMjIlM0ElMjAlMjJhc3Npc3RhbnQlMjIlMkMlMjAlMjJjb250ZW50JTIyJTNBJTIwY29tcGxldGlvbiU3RCk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer,AutoModelForCausalLM
<span class="hljs-keyword">from</span> transformers.cache_utils <span class="hljs-keyword">import</span> (
    DynamicCache,
    SinkCache,
    StaticCache,
    SlidingWindowCache,
    QuantoQuantizedCache,
    QuantizedCacheConfig,
)

model_id = <span class="hljs-string">&quot;meta-llama/Llama-2-7b-chat-hf&quot;</span>
model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, device_map=<span class="hljs-string">&#x27;auto&#x27;</span>)
tokenizer = AutoTokenizer.from_pretrained(model_id)

user_prompts = [<span class="hljs-string">&quot;Hello, what&#x27;s your name?&quot;</span>, <span class="hljs-string">&quot;Btw, yesterday I was on a rock concert.&quot;</span>]

past_key_values = DynamicCache()
max_cache_length = past_key_values.get_max_length()

messages = []
<span class="hljs-keyword">for</span> prompt <span class="hljs-keyword">in</span> user_prompts:
    messages.append({<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: prompt})
    inputs = tokenizer.apply_chat_template(messages, add_generation_prompt=<span class="hljs-literal">True</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>, return_dict=<span class="hljs-literal">True</span>).to(model.device)
    <span class="hljs-keyword">if</span> <span class="hljs-built_in">isinstance</span>(past_key_values, SinkCache):
        inputs = {k: v[:, -max_cache_length:] <span class="hljs-keyword">for</span> k, v <span class="hljs-keyword">in</span> inputs.items()}
    input_length = inputs[<span class="hljs-string">&quot;input_ids&quot;</span>].shape[<span class="hljs-number">1</span>]
    outputs = model.generate(**inputs, do_sample=<span class="hljs-literal">False</span>, max_new_tokens=<span class="hljs-number">256</span>, past_key_values=past_key_values)
    completion = tokenizer.decode(outputs[<span class="hljs-number">0</span>, input_length: ], skip_special_tokens=<span class="hljs-literal">True</span>)
    messages.append({<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;assistant&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: completion})`,wrap:!1}}),ve=new U({props:{title:"Prefill a cache",local:"prefill-a-cache",headingTag:"h2"}}),Ve=new k({props:{code:"aW1wb3J0JTIwY29weSUwQWltcG9ydCUyMHRvcmNoJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQXV0b1Rva2VuaXplciUyQyUyMER5bmFtaWNDYWNoZSUyQyUyMFN0YXRpY0NhY2hlJTBBJTBBbW9kZWxfaWQlMjAlM0QlMjAlMjJtZXRhLWxsYW1hJTJGTGxhbWEtMi03Yi1jaGF0LWhmJTIyJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQlMkMlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTJDJTIwZGV2aWNlX21hcCUzRCUyMmN1ZGElMjIpJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQobW9kZWxfaWQpJTBBJTBBJTIzJTIwSW5pdCUyMFN0YXRpY0NhY2hlJTIwd2l0aCUyMGJpZyUyMGVub3VnaCUyMG1heC1sZW5ndGglMjAoMTAyNCUyMHRva2VucyUyMGZvciUyMHRoZSUyMGJlbG93JTIwZXhhbXBsZSklMjAlMEElMjMlMjBZb3UlMjBjYW4lMjBhbHNvJTIwaW5pdCUyMGElMjBEeW5hbWljQ2FjaGUlMkMlMjBpZiUyMHRoYXQlMjBzdWl0cyUyMHlvdSUyMGJldHRlciUwQXByb21wdF9jYWNoZSUyMCUzRCUyMFN0YXRpY0NhY2hlKGNvbmZpZyUzRG1vZGVsLmNvbmZpZyUyQyUyMG1heF9iYXRjaF9zaXplJTNEMSUyQyUyMG1heF9jYWNoZV9sZW4lM0QxMDI0JTJDJTIwZGV2aWNlJTNEJTIyY3VkYSUyMiUyQyUyMGR0eXBlJTNEdG9yY2guYmZsb2F0MTYpJTBBJTBBSU5JVElBTF9QUk9NUFQlMjAlM0QlMjAlMjJZb3UlMjBhcmUlMjBhJTIwaGVscGZ1bCUyMGFzc2lzdGFudC4lMjAlMjIlMEFpbnB1dHNfaW5pdGlhbF9wcm9tcHQlMjAlM0QlMjB0b2tlbml6ZXIoSU5JVElBTF9QUk9NUFQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byglMjJjdWRhJTIyKSUwQSUyMyUyMFRoaXMlMjBpcyUyMHRoZSUyMGNvbW1vbiUyMHByb21wdCUyMGNhY2hlZCUyQyUyMHdlJTIwbmVlZCUyMHRvJTIwcnVuJTIwZm9yd2FyZCUyMHdpdGhvdXQlMjBncmFkJTIwdG8lMjBiZSUyMGFibGUlMjB0byUyMGNvcHklMEF3aXRoJTIwdG9yY2gubm9fZ3JhZCgpJTNBJTBBJTIwJTIwJTIwJTIwJTIwcHJvbXB0X2NhY2hlJTIwJTNEJTIwbW9kZWwoKippbnB1dHNfaW5pdGlhbF9wcm9tcHQlMkMlMjBwYXN0X2tleV92YWx1ZXMlMjAlM0QlMjBwcm9tcHRfY2FjaGUpLnBhc3Rfa2V5X3ZhbHVlcyUwQSUwQXByb21wdHMlMjAlM0QlMjAlNUIlMjJIZWxwJTIwbWUlMjB0byUyMHdyaXRlJTIwYSUyMGJsb2dwb3N0JTIwYWJvdXQlMjB0cmF2ZWxsaW5nLiUyMiUyQyUyMCUyMldoYXQlMjBpcyUyMHRoZSUyMGNhcGl0YWwlMjBvZiUyMEZyYW5jZSUzRiUyMiU1RCUwQXJlc3BvbnNlcyUyMCUzRCUyMCU1QiU1RCUwQWZvciUyMHByb21wdCUyMGluJTIwcHJvbXB0cyUzQSUwQSUyMCUyMCUyMCUyMG5ld19pbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIoSU5JVElBTF9QUk9NUFQlMjAlMkIlMjBwcm9tcHQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKS50byglMjJjdWRhJTIyKSUwQSUyMCUyMCUyMCUyMHBhc3Rfa2V5X3ZhbHVlcyUyMCUzRCUyMGNvcHkuZGVlcGNvcHkocHJvbXB0X2NhY2hlKSUwQSUyMCUyMCUyMCUyMG91dHB1dHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKm5ld19pbnB1dHMlMkMlMjBwYXN0X2tleV92YWx1ZXMlM0RwYXN0X2tleV92YWx1ZXMlMkNtYXhfbmV3X3Rva2VucyUzRDIwKSUyMCUwQSUyMCUyMCUyMCUyMHJlc3BvbnNlJTIwJTNEJTIwdG9rZW5pemVyLmJhdGNoX2RlY29kZShvdXRwdXRzKSU1QjAlNUQlMEElMjAlMjAlMjAlMjByZXNwb25zZXMuYXBwZW5kKHJlc3BvbnNlKSUwQSUwQXByaW50KHJlc3BvbnNlcyk=",highlighted:`<span class="hljs-keyword">import</span> copy
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer, DynamicCache, StaticCache

model_id = <span class="hljs-string">&quot;meta-llama/Llama-2-7b-chat-hf&quot;</span>
model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, device_map=<span class="hljs-string">&quot;cuda&quot;</span>)
tokenizer = AutoTokenizer.from_pretrained(model_id)

<span class="hljs-comment"># Init StaticCache with big enough max-length (1024 tokens for the below example) </span>
<span class="hljs-comment"># You can also init a DynamicCache, if that suits you better</span>
prompt_cache = StaticCache(config=model.config, max_batch_size=<span class="hljs-number">1</span>, max_cache_len=<span class="hljs-number">1024</span>, device=<span class="hljs-string">&quot;cuda&quot;</span>, dtype=torch.bfloat16)

INITIAL_PROMPT = <span class="hljs-string">&quot;You are a helpful assistant. &quot;</span>
inputs_initial_prompt = tokenizer(INITIAL_PROMPT, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)
<span class="hljs-comment"># This is the common prompt cached, we need to run forward without grad to be able to copy</span>
<span class="hljs-keyword">with</span> torch.no_grad():
     prompt_cache = model(**inputs_initial_prompt, past_key_values = prompt_cache).past_key_values

prompts = [<span class="hljs-string">&quot;Help me to write a blogpost about travelling.&quot;</span>, <span class="hljs-string">&quot;What is the capital of France?&quot;</span>]
responses = []
<span class="hljs-keyword">for</span> prompt <span class="hljs-keyword">in</span> prompts:
    new_inputs = tokenizer(INITIAL_PROMPT + prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)
    past_key_values = copy.deepcopy(prompt_cache)
    outputs = model.generate(**new_inputs, past_key_values=past_key_values,max_new_tokens=<span class="hljs-number">20</span>) 
    response = tokenizer.batch_decode(outputs)[<span class="hljs-number">0</span>]
    responses.append(response)

<span class="hljs-built_in">print</span>(responses)`,wrap:!1}}),Qe=new ls({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/kv_cache.md"}}),{c(){i=r("meta"),T=l(),m=r("p"),b=l(),d(h.$$.fragment),p=l(),w=r("p"),w.innerHTML=ze,He=l(),I=r("p"),I.innerHTML=na,Fe=l(),G=r("p"),G.innerHTML=ia,xe=l(),_=r("table"),_.innerHTML=ra,Ee=l(),X=r("p"),X.innerHTML=oa,qe=l(),d(B.$$.fragment),Se=l(),v=r("p"),v.innerHTML=ca,Le=l(),N=r("p"),N.innerHTML=pa,Ae=l(),d(R.$$.fragment),De=l(),V=r("p"),V.innerHTML=ma,Ke=l(),Q=r("p"),Q.innerHTML=ha,Pe=l(),d(z.$$.fragment),Oe=l(),d(Y.$$.fragment),et=l(),$=r("p"),$.innerHTML=da,tt=l(),d(H.$$.fragment),at=l(),F=r("p"),F.innerHTML=Ma,st=l(),x=r("p"),x.innerHTML=ua,lt=l(),d(C.$$.fragment),nt=l(),E=r("p"),E.innerHTML=ya,it=l(),d(q.$$.fragment),rt=l(),S=r("p"),S.innerHTML=fa,ot=l(),d(L.$$.fragment),ct=l(),d(A.$$.fragment),pt=l(),D=r("p"),D.innerHTML=Ja,mt=l(),K=r("ul"),K.innerHTML=ba,ht=l(),d(g.$$.fragment),dt=l(),P=r("p"),P.innerHTML=Ta,Mt=l(),d(Z.$$.fragment),ut=l(),d(O.$$.fragment),yt=l(),ee=r("p"),ee.innerHTML=wa,ft=l(),te=r("p"),te.textContent=Ua,Jt=l(),ae=r("p"),ae.innerHTML=ja,bt=l(),d(se.$$.fragment),Tt=l(),d(le.$$.fragment),wt=l(),ne=r("p"),ne.innerHTML=ka,Ut=l(),d(ie.$$.fragment),jt=l(),re=r("p"),re.innerHTML=Ca,kt=l(),oe=r("p"),oe.innerHTML=ga,Ct=l(),d(ce.$$.fragment),gt=l(),d(pe.$$.fragment),Zt=l(),me=r("p"),me.innerHTML=Za,Wt=l(),he=r("p"),he.innerHTML=Wa,It=l(),d(de.$$.fragment),Gt=l(),Me=r("p"),Me.textContent=Ia,_t=l(),d(ue.$$.fragment),Xt=l(),ye=r("p"),ye.innerHTML=Ga,Bt=l(),fe=r("p"),fe.innerHTML=_a,vt=l(),d(Je.$$.fragment),Nt=l(),d(be.$$.fragment),Rt=l(),Te=r("p"),Te.innerHTML=Xa,Vt=l(),d(we.$$.fragment),Qt=l(),Ue=r("p"),Ue.innerHTML=Ba,zt=l(),je=r("p"),je.innerHTML=va,Yt=l(),d(W.$$.fragment),$t=l(),d(ke.$$.fragment),Ht=l(),Ce=r("p"),Ce.textContent=Na,Ft=l(),ge=r("p"),ge.innerHTML=Ra,xt=l(),Ze=r("p"),Ze.innerHTML=Va,Et=l(),d(We.$$.fragment),qt=l(),Ie=r("p"),Ie.textContent=Qa,St=l(),Ge=r("p"),Ge.innerHTML=za,Lt=l(),_e=r("p"),_e.innerHTML=Ya,At=l(),Xe=r("p"),Xe.textContent=$a,Dt=l(),d(Be.$$.fragment),Kt=l(),d(ve.$$.fragment),Pt=l(),Ne=r("p"),Ne.innerHTML=Ha,Ot=l(),Re=r("p"),Re.innerHTML=Fa,ea=l(),d(Ve.$$.fragment),ta=l(),d(Qe.$$.fragment),aa=l(),Ye=r("p"),this.h()},l(e){const t=ts("svelte-u9bgzb",document.head);i=o(t,"META",{name:!0,content:!0}),t.forEach(a),T=n(e),m=o(e,"P",{}),La(m).forEach(a),b=n(e),M(h.$$.fragment,e),p=n(e),w=o(e,"P",{"data-svelte-h":!0}),c(w)!=="svelte-1hxt7ht"&&(w.innerHTML=ze),He=n(e),I=o(e,"P",{"data-svelte-h":!0}),c(I)!=="svelte-og33ji"&&(I.innerHTML=na),Fe=n(e),G=o(e,"P",{"data-svelte-h":!0}),c(G)!=="svelte-guzt1y"&&(G.innerHTML=ia),xe=n(e),_=o(e,"TABLE",{"data-svelte-h":!0}),c(_)!=="svelte-1w6mv1t"&&(_.innerHTML=ra),Ee=n(e),X=o(e,"P",{"data-svelte-h":!0}),c(X)!=="svelte-99hnd9"&&(X.innerHTML=oa),qe=n(e),M(B.$$.fragment,e),Se=n(e),v=o(e,"P",{"data-svelte-h":!0}),c(v)!=="svelte-33g5j4"&&(v.innerHTML=ca),Le=n(e),N=o(e,"P",{"data-svelte-h":!0}),c(N)!=="svelte-lxidk8"&&(N.innerHTML=pa),Ae=n(e),M(R.$$.fragment,e),De=n(e),V=o(e,"P",{"data-svelte-h":!0}),c(V)!=="svelte-nbl90g"&&(V.innerHTML=ma),Ke=n(e),Q=o(e,"P",{"data-svelte-h":!0}),c(Q)!=="svelte-109dkae"&&(Q.innerHTML=ha),Pe=n(e),M(z.$$.fragment,e),Oe=n(e),M(Y.$$.fragment,e),et=n(e),$=o(e,"P",{"data-svelte-h":!0}),c($)!=="svelte-15jbxrx"&&($.innerHTML=da),tt=n(e),M(H.$$.fragment,e),at=n(e),F=o(e,"P",{"data-svelte-h":!0}),c(F)!=="svelte-co1r6m"&&(F.innerHTML=Ma),st=n(e),x=o(e,"P",{"data-svelte-h":!0}),c(x)!=="svelte-glo2x6"&&(x.innerHTML=ua),lt=n(e),M(C.$$.fragment,e),nt=n(e),E=o(e,"P",{"data-svelte-h":!0}),c(E)!=="svelte-110v56p"&&(E.innerHTML=ya),it=n(e),M(q.$$.fragment,e),rt=n(e),S=o(e,"P",{"data-svelte-h":!0}),c(S)!=="svelte-qx2uph"&&(S.innerHTML=fa),ot=n(e),M(L.$$.fragment,e),ct=n(e),M(A.$$.fragment,e),pt=n(e),D=o(e,"P",{"data-svelte-h":!0}),c(D)!=="svelte-55rmab"&&(D.innerHTML=Ja),mt=n(e),K=o(e,"UL",{"data-svelte-h":!0}),c(K)!=="svelte-12t3rwa"&&(K.innerHTML=ba),ht=n(e),M(g.$$.fragment,e),dt=n(e),P=o(e,"P",{"data-svelte-h":!0}),c(P)!=="svelte-1w224us"&&(P.innerHTML=Ta),Mt=n(e),M(Z.$$.fragment,e),ut=n(e),M(O.$$.fragment,e),yt=n(e),ee=o(e,"P",{"data-svelte-h":!0}),c(ee)!=="svelte-afybwq"&&(ee.innerHTML=wa),ft=n(e),te=o(e,"P",{"data-svelte-h":!0}),c(te)!=="svelte-z2ocp4"&&(te.textContent=Ua),Jt=n(e),ae=o(e,"P",{"data-svelte-h":!0}),c(ae)!=="svelte-t73h6l"&&(ae.innerHTML=ja),bt=n(e),M(se.$$.fragment,e),Tt=n(e),M(le.$$.fragment,e),wt=n(e),ne=o(e,"P",{"data-svelte-h":!0}),c(ne)!=="svelte-guc2fw"&&(ne.innerHTML=ka),Ut=n(e),M(ie.$$.fragment,e),jt=n(e),re=o(e,"P",{"data-svelte-h":!0}),c(re)!=="svelte-1f63xpl"&&(re.innerHTML=Ca),kt=n(e),oe=o(e,"P",{"data-svelte-h":!0}),c(oe)!=="svelte-zxmhi"&&(oe.innerHTML=ga),Ct=n(e),M(ce.$$.fragment,e),gt=n(e),M(pe.$$.fragment,e),Zt=n(e),me=o(e,"P",{"data-svelte-h":!0}),c(me)!=="svelte-1j4m8zn"&&(me.innerHTML=Za),Wt=n(e),he=o(e,"P",{"data-svelte-h":!0}),c(he)!=="svelte-17jft9b"&&(he.innerHTML=Wa),It=n(e),M(de.$$.fragment,e),Gt=n(e),Me=o(e,"P",{"data-svelte-h":!0}),c(Me)!=="svelte-zdzdtx"&&(Me.textContent=Ia),_t=n(e),M(ue.$$.fragment,e),Xt=n(e),ye=o(e,"P",{"data-svelte-h":!0}),c(ye)!=="svelte-mim534"&&(ye.innerHTML=Ga),Bt=n(e),fe=o(e,"P",{"data-svelte-h":!0}),c(fe)!=="svelte-1vobvz7"&&(fe.innerHTML=_a),vt=n(e),M(Je.$$.fragment,e),Nt=n(e),M(be.$$.fragment,e),Rt=n(e),Te=o(e,"P",{"data-svelte-h":!0}),c(Te)!=="svelte-kqp2g5"&&(Te.innerHTML=Xa),Vt=n(e),M(we.$$.fragment,e),Qt=n(e),Ue=o(e,"P",{"data-svelte-h":!0}),c(Ue)!=="svelte-1ljey3b"&&(Ue.innerHTML=Ba),zt=n(e),je=o(e,"P",{"data-svelte-h":!0}),c(je)!=="svelte-1mnw3g7"&&(je.innerHTML=va),Yt=n(e),M(W.$$.fragment,e),$t=n(e),M(ke.$$.fragment,e),Ht=n(e),Ce=o(e,"P",{"data-svelte-h":!0}),c(Ce)!=="svelte-u1f4zv"&&(Ce.textContent=Na),Ft=n(e),ge=o(e,"P",{"data-svelte-h":!0}),c(ge)!=="svelte-qqgsic"&&(ge.innerHTML=Ra),xt=n(e),Ze=o(e,"P",{"data-svelte-h":!0}),c(Ze)!=="svelte-11xfan5"&&(Ze.innerHTML=Va),Et=n(e),M(We.$$.fragment,e),qt=n(e),Ie=o(e,"P",{"data-svelte-h":!0}),c(Ie)!=="svelte-67pkdu"&&(Ie.textContent=Qa),St=n(e),Ge=o(e,"P",{"data-svelte-h":!0}),c(Ge)!=="svelte-1ejbghf"&&(Ge.innerHTML=za),Lt=n(e),_e=o(e,"P",{"data-svelte-h":!0}),c(_e)!=="svelte-dh567v"&&(_e.innerHTML=Ya),At=n(e),Xe=o(e,"P",{"data-svelte-h":!0}),c(Xe)!=="svelte-18xhg0c"&&(Xe.textContent=$a),Dt=n(e),M(Be.$$.fragment,e),Kt=n(e),M(ve.$$.fragment,e),Pt=n(e),Ne=o(e,"P",{"data-svelte-h":!0}),c(Ne)!=="svelte-1m2p4k0"&&(Ne.innerHTML=Ha),Ot=n(e),Re=o(e,"P",{"data-svelte-h":!0}),c(Re)!=="svelte-fds5ay"&&(Re.innerHTML=Fa),ea=n(e),M(Ve.$$.fragment,e),ta=n(e),M(Qe.$$.fragment,e),aa=n(e),Ye=o(e,"P",{}),La(Ye).forEach(a),this.h()},h(){Aa(i,"name","hf:doc:metadata"),Aa(i,"content",hs)},m(e,t){as(document.head,i),s(e,T,t),s(e,m,t),s(e,b,t),u(h,e,t),s(e,p,t),s(e,w,t),s(e,He,t),s(e,I,t),s(e,Fe,t),s(e,G,t),s(e,xe,t),s(e,_,t),s(e,Ee,t),s(e,X,t),s(e,qe,t),u(B,e,t),s(e,Se,t),s(e,v,t),s(e,Le,t),s(e,N,t),s(e,Ae,t),u(R,e,t),s(e,De,t),s(e,V,t),s(e,Ke,t),s(e,Q,t),s(e,Pe,t),u(z,e,t),s(e,Oe,t),u(Y,e,t),s(e,et,t),s(e,$,t),s(e,tt,t),u(H,e,t),s(e,at,t),s(e,F,t),s(e,st,t),s(e,x,t),s(e,lt,t),u(C,e,t),s(e,nt,t),s(e,E,t),s(e,it,t),u(q,e,t),s(e,rt,t),s(e,S,t),s(e,ot,t),u(L,e,t),s(e,ct,t),u(A,e,t),s(e,pt,t),s(e,D,t),s(e,mt,t),s(e,K,t),s(e,ht,t),u(g,e,t),s(e,dt,t),s(e,P,t),s(e,Mt,t),u(Z,e,t),s(e,ut,t),u(O,e,t),s(e,yt,t),s(e,ee,t),s(e,ft,t),s(e,te,t),s(e,Jt,t),s(e,ae,t),s(e,bt,t),u(se,e,t),s(e,Tt,t),u(le,e,t),s(e,wt,t),s(e,ne,t),s(e,Ut,t),u(ie,e,t),s(e,jt,t),s(e,re,t),s(e,kt,t),s(e,oe,t),s(e,Ct,t),u(ce,e,t),s(e,gt,t),u(pe,e,t),s(e,Zt,t),s(e,me,t),s(e,Wt,t),s(e,he,t),s(e,It,t),u(de,e,t),s(e,Gt,t),s(e,Me,t),s(e,_t,t),u(ue,e,t),s(e,Xt,t),s(e,ye,t),s(e,Bt,t),s(e,fe,t),s(e,vt,t),u(Je,e,t),s(e,Nt,t),u(be,e,t),s(e,Rt,t),s(e,Te,t),s(e,Vt,t),u(we,e,t),s(e,Qt,t),s(e,Ue,t),s(e,zt,t),s(e,je,t),s(e,Yt,t),u(W,e,t),s(e,$t,t),u(ke,e,t),s(e,Ht,t),s(e,Ce,t),s(e,Ft,t),s(e,ge,t),s(e,xt,t),s(e,Ze,t),s(e,Et,t),u(We,e,t),s(e,qt,t),s(e,Ie,t),s(e,St,t),s(e,Ge,t),s(e,Lt,t),s(e,_e,t),s(e,At,t),s(e,Xe,t),s(e,Dt,t),u(Be,e,t),s(e,Kt,t),u(ve,e,t),s(e,Pt,t),s(e,Ne,t),s(e,Ot,t),s(e,Re,t),s(e,ea,t),u(Ve,e,t),s(e,ta,t),u(Qe,e,t),s(e,aa,t),s(e,Ye,t),sa=!0},p(e,[t]){const xa={};t&2&&(xa.$$scope={dirty:t,ctx:e}),C.$set(xa);const Ea={};t&2&&(Ea.$$scope={dirty:t,ctx:e}),g.$set(Ea);const qa={};t&2&&(qa.$$scope={dirty:t,ctx:e}),Z.$set(qa);const Sa={};t&2&&(Sa.$$scope={dirty:t,ctx:e}),W.$set(Sa)},i(e){sa||(y(h.$$.fragment,e),y(B.$$.fragment,e),y(R.$$.fragment,e),y(z.$$.fragment,e),y(Y.$$.fragment,e),y(H.$$.fragment,e),y(C.$$.fragment,e),y(q.$$.fragment,e),y(L.$$.fragment,e),y(A.$$.fragment,e),y(g.$$.fragment,e),y(Z.$$.fragment,e),y(O.$$.fragment,e),y(se.$$.fragment,e),y(le.$$.fragment,e),y(ie.$$.fragment,e),y(ce.$$.fragment,e),y(pe.$$.fragment,e),y(de.$$.fragment,e),y(ue.$$.fragment,e),y(Je.$$.fragment,e),y(be.$$.fragment,e),y(we.$$.fragment,e),y(W.$$.fragment,e),y(ke.$$.fragment,e),y(We.$$.fragment,e),y(Be.$$.fragment,e),y(ve.$$.fragment,e),y(Ve.$$.fragment,e),y(Qe.$$.fragment,e),sa=!0)},o(e){f(h.$$.fragment,e),f(B.$$.fragment,e),f(R.$$.fragment,e),f(z.$$.fragment,e),f(Y.$$.fragment,e),f(H.$$.fragment,e),f(C.$$.fragment,e),f(q.$$.fragment,e),f(L.$$.fragment,e),f(A.$$.fragment,e),f(g.$$.fragment,e),f(Z.$$.fragment,e),f(O.$$.fragment,e),f(se.$$.fragment,e),f(le.$$.fragment,e),f(ie.$$.fragment,e),f(ce.$$.fragment,e),f(pe.$$.fragment,e),f(de.$$.fragment,e),f(ue.$$.fragment,e),f(Je.$$.fragment,e),f(be.$$.fragment,e),f(we.$$.fragment,e),f(W.$$.fragment,e),f(ke.$$.fragment,e),f(We.$$.fragment,e),f(Be.$$.fragment,e),f(ve.$$.fragment,e),f(Ve.$$.fragment,e),f(Qe.$$.fragment,e),sa=!1},d(e){e&&(a(T),a(m),a(b),a(p),a(w),a(He),a(I),a(Fe),a(G),a(xe),a(_),a(Ee),a(X),a(qe),a(Se),a(v),a(Le),a(N),a(Ae),a(De),a(V),a(Ke),a(Q),a(Pe),a(Oe),a(et),a($),a(tt),a(at),a(F),a(st),a(x),a(lt),a(nt),a(E),a(it),a(rt),a(S),a(ot),a(ct),a(pt),a(D),a(mt),a(K),a(ht),a(dt),a(P),a(Mt),a(ut),a(yt),a(ee),a(ft),a(te),a(Jt),a(ae),a(bt),a(Tt),a(wt),a(ne),a(Ut),a(jt),a(re),a(kt),a(oe),a(Ct),a(gt),a(Zt),a(me),a(Wt),a(he),a(It),a(Gt),a(Me),a(_t),a(Xt),a(ye),a(Bt),a(fe),a(vt),a(Nt),a(Rt),a(Te),a(Vt),a(Qt),a(Ue),a(zt),a(je),a(Yt),a($t),a(Ht),a(Ce),a(Ft),a(ge),a(xt),a(Ze),a(Et),a(qt),a(Ie),a(St),a(Ge),a(Lt),a(_e),a(At),a(Xe),a(Dt),a(Kt),a(Pt),a(Ne),a(Ot),a(Re),a(ea),a(ta),a(aa),a(Ye)),a(i),J(h,e),J(B,e),J(R,e),J(z,e),J(Y,e),J(H,e),J(C,e),J(q,e),J(L,e),J(A,e),J(g,e),J(Z,e),J(O,e),J(se,e),J(le,e),J(ie,e),J(ce,e),J(pe,e),J(de,e),J(ue,e),J(Je,e),J(be,e),J(we,e),J(W,e),J(ke,e),J(We,e),J(Be,e),J(ve,e),J(Ve,e),J(Qe,e)}}}const hs='{"title":"KV cache strategies","local":"kv-cache-strategies","sections":[{"title":"Default cache","local":"default-cache","sections":[],"depth":2},{"title":"Memory efficient caches","local":"memory-efficient-caches","sections":[{"title":"Offloaded cache","local":"offloaded-cache","sections":[],"depth":3},{"title":"Quantized cache","local":"quantized-cache","sections":[],"depth":3},{"title":"Sink cache","local":"sink-cache","sections":[],"depth":3}],"depth":2},{"title":"Speed optimized caches","local":"speed-optimized-caches","sections":[{"title":"Static cache","local":"static-cache","sections":[],"depth":3},{"title":"Offloaded static cache","local":"offloaded-static-cache","sections":[],"depth":3},{"title":"Sliding window cache","local":"sliding-window-cache","sections":[],"depth":3}],"depth":2},{"title":"Model caches","local":"model-caches","sections":[{"title":"Encoder-decoder cache","local":"encoder-decoder-cache","sections":[],"depth":3},{"title":"Model-specific caches","local":"model-specific-caches","sections":[],"depth":3}],"depth":2},{"title":"Iterative generation","local":"iterative-generation","sections":[],"depth":2},{"title":"Prefill a cache","local":"prefill-a-cache","sections":[],"depth":2}],"depth":1}';function ds(j){return Pa(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ws extends Oa{constructor(i){super(),es(this,i,ds,ms,Ka,{})}}export{ws as component};
