import{s as Gn,o as qn,n as Se}from"../chunks/scheduler.18a86fab.js";import{S as Wn,i as Fn,g as i,s,r as u,A as En,h as d,f as l,c as a,j as L,x as f,u as h,k as z,l as Rn,y as t,a as c,v as _,d as g,t as b,w as k}from"../chunks/index.98837b22.js";import{T as Vn}from"../chunks/Tip.77304350.js";import{D as G}from"../chunks/Docstring.6758ea23.js";import{C as Q}from"../chunks/CodeBlock.8d0c2e8a.js";import{E as Zn}from"../chunks/ExampleCodeBlock.0c42463f.js";import{H as yt}from"../chunks/Heading.7e7e0c5b.js";import{H as Bn,a as nn}from"../chunks/HfOption.6641485e.js";import{E as An}from"../chunks/index.499f366c.js";function Nn(j){let n,m="Click on the Code Llama models in the right sidebar for more examples of how to apply Code Llama to different coding tasks.";return{c(){n=i("p"),n.textContent=m},l(o){n=d(o,"P",{"data-svelte-h":!0}),f(n)!=="svelte-yvglo7"&&(n.textContent=m)},m(o,M){c(o,n,M)},p:Se,d(o){o&&l(n)}}}function Xn(j){let n,m;return n=new Q({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwcGlwZWxpbmUlMEElMEFwaXBlJTIwJTNEJTIwcGlwZWxpbmUoJTBBJTIwJTIwJTIwJTIwJTIydGV4dC1nZW5lcmF0aW9uJTIyJTJDJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0QlMjJtZXRhLWxsYW1hJTJGQ29kZUxsYW1hLTdiLWhmJTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRDAlMEEpJTBBJTBBJTIzJTIwYmFzaWMlMjBjb2RlJTIwZ2VuZXJhdGlvbiUwQXJlc3VsdCUyMCUzRCUyMHBpcGUoJTIyJTIzJTIwRnVuY3Rpb24lMjB0byUyMGNhbGN1bGF0ZSUyMHRoZSUyMGZhY3RvcmlhbCUyMG9mJTIwYSUyMG51bWJlciU1Q25kZWYlMjBmYWN0b3JpYWwobiklM0ElMjIlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDI1NiklMEFwcmludChyZXN1bHQlNUIwJTVEJTVCJ2dlbmVyYXRlZF90ZXh0JyU1RCklMEElMEElMjMlMjBpbmZpbGxpbmclMEFpbmZpbGxfcmVzdWx0JTIwJTNEJTIwcGlwZSglMjJkZWYlMjByZW1vdmVfbm9uX2FzY2lpKHMlM0ElMjBzdHIpJTIwLSUzRSUyMHN0ciUzQSU1Q24lMjAlMjAlMjAlMjAlNUMlMjIlNUMlMjIlNUMlMjIlMjAlM0NGSUxMX01FJTNFJTVDbiUyMCUyMCUyMCUyMHJldHVybiUyMHJlc3VsdCUyMiUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMjAwKSUwQXByaW50KGluZmlsbF9yZXN1bHQlNUIwJTVEJTVCJ2dlbmVyYXRlZF90ZXh0JyU1RCk=",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

pipe = pipeline(
    <span class="hljs-string">&quot;text-generation&quot;</span>,
    model=<span class="hljs-string">&quot;meta-llama/CodeLlama-7b-hf&quot;</span>,
    torch_dtype=torch.float16,
    device_map=<span class="hljs-number">0</span>
)

<span class="hljs-comment"># basic code generation</span>
result = pipe(<span class="hljs-string">&quot;# Function to calculate the factorial of a number\\ndef factorial(n):&quot;</span>, max_new_tokens=<span class="hljs-number">256</span>)
<span class="hljs-built_in">print</span>(result[<span class="hljs-number">0</span>][<span class="hljs-string">&#x27;generated_text&#x27;</span>])

<span class="hljs-comment"># infilling</span>
infill_result = pipe(<span class="hljs-string">&quot;def remove_non_ascii(s: str) -&gt; str:\\n    \\&quot;\\&quot;\\&quot; &lt;FILL_ME&gt;\\n    return result&quot;</span>, max_new_tokens=<span class="hljs-number">200</span>)
<span class="hljs-built_in">print</span>(infill_result[<span class="hljs-number">0</span>][<span class="hljs-string">&#x27;generated_text&#x27;</span>])`,wrap:!1}}),{c(){u(n.$$.fragment)},l(o){h(n.$$.fragment,o)},m(o,M){_(n,o,M),m=!0},p:Se,i(o){m||(g(n.$$.fragment,o),m=!0)},o(o){b(n.$$.fragment,o),m=!1},d(o){k(n,o)}}}function Hn(j){let n,m;return n=new Q({props:{code:"aW1wb3J0JTIwdG9yY2glMEFmcm9tJTIwdHJhbnNmb3JtZXJzJTIwaW1wb3J0JTIwQXV0b01vZGVsRm9yQ2F1c2FsTE0lMkMlMjBBdXRvVG9rZW5pemVyJTBBJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIybWV0YS1sbGFtYSUyRkNvZGVMbGFtYS03Yi1oZiUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJtZXRhLWxsYW1hJTJGQ29kZUxsYW1hLTdiLWhmJTIyJTJDJTBBJTIwJTIwJTIwJTIwdG9yY2hfZHR5cGUlM0R0b3JjaC5mbG9hdDE2JTJDJTBBJTIwJTIwJTIwJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjBhdHRuX2ltcGxlbWVudGF0aW9uJTNEJTIyc2RwYSUyMiUwQSklMEElMEElMjMlMjBiYXNpYyUyMGNvZGUlMjBnZW5lcmF0aW9uJTBBcHJvbXB0JTIwJTNEJTIwJTIyJTIzJTIwRnVuY3Rpb24lMjB0byUyMGNhbGN1bGF0ZSUyMHRoZSUyMGZhY3RvcmlhbCUyMG9mJTIwYSUyMG51bWJlciU1Q25kZWYlMjBmYWN0b3JpYWwobiklM0ElMjIlMEFpbnB1dF9pZHMlMjAlM0QlMjB0b2tlbml6ZXIocHJvbXB0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8oJTIyY3VkYSUyMiklMEElMEFvdXRwdXQlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSglMEElMjAlMjAlMjAlMjAqKmlucHV0X2lkcyUyQyUwQSUyMCUyMCUyMCUyMG1heF9uZXdfdG9rZW5zJTNEMjU2JTJDJTBBJTIwJTIwJTIwJTIwY2FjaGVfaW1wbGVtZW50YXRpb24lM0QlMjJzdGF0aWMlMjIlMEEpJTBBcHJpbnQodG9rZW5pemVyLmRlY29kZShvdXRwdXQlNUIwJTVEJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpKSUwQSUwQSUyMyUyMGluZmlsbGluZyUwQWluZmlsbF9wcm9tcHQlMjAlM0QlMjAlMjJkZWYlMjByZW1vdmVfbm9uX2FzY2lpKHMlM0ElMjBzdHIpJTIwLSUzRSUyMHN0ciUzQSU1Q24lMjAlMjAlMjAlMjAlNUMlMjIlNUMlMjIlNUMlMjIlMjAlM0NGSUxMX01FJTNFJTVDbiUyMCUyMCUyMCUyMHJldHVybiUyMHJlc3VsdCUyMiUwQWlucHV0X2lkcyUyMCUzRCUyMHRva2VuaXplcihpbmZpbGxfcHJvbXB0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8obW9kZWwuZGV2aWNlKSUwQSUwQWZpbGxlZF9vdXRwdXQlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0X2lkcyUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMjAwKSUwQWZpbGxlZF90ZXh0JTIwJTNEJTIwdG9rZW5pemVyLmRlY29kZShmaWxsZWRfb3V0cHV0JTVCMCU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSUwQXByaW50KGZpbGxlZF90ZXh0KQ==",highlighted:`<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/CodeLlama-7b-hf&quot;</span>)
model = AutoModelForCausalLM.from_pretrained(
    <span class="hljs-string">&quot;meta-llama/CodeLlama-7b-hf&quot;</span>,
    torch_dtype=torch.float16,
    device_map=<span class="hljs-string">&quot;auto&quot;</span>,
    attn_implementation=<span class="hljs-string">&quot;sdpa&quot;</span>
)

<span class="hljs-comment"># basic code generation</span>
prompt = <span class="hljs-string">&quot;# Function to calculate the factorial of a number\\ndef factorial(n):&quot;</span>
input_ids = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)

output = model.generate(
    **input_ids,
    max_new_tokens=<span class="hljs-number">256</span>,
    cache_implementation=<span class="hljs-string">&quot;static&quot;</span>
)
<span class="hljs-built_in">print</span>(tokenizer.decode(output[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))

<span class="hljs-comment"># infilling</span>
infill_prompt = <span class="hljs-string">&quot;def remove_non_ascii(s: str) -&gt; str:\\n    \\&quot;\\&quot;\\&quot; &lt;FILL_ME&gt;\\n    return result&quot;</span>
input_ids = tokenizer(infill_prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(model.device)

filled_output = model.generate(**input_ids, max_new_tokens=<span class="hljs-number">200</span>)
filled_text = tokenizer.decode(filled_output[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>)
<span class="hljs-built_in">print</span>(filled_text)`,wrap:!1}}),{c(){u(n.$$.fragment)},l(o){h(n.$$.fragment,o)},m(o,M){_(n,o,M),m=!0},p:Se,i(o){m||(g(n.$$.fragment,o),m=!0)},o(o){b(n.$$.fragment,o),m=!1},d(o){k(n,o)}}}function Yn(j){let n,m;return n=new Q({props:{code:"ZWNobyUyMC1lJTIwJTIyJTIzJTIwRnVuY3Rpb24lMjB0byUyMGNhbGN1bGF0ZSUyMHRoZSUyMGZhY3RvcmlhbCUyMG9mJTIwYSUyMG51bWJlciU1Q25kZWYlMjBmYWN0b3JpYWwobiklM0ElMjIlMjAlN0MlMjB0cmFuc2Zvcm1lcnMlMjBydW4lMjAtLXRhc2slMjB0ZXh0LWdlbmVyYXRpb24lMjAtLW1vZGVsJTIwbWV0YS1sbGFtYSUyRkNvZGVMbGFtYS03Yi1oZiUyMC0tZGV2aWNlJTIwMA==",highlighted:'<span class="hljs-built_in">echo</span> -e <span class="hljs-string">&quot;# Function to calculate the factorial of a number\\ndef factorial(n):&quot;</span> | transformers run --task text-generation --model meta-llama/CodeLlama-7b-hf --device 0',wrap:!1}}),{c(){u(n.$$.fragment)},l(o){h(n.$$.fragment,o)},m(o,M){_(n,o,M),m=!0},p:Se,i(o){m||(g(n.$$.fragment,o),m=!0)},o(o){b(n.$$.fragment,o),m=!1},d(o){k(n,o)}}}function Sn(j){let n,m,o,M,T,v;return n=new nn({props:{id:"usage",option:"Pipeline",$$slots:{default:[Xn]},$$scope:{ctx:j}}}),o=new nn({props:{id:"usage",option:"AutoModel",$$slots:{default:[Hn]},$$scope:{ctx:j}}}),T=new nn({props:{id:"usage",option:"transformers CLI",$$slots:{default:[Yn]},$$scope:{ctx:j}}}),{c(){u(n.$$.fragment),m=s(),u(o.$$.fragment),M=s(),u(T.$$.fragment)},l(p){h(n.$$.fragment,p),m=a(p),h(o.$$.fragment,p),M=a(p),h(T.$$.fragment,p)},m(p,w){_(n,p,w),c(p,m,w),_(o,p,w),c(p,M,w),_(T,p,w),v=!0},p(p,w){const D={};w&2&&(D.$$scope={dirty:w,ctx:p}),n.$set(D);const q={};w&2&&(q.$$scope={dirty:w,ctx:p}),o.$set(q);const Qe={};w&2&&(Qe.$$scope={dirty:w,ctx:p}),T.$set(Qe)},i(p){v||(g(n.$$.fragment,p),g(o.$$.fragment,p),g(T.$$.fragment,p),v=!0)},o(p){b(n.$$.fragment,p),b(o.$$.fragment,p),b(T.$$.fragment,p),v=!1},d(p){p&&(l(m),l(M)),k(n,p),k(o,p),k(T,p)}}}function Qn(j){let n,m="sequence pair mask has the following format:",o,M,T;return M=new Q({props:{code:"MCUyMDAlMjAwJTIwMCUyMDAlMjAwJTIwMCUyMDAlMjAwJTIwMCUyMDAlMjAxJTIwMSUyMDElMjAxJTIwMSUyMDElMjAxJTIwMSUyMDElMEElN0MlMjBmaXJzdCUyMHNlcXVlbmNlJTIwJTIwJTIwJTIwJTdDJTIwc2Vjb25kJTIwc2VxdWVuY2UlMjAlN0M=",highlighted:`0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 1 </span>1<span class="hljs-number"> 1 </span>1<span class="hljs-number"> 1 </span>1<span class="hljs-number"> 1 </span>1 1
| first sequence    | second sequence |`,wrap:!1}}),{c(){n=i("p"),n.textContent=m,o=s(),u(M.$$.fragment)},l(v){n=d(v,"P",{"data-svelte-h":!0}),f(n)!=="svelte-16klr56"&&(n.textContent=m),o=a(v),h(M.$$.fragment,v)},m(v,p){c(v,n,p),c(v,o,p),_(M,v,p),T=!0},p:Se,i(v){T||(g(M.$$.fragment,v),T=!0)},o(v){b(M.$$.fragment,v),T=!1},d(v){v&&(l(n),l(o)),k(M,v)}}}function Dn(j){let n,m;return n=new Q({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMENvZGVMbGFtYVRva2VuaXplckZhc3QlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBDb2RlTGxhbWFUb2tlbml6ZXJGYXN0LmZyb21fcHJldHJhaW5lZCglMjJoZi1pbnRlcm5hbC10ZXN0aW5nJTJGbGxhbWEtdG9rZW5pemVyJTIyKSUwQXRva2VuaXplci5lbmNvZGUoJTIySGVsbG8lMjB0aGlzJTIwaXMlMjBhJTIwdGVzdCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> CodeLlamaTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = CodeLlamaTokenizerFast.from_pretrained(<span class="hljs-string">&quot;hf-internal-testing/llama-tokenizer&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.encode(<span class="hljs-string">&quot;Hello this is a test&quot;</span>)
[<span class="hljs-number">1</span>, <span class="hljs-number">15043</span>, <span class="hljs-number">445</span>, <span class="hljs-number">338</span>, <span class="hljs-number">263</span>, <span class="hljs-number">1243</span>]`,wrap:!1}}),{c(){u(n.$$.fragment)},l(o){h(n.$$.fragment,o)},m(o,M){_(n,o,M),m=!0},p:Se,i(o){m||(g(n.$$.fragment,o),m=!0)},o(o){b(n.$$.fragment,o),m=!1},d(o){k(n,o)}}}function Pn(j){let n,m,o,M,T,v=`<div class="flex flex-wrap space-x-1"><img alt="PyTorch" src="https://img.shields.io/badge/PyTorch-DE3412?style=flat&amp;logo=pytorch&amp;logoColor=white"/> <img alt="Flax" src="https://img.shields.io/badge/Flax-29a79b.svg?style=flat&amp;logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAC0AAAAtCAMAAAANxBKoAAAC7lBMVEUAAADg5vYHPVgAoJH+/v76+v39/f9JbLP///9+AIgAnY3///+mcqzt8fXy9fgkXa3Ax9709fr+///9/f8qXq49qp5AaLGMwrv8/P0eW60VWawxYq8yqJzG2dytt9Wyu9elzci519Lf3O3S2efY3OrY0+Xp7PT///////+dqNCexMc6Z7AGpJeGvbenstPZ5ejQ1OfJzOLa7ejh4+/r8fT29vpccbklWK8PVa0AS6ghW63O498vYa+lsdKz1NDRt9Kw1c672tbD3tnAxt7R6OHp5vDe7OrDyuDn6vLl6/EAQKak0MgATakkppo3ZK/Bz9y8w9yzu9jey97axdvHzeG21NHH4trTwthKZrVGZLSUSpuPQJiGAI+GAI8SWKydycLL4d7f2OTi1+S9xNzL0ePT6OLGzeEAo5U0qJw/aLEAo5JFa7JBabEAp5Y4qZ2QxLyKmsm3kL2xoMOehrRNb7RIbbOZgrGre68AUqwAqZqNN5aKJ5N/lMq+qsd8kMa4pcWzh7muhLMEV69juq2kbKqgUaOTR5uMMZWLLZSGAI5VAIdEAH+ovNDHuNCnxcy3qcaYx8K8msGplrx+wLahjbYdXrV6vbMvYK9DrZ8QrZ8tqJuFms+Sos6sw8ecy8RffsNVeMCvmb43aLltv7Q4Y7EZWK4QWa1gt6meZKUdr6GOAZVeA4xPAISyveLUwtivxtKTpNJ2jcqfvcltiMiwwcfAoMVxhL+Kx7xjdrqTe60tsaNQs6KaRKACrJ6UTZwkqpqTL5pkHY4AloSgsd2ptNXPvNOOncuxxsqFl8lmg8apt8FJcr9EbryGxLqlkrkrY7dRa7ZGZLQ5t6iXUZ6PPpgVpZeJCJFKAIGareTa0+KJod3H0deY2M+esM25usmYu8d2zsJOdcBVvrCLbqcAOaaHaKQAMaScWqKBXqCXMJ2RHpiLF5NmJZAdAHN2kta11dKu1M+DkcZLdb+Mcql3TppyRJdzQ5ZtNZNlIY+DF4+voCOQAAAAZ3RSTlMABAT+MEEJ/RH+/TP+Zlv+pUo6Ifz8+fco/fz6+evr39S9nJmOilQaF/7+/f38+smmoYp6b1T+/v7++vj189zU0tDJxsGzsrKSfv34+Pf27dDOysG9t6+n/vv6+vr59uzr1tG+tZ6Qg9Ym3QAABR5JREFUSMeNlVVUG1EQhpcuxEspXqS0SKEtxQp1d3d332STTRpIQhIISQgJhODu7lAoDoUCpe7u7u7+1puGpqnCPOyZvffbOXPm/PsP9JfQgyCC+tmTABTOcbxDz/heENS7/1F+9nhvkHePG0wNDLbGWwdXL+rbLWvpmZHXD8+gMfBjTh+aSe6Gnn7lwQIOTR0c8wfX3PWgv7avbdKwf/ZoBp1Gp/PvuvXW3vw5ib7emnTW4OR+3D4jB9vjNJ/7gNvfWWeH/TO/JyYrsiKCRjVEZA3UB+96kON+DxOQ/NLE8PE5iUYgIXjFnCOlxEQMaSGVxjg4gxOnEycGz8bptuNjVx08LscIgrzH3umcn+KKtiBIyvzOO2O99aAdR8cF19oZalnCtvREUw79tCd5sow1g1UKM6kXqUx4T8wsi3sTjJ3yzDmmhenLXLpo8u45eG5y4Vvbk6kkC4LLtJMowkSQxmk4ggVJEG+7c6QpHT8vvW9X7/o7+3ELmiJi2mEzZJiz8cT6TBlanBk70cB5GGIGC1gRDdZ00yADLW1FL6gqhtvNXNG5S9gdSrk4M1qu7JAsmYshzDS4peoMrU/gT7qQdqYGZaYhxZmVbGJAm/CS/HloWyhRUlknQ9KYcExTwS80d3VNOxUZJpITYyspl0LbhArhpZCD9cRWEQuhYkNGMHToQ/2Cs6swJlb39CsllxdXX6IUKh/H5jbnSsPKjgmoaFQ1f8wRLR0UnGE/RcDEjj2jXG1WVTwUs8+zxfcrVO+vSsuOpVKxCfYZiQ0/aPKuxQbQ8lIz+DClxC8u+snlcJ7Yr1z1JPqUH0V+GDXbOwAib931Y4Imaq0NTIXPXY+N5L18GJ37SVWu+hwXff8l72Ds9XuwYIBaXPq6Shm4l+Vl/5QiOlV+uTk6YR9PxKsI9xNJny31ygK1e+nIRC1N97EGkFPI+jCpiHe5PCEy7oWqWSwRrpOvhFzcbTWMbm3ZJAOn1rUKpYIt/lDhW/5RHHteeWFN60qo98YJuoq1nK3uW5AabyspC1BcIEpOhft+SZAShYoLSvnmSfnYADUERP5jJn2h5XtsgCRuhYQqAvwTwn33+YWEKUI72HX5AtfSAZDe8F2DtPPm77afhl0EkthzuCQU0BWApgQIH9+KB0JhopMM7bJrdTRoleM2JAVNMyPF+wdoaz+XJpGoVAQ7WXUkcV7gT3oUZyi/ISIJAVKhgNp+4b4veCFhYVJw4locdSjZCp9cPUhLF9EZ3KKzURepMEtCDPP3VcWFx4UIiZIklIpFNfHpdEafIF2aRmOcrUmjohbT2WUllbmRvgfbythbQO3222fpDJoufaQPncYYuqoGtUEsCJZL6/3PR5b4syeSjZMQG/T2maGANlXT2v8S4AULWaUkCxfLyW8iW4kdka+nEMjxpL2NCwsYNBp+Q61PF43zyDg9Bm9+3NNySn78jMZUUkumqE4Gp7JmFOdP1vc8PpRrzj9+wPinCy8K1PiJ4aYbnTYpCCbDkBSbzhu2QJ1Gd82t8jI8TH51+OzvXoWbnXUOBkNW+0mWFwGcGOUVpU81/n3TOHb5oMt2FgYGjzau0Nif0Ss7Q3XB33hjjQHjHA5E5aOyIQc8CBrLdQSs3j92VG+3nNEjbkbdbBr9zm04ruvw37vh0QKOdeGIkckc80fX3KH/h7PT4BOjgCty8VZ5ux1MoO5Cf5naca2LAsEgehI+drX8o/0Nu+W0m6K/I9gGPd/dfx/EN/wN62AhsBWuAAAAAElFTkSuQmCC
        "/></div>`,p,w,D,q,Qe='<a href="https://huggingface.co/papers/2308.12950" rel="nofollow">Code Llama</a> is a specialized family of large language models based on <a href="./llama2">Llama 2</a> for coding tasks.  It comes in different flavors - general code, Python-specific, and instruction-following variant - all available in 7B, 13B, 34B, and 70B parameters. Code Llama models can generate, explain, and even fill in missing parts of your code (called “infilling”). It can also handle very long contexts with stable generation up to 100k tokens, even though it was trained on sequences of 16K tokens.',Oe,P,on='You can find all the original Code Llama checkpoints under the <a href="https://huggingface.co/collections/meta-llama/code-llama-family-661da32d0a9d678b6f55b933" rel="nofollow">Code Llama</a> collection.',Ke,E,et,O,sn='The example below demonstrates how to generate code with <a href="/docs/transformers/main/en/main_classes/pipelines#transformers.Pipeline">Pipeline</a>, or the <a href="/docs/transformers/main/en/model_doc/auto#transformers.AutoModel">AutoModel</a>, and from the command line.',tt,R,nt,K,an='Quantization reduces the memory burden of large models by representing the weights in a lower precision. Refer to the <a href="../quantization/overview">Quantization</a> overview for more available quantization backends.',ot,ee,ln='The example below uses <a href="../quantization/bitsandbytes">bitsandbytes</a> to only quantize the weights to 4-bits.',st,te,at,ne,rn='Use the <a href="https://github.com/huggingface/transformers/blob/beb9b5b02246b9b7ee81ddf938f93f44cfeaad19/src/transformers/utils/attention_visualizer.py#L139" rel="nofollow">AttentionMaskVisualizer</a> to better understand what tokens the model can and cannot attend to.',lt,oe,rt,V,dn='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/codellama-attn-mask.png"/>',it,se,dt,$,Te,cn="<p>Infilling is only available in the 7B and 13B base models, and not in the Python, Instruct, 34B, or 70B models.</p>",Tt,ae,ve,pn='Use the <code>&lt;FILL_ME&gt;</code> token where you want your input to be filled. The tokenizer splits this token to create a formatted input string that follows the <a href="https://github.com/facebookresearch/codellama/blob/cb51c14ec761370ba2e2bc351374a79265d0465e/llama/generation.py#L402" rel="nofollow">original training pattern</a>. This is more robust than preparing the pattern yourself.',vt,le,Jt,Je,mn="<p>Use <code>bfloat16</code> for further training or fine-tuning and <code>float16</code> for inference.</p>",wt,we,fn="<p>The <code>BOS</code> character is not used for infilling when encoding the prefix or suffix, but only at the beginning of each prompt.</p>",xt,xe,un='<p>The tokenizer is a byte-pair encoding model based on <a href="https://github.com/google/sentencepiece" rel="nofollow">SentencePiece</a>. During decoding, if the first token is the start of the word (for example, “Banana”), the tokenizer doesn’t prepend the prefix space to the string.</p>',ct,re,pt,x,ie,Ct,Ce,hn=`Construct a CodeLlama tokenizer. Based on byte-level Byte-Pair-Encoding. The default padding token is unset as
there is no padding token in the original model.`,jt,je,_n=`The default configuration match that of
<a href="https://huggingface.co/meta-llama/CodeLlama-7b-Instruct-hf/blob/main/tokenizer_config.json" rel="nofollow">codellama/CodeLlama-7b-Instruct-hf</a>
which supports prompt infilling.`,Lt,Le,de,$t,B,ce,Ut,$e,gn=`Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer <code>prepare_for_model</code> method.`,zt,Z,pe,It,Ue,bn="Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT",Zt,A,Gt,ze,kn="if token_ids_1 is None, only returns the first portion of the mask (0s).",qt,N,me,Wt,Ie,Mn="Save the vocabulary and special tokens file to a directory.",mt,fe,ft,y,ue,Ft,Ze,yn="Construct a Llama tokenizer. Based on byte-level Byte-Pair-Encoding.",Et,Ge,Tn="This uses notably ByteFallback and no normalization.",Rt,X,Vt,qe,vn=`If you want to change the <code>bos_token</code> or the <code>eos_token</code>, make sure to specify them when initializing the model, or
call <code>tokenizer.update_post_processor()</code> to make sure that the post-processing is correctly done (otherwise the
values of the first token and final token of an encoded sequence will not be correct). For more details, checkout
[post-processors] (<a href="https://huggingface.co/docs/tokenizers/api/post-processors" rel="nofollow">https://huggingface.co/docs/tokenizers/api/post-processors</a>) documentation.`,Bt,We,Jn=`This tokenizer inherits from <a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a> which contains most of the main methods. Users should
refer to this superclass for more information regarding those methods. The default configuration match that of
<a href="https://huggingface.co/meta-llama/CodeLlama-7b-Instruct-hf/blob/main/tokenizer_config.json" rel="nofollow">meta-llama/CodeLlama-7b-Instruct-hf</a>
which supports prompt infilling.`,At,U,he,Nt,Fe,wn=`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. The special tokens depend on calling set_lang.`,Xt,Ee,xn="An NLLB sequence has the following format, where <code>X</code> represents the sequence:",Ht,Re,Cn="<li><code>input_ids</code> (for encoder) <code>X [eos, src_lang_code]</code></li> <li><code>decoder_input_ids</code>: (for decoder) <code>X [eos, tgt_lang_code]</code></li>",Yt,Ve,jn=`BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a
separator.`,St,H,_e,Qt,Be,Ln=`Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer <code>prepare_for_model</code> or <code>encode_plus</code> methods.`,Dt,W,ge,Pt,Ae,$n=`Create the token type IDs corresponding to the sequences passed. <a href="../glossary#token-type-ids">What are token type
IDs?</a>`,Ot,Ne,Un="Should be overridden in a subclass if the model has a special way of building those.",Kt,Y,be,en,Xe,zn="Updates the underlying post processor with the current <code>bos_token</code> and <code>eos_token</code>.",tn,He,ke,ut,Me,ht,De,_t;return w=new yt({props:{title:"CodeLlama",local:"codellama",headingTag:"h1"}}),E=new Vn({props:{warning:!1,$$slots:{default:[Nn]},$$scope:{ctx:j}}}),R=new Bn({props:{id:"usage",options:["Pipeline","AutoModel","transformers CLI"],$$slots:{default:[Sn]},$$scope:{ctx:j}}}),te=new Q({props:{code:"JTIzJTIwcGlwJTIwaW5zdGFsbCUyMGJpdHNhbmRieXRlcyUwQWltcG9ydCUyMHRvcmNoJTBBZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNJTJDJTIwQ29kZUxsYW1hVG9rZW5pemVyJTJDJTIwQml0c0FuZEJ5dGVzQ29uZmlnJTBBJTBBYm5iX2NvbmZpZyUyMCUzRCUyMEJpdHNBbmRCeXRlc0NvbmZpZyhsb2FkX2luXzRiaXQlM0RUcnVlJTJDJTIwYm5iXzRiaXRfY29tcHV0ZV9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTJDJTIwYm5iXzRiaXRfcXVhbnRfdHlwZSUzRCUyMm5mNCUyMiUyQyUyMGJuYl80Yml0X3VzZV9kb3VibGVfcXVhbnQlM0RUcnVlKSUwQXRva2VuaXplciUyMCUzRCUyMENvZGVMbGFtYVRva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIybWV0YS1sbGFtYSUyRkNvZGVMbGFtYS0zNGItaGYlMjIpJTBBbW9kZWwlMjAlM0QlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTS5mcm9tX3ByZXRyYWluZWQoJTBBJTIwJTIwJTIwJTIybWV0YS1sbGFtYSUyRkNvZGVMbGFtYS0zNGItaGYlMjIlMkMlMEElMjAlMjAlMjB0b3JjaF9kdHlwZSUzRHRvcmNoLmJmbG9hdDE2JTJDJTBBJTIwJTIwJTIwZGV2aWNlX21hcCUzRCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjBxdWFudGl6YXRpb25fY29uZmlnJTNEYm5iX2NvbmZpZyUwQSklMEElMEFwcm9tcHQlMjAlM0QlMjAlMjIlMjMlMjBXcml0ZSUyMGElMjBQeXRob24lMjBmdW5jdGlvbiUyMHRvJTIwY2hlY2slMjBpZiUyMGElMjBzdHJpbmclMjBpcyUyMGElMjBwYWxpbmRyb21lJTVDbmRlZiUyMGlzX3BhbGluZHJvbWUocyklM0ElMjIlMEFpbnB1dF9pZHMlMjAlM0QlMjB0b2tlbml6ZXIocHJvbXB0JTJDJTIwcmV0dXJuX3RlbnNvcnMlM0QlMjJwdCUyMikudG8oJTIyY3VkYSUyMiklMEElMEFvdXRwdXQlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZSgqKmlucHV0X2lkcyUyQyUyMG1heF9uZXdfdG9rZW5zJTNEMjAwJTJDJTIwY2FjaGVfaW1wbGVtZW50YXRpb24lM0QlMjJzdGF0aWMlMjIpJTBBcHJpbnQodG9rZW5pemVyLmRlY29kZShvdXRwdXQlNUIwJTVEJTJDJTIwc2tpcF9zcGVjaWFsX3Rva2VucyUzRFRydWUpKQ==",highlighted:`<span class="hljs-comment"># pip install bitsandbytes</span>
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForCausalLM, CodeLlamaTokenizer, BitsAndBytesConfig

bnb_config = BitsAndBytesConfig(load_in_4bit=<span class="hljs-literal">True</span>, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_quant_type=<span class="hljs-string">&quot;nf4&quot;</span>, bnb_4bit_use_double_quant=<span class="hljs-literal">True</span>)
tokenizer = CodeLlamaTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/CodeLlama-34b-hf&quot;</span>)
model = AutoModelForCausalLM.from_pretrained(
   <span class="hljs-string">&quot;meta-llama/CodeLlama-34b-hf&quot;</span>,
   torch_dtype=torch.bfloat16,
   device_map=<span class="hljs-string">&quot;auto&quot;</span>,
   quantization_config=bnb_config
)

prompt = <span class="hljs-string">&quot;# Write a Python function to check if a string is a palindrome\\ndef is_palindrome(s):&quot;</span>
input_ids = tokenizer(prompt, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).to(<span class="hljs-string">&quot;cuda&quot;</span>)

output = model.generate(**input_ids, max_new_tokens=<span class="hljs-number">200</span>, cache_implementation=<span class="hljs-string">&quot;static&quot;</span>)
<span class="hljs-built_in">print</span>(tokenizer.decode(output[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`,wrap:!1}}),oe=new Q({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy51dGlscy5hdHRlbnRpb25fdmlzdWFsaXplciUyMGltcG9ydCUyMEF0dGVudGlvbk1hc2tWaXN1YWxpemVyJTBBJTBBdmlzdWFsaXplciUyMCUzRCUyMEF0dGVudGlvbk1hc2tWaXN1YWxpemVyKCUyMm1ldGEtbGxhbWElMkZDb2RlTGxhbWEtN2ItaGYlMjIpJTBBdmlzdWFsaXplciglMjIlMjIlMjJkZWYlMjBmdW5jKGElMkMlMjBiKSUzQSUwQSUyMCUyMHJldHVybiUyMGElMjAlMkIlMjBiJTIyJTIyJTIyKQ==",highlighted:`<span class="hljs-keyword">from</span> transformers.utils.attention_visualizer <span class="hljs-keyword">import</span> AttentionMaskVisualizer

visualizer = AttentionMaskVisualizer(<span class="hljs-string">&quot;meta-llama/CodeLlama-7b-hf&quot;</span>)
visualizer(<span class="hljs-string">&quot;&quot;&quot;def func(a, b):
  return a + b&quot;&quot;&quot;</span>)`,wrap:!1}}),se=new yt({props:{title:"Notes",local:"notes",headingTag:"h2"}}),le=new Q({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMExsYW1hRm9yQ2F1c2FsTE0lMkMlMjBDb2RlTGxhbWFUb2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBDb2RlTGxhbWFUb2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMm1ldGEtbGxhbWElMkZDb2RlTGxhbWEtN2ItaGYlMjIpJTBBbW9kZWwlMjAlM0QlMjBMbGFtYUZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJtZXRhLWxsYW1hJTJGQ29kZUxsYW1hLTdiLWhmJTIyKSUwQVBST01QVCUyMCUzRCUyMCcnJ2RlZiUyMHJlbW92ZV9ub25fYXNjaWkocyUzQSUyMHN0ciklMjAtJTNFJTIwc3RyJTNBJTBBJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTIwJTNDRklMTF9NRSUzRSUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHJlc3VsdCUwQScnJyUwQWlucHV0X2lkcyUyMCUzRCUyMHRva2VuaXplcihQUk9NUFQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSU1QiUyMmlucHV0X2lkcyUyMiU1RCUwQWdlbmVyYXRlZF9pZHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZShpbnB1dF9pZHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDEyOCklMEElMEFmaWxsaW5nJTIwJTNEJTIwdG9rZW5pemVyLmJhdGNoX2RlY29kZShnZW5lcmF0ZWRfaWRzJTVCJTNBJTJDJTIwaW5wdXRfaWRzLnNoYXBlJTVCMSU1RCUzQSU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlMjAlM0QlMjBUcnVlKSU1QjAlNUQlMEFwcmludChQUk9NUFQucmVwbGFjZSglMjIlM0NGSUxMX01FJTNFJTIyJTJDJTIwZmlsbGluZykp",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> LlamaForCausalLM, CodeLlamaTokenizer

tokenizer = CodeLlamaTokenizer.from_pretrained(<span class="hljs-string">&quot;meta-llama/CodeLlama-7b-hf&quot;</span>)
model = LlamaForCausalLM.from_pretrained(<span class="hljs-string">&quot;meta-llama/CodeLlama-7b-hf&quot;</span>)
PROMPT = <span class="hljs-string">&#x27;&#x27;&#x27;def remove_non_ascii(s: str) -&gt; str:
    &quot;&quot;&quot; &lt;FILL_ME&gt;
    return result
&#x27;&#x27;&#x27;</span>
input_ids = tokenizer(PROMPT, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]
generated_ids = model.generate(input_ids, max_new_tokens=<span class="hljs-number">128</span>)

filling = tokenizer.batch_decode(generated_ids[:, input_ids.shape[<span class="hljs-number">1</span>]:], skip_special_tokens = <span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
<span class="hljs-built_in">print</span>(PROMPT.replace(<span class="hljs-string">&quot;&lt;FILL_ME&gt;&quot;</span>, filling))`,wrap:!1}}),re=new yt({props:{title:"CodeLlamaTokenizer",local:"transformers.CodeLlamaTokenizer",headingTag:"h2"}}),ie=new G({props:{name:"class transformers.CodeLlamaTokenizer",anchor:"transformers.CodeLlamaTokenizer",parameters:[{name:"vocab_file",val:""},{name:"unk_token",val:" = '<unk>'"},{name:"bos_token",val:" = '<s>'"},{name:"eos_token",val:" = '</s>'"},{name:"prefix_token",val:" = '▁<PRE>'"},{name:"middle_token",val:" = '▁<MID>'"},{name:"suffix_token",val:" = '▁<SUF>'"},{name:"eot_token",val:" = '▁<EOT>'"},{name:"fill_token",val:" = '<FILL_ME>'"},{name:"suffix_first",val:" = False"},{name:"sp_model_kwargs",val:": typing.Optional[typing.Dict[str, typing.Any]] = None"},{name:"add_bos_token",val:" = True"},{name:"add_eos_token",val:" = False"},{name:"clean_up_tokenization_spaces",val:" = False"},{name:"additional_special_tokens",val:" = None"},{name:"use_default_system_prompt",val:" = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizer.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
Path to the vocabulary file.`,name:"vocab_file"},{anchor:"transformers.CodeLlamaTokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;unk&gt;&quot;</code>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead.`,name:"unk_token"},{anchor:"transformers.CodeLlamaTokenizer.bos_token",description:`<strong>bos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;s&gt;&quot;</code>) &#x2014;
The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.`,name:"bos_token"},{anchor:"transformers.CodeLlamaTokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;/s&gt;&quot;</code>) &#x2014;
The end of sequence token.</p>
<div class="course-tip  bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">
						
<p>When building a sequence using special tokens, this is not the token that is used for the end of sequence.
The token used is the <code>sep_token</code>.</p>

					</div>`,name:"eos_token"},{anchor:"transformers.CodeLlamaTokenizer.prefix_token",description:`<strong>prefix_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;PRE&gt;&quot;</code>) &#x2014;
Prefix token used for infilling.`,name:"prefix_token"},{anchor:"transformers.CodeLlamaTokenizer.middle_token",description:`<strong>middle_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;MID&gt;&quot;</code>) &#x2014;
Middle token used for infilling.`,name:"middle_token"},{anchor:"transformers.CodeLlamaTokenizer.suffix_token",description:`<strong>suffix_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;SUF&gt;&quot;</code>) &#x2014;
Suffix token used for infilling.`,name:"suffix_token"},{anchor:"transformers.CodeLlamaTokenizer.eot_token",description:`<strong>eot_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;EOT&gt;&quot;</code>) &#x2014;
End of text token used for infilling.`,name:"eot_token"},{anchor:"transformers.CodeLlamaTokenizer.fill_token",description:`<strong>fill_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;FILL_ME&gt;&quot;</code>) &#x2014;
The token used to split the input between the prefix and suffix.`,name:"fill_token"},{anchor:"transformers.CodeLlamaTokenizer.suffix_first",description:`<strong>suffix_first</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether the input prompt and suffix should be formatted with the suffix first.`,name:"suffix_first"},{anchor:"transformers.CodeLlamaTokenizer.sp_model_kwargs",description:`<strong>sp_model_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Will be passed to the <code>SentencePieceProcessor.__init__()</code> method. The <a href="https://github.com/google/sentencepiece/tree/master/python" rel="nofollow">Python wrapper for
SentencePiece</a> can be used, among other things,
to set:</p>
<ul>
<li>
<p><code>enable_sampling</code>: Enable subword regularization.</p>
</li>
<li>
<p><code>nbest_size</code>: Sampling parameters for unigram. Invalid for BPE-Dropout.</p>
<ul>
<li><code>nbest_size = {0,1}</code>: No sampling is performed.</li>
<li><code>nbest_size &gt; 1</code>: samples from the nbest_size results.</li>
<li><code>nbest_size &lt; 0</code>: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
using forward-filtering-and-backward-sampling algorithm.</li>
</ul>
</li>
<li>
<p><code>alpha</code>: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
BPE-dropout.</p>
</li>
</ul>`,name:"sp_model_kwargs"},{anchor:"transformers.CodeLlamaTokenizer.add_bos_token",description:`<strong>add_bos_token</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to add a beginning of sequence token at the start of sequences.`,name:"add_bos_token"},{anchor:"transformers.CodeLlamaTokenizer.add_eos_token",description:`<strong>add_eos_token</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to add an end of sequence token at the end of sequences.`,name:"add_eos_token"},{anchor:"transformers.CodeLlamaTokenizer.clean_up_tokenization_spaces",description:`<strong>clean_up_tokenization_spaces</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to clean up the tokenization spaces.`,name:"clean_up_tokenization_spaces"},{anchor:"transformers.CodeLlamaTokenizer.additional_special_tokens",description:`<strong>additional_special_tokens</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
Additional special tokens used by the tokenizer.`,name:"additional_special_tokens"},{anchor:"transformers.CodeLlamaTokenizer.use_default_system_prompt",description:`<strong>use_default_system_prompt</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the default system prompt for Llama should be used.`,name:"use_default_system_prompt"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/code_llama/tokenization_code_llama.py#L50"}}),de=new G({props:{name:"build_inputs_with_special_tokens",anchor:"transformers.CodeLlamaTokenizer.build_inputs_with_special_tokens",parameters:[{name:"token_ids_0",val:""},{name:"token_ids_1",val:" = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/code_llama/tokenization_code_llama.py#L359"}}),ce=new G({props:{name:"get_special_tokens_mask",anchor:"transformers.CodeLlamaTokenizer.get_special_tokens_mask",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"},{name:"already_has_special_tokens",val:": bool = False"}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizer.get_special_tokens_mask.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.CodeLlamaTokenizer.get_special_tokens_mask.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"},{anchor:"transformers.CodeLlamaTokenizer.get_special_tokens_mask.already_has_special_tokens",description:`<strong>already_has_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the token list is already formatted with special tokens for the model.`,name:"already_has_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/code_llama/tokenization_code_llama.py#L371",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),pe=new G({props:{name:"create_token_type_ids_from_sequences",anchor:"transformers.CodeLlamaTokenizer.create_token_type_ids_from_sequences",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizer.create_token_type_ids_from_sequences.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of ids.`,name:"token_ids_0"},{anchor:"transformers.CodeLlamaTokenizer.create_token_type_ids_from_sequences.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/code_llama/tokenization_code_llama.py#L409",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>List of <a href="../glossary#token-type-ids">token type IDs</a> according to the given sequence(s).</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),A=new Zn({props:{anchor:"transformers.CodeLlamaTokenizer.create_token_type_ids_from_sequences.example",$$slots:{default:[Qn]},$$scope:{ctx:j}}}),me=new G({props:{name:"save_vocabulary",anchor:"transformers.CodeLlamaTokenizer.save_vocabulary",parameters:[{name:"save_directory",val:""},{name:"filename_prefix",val:": typing.Optional[str] = None"}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizer.save_vocabulary.save_directory",description:`<strong>save_directory</strong> (<code>str</code>) &#x2014;
The directory in which to save the vocabulary.`,name:"save_directory"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/code_llama/tokenization_code_llama.py#L331",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>Paths to the files saved.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>Tuple(str)</code></p>
`}}),fe=new yt({props:{title:"CodeLlamaTokenizerFast",local:"transformers.CodeLlamaTokenizerFast",headingTag:"h2"}}),ue=new G({props:{name:"class transformers.CodeLlamaTokenizerFast",anchor:"transformers.CodeLlamaTokenizerFast",parameters:[{name:"vocab_file",val:" = None"},{name:"tokenizer_file",val:" = None"},{name:"clean_up_tokenization_spaces",val:" = False"},{name:"unk_token",val:" = '<unk>'"},{name:"bos_token",val:" = '<s>'"},{name:"eos_token",val:" = '</s>'"},{name:"prefix_token",val:" = '▁<PRE>'"},{name:"middle_token",val:" = '▁<MID>'"},{name:"suffix_token",val:" = '▁<SUF>'"},{name:"eot_token",val:" = '▁<EOT>'"},{name:"fill_token",val:" = '<FILL_ME>'"},{name:"additional_special_tokens",val:" = None"},{name:"add_bos_token",val:" = True"},{name:"add_eos_token",val:" = False"},{name:"use_default_system_prompt",val:" = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizerFast.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>, <em>optional</em>) &#x2014;
<a href="https://github.com/google/sentencepiece" rel="nofollow">SentencePiece</a> file (generally has a .model extension) that
contains the vocabulary necessary to instantiate a tokenizer.`,name:"vocab_file"},{anchor:"transformers.CodeLlamaTokenizerFast.tokenizer_file",description:`<strong>tokenizer_file</strong> (<code>str</code>, <em>optional</em>) &#x2014;
<a href="https://github.com/huggingface/tokenizers" rel="nofollow">tokenizers</a> file (generally has a .json extension) that
contains everything needed to load the tokenizer.`,name:"tokenizer_file"},{anchor:"transformers.CodeLlamaTokenizerFast.clean_up_tokenization_spaces",description:`<strong>clean_up_tokenization_spaces</strong> (<code>str</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to cleanup spaces after decoding, cleanup consists in removing potential artifacts like extra
spaces.`,name:"clean_up_tokenization_spaces"},{anchor:"transformers.CodeLlamaTokenizerFast.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;unk&gt;&quot;</code>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead.`,name:"unk_token"},{anchor:"transformers.CodeLlamaTokenizerFast.bos_token",description:`<strong>bos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;s&gt;&quot;</code>) &#x2014;
The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.`,name:"bos_token"},{anchor:"transformers.CodeLlamaTokenizerFast.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;/s&gt;&quot;</code>) &#x2014;
The end of sequence token.`,name:"eos_token"},{anchor:"transformers.CodeLlamaTokenizerFast.prefix_token",description:`<strong>prefix_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;PRE&gt;&quot;</code>) &#x2014;
Prefix token used for infilling.`,name:"prefix_token"},{anchor:"transformers.CodeLlamaTokenizerFast.middle_token",description:`<strong>middle_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;MID&gt;&quot;</code>) &#x2014;
Middle token used for infilling.`,name:"middle_token"},{anchor:"transformers.CodeLlamaTokenizerFast.suffix_token",description:`<strong>suffix_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;SUF&gt;&quot;</code>) &#x2014;
Suffix token used for infilling.`,name:"suffix_token"},{anchor:"transformers.CodeLlamaTokenizerFast.eot_token",description:`<strong>eot_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;EOT&gt;&quot;</code>) &#x2014;
End of text token used for infilling.`,name:"eot_token"},{anchor:"transformers.CodeLlamaTokenizerFast.fill_token",description:`<strong>fill_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;FILL_ME&gt;&quot;</code>) &#x2014;
The token used to split the input between the prefix and suffix.`,name:"fill_token"},{anchor:"transformers.CodeLlamaTokenizerFast.additional_special_tokens",description:`<strong>additional_special_tokens</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
Additional special tokens used by the tokenizer.`,name:"additional_special_tokens"},{anchor:"transformers.CodeLlamaTokenizerFast.add_bos_token",description:`<strong>add_bos_token</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to add a beginning of sequence token at the start of sequences.`,name:"add_bos_token"},{anchor:"transformers.CodeLlamaTokenizerFast.add_eos_token",description:`<strong>add_eos_token</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to add an end of sequence token at the end of sequences.`,name:"add_eos_token"},{anchor:"transformers.CodeLlamaTokenizerFast.use_default_system_prompt",description:`<strong>use_default_system_prompt</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the default system prompt for Llama should be used.`,name:"use_default_system_prompt"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/code_llama/tokenization_code_llama_fast.py#L49"}}),X=new Zn({props:{anchor:"transformers.CodeLlamaTokenizerFast.example",$$slots:{default:[Dn]},$$scope:{ctx:j}}}),he=new G({props:{name:"build_inputs_with_special_tokens",anchor:"transformers.CodeLlamaTokenizerFast.build_inputs_with_special_tokens",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizerFast.build_inputs_with_special_tokens.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs to which the special tokens will be added.`,name:"token_ids_0"},{anchor:"transformers.CodeLlamaTokenizerFast.build_inputs_with_special_tokens.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/code_llama/tokenization_code_llama_fast.py#L349",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>list of <a href="../glossary#input-ids">input IDs</a> with the appropriate special tokens.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),_e=new G({props:{name:"get_special_tokens_mask",anchor:"transformers.CodeLlamaTokenizerFast.get_special_tokens_mask",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"},{name:"already_has_special_tokens",val:": bool = False"}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizerFast.get_special_tokens_mask.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of ids of the first sequence.`,name:"token_ids_0"},{anchor:"transformers.CodeLlamaTokenizerFast.get_special_tokens_mask.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
List of ids of the second sequence.`,name:"token_ids_1"},{anchor:"transformers.CodeLlamaTokenizerFast.get_special_tokens_mask.already_has_special_tokens",description:`<strong>already_has_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the token list is already formatted with special tokens for the model.`,name:"already_has_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/tokenization_utils_base.py#L3865",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>1 for a special token, 0 for a sequence token.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A list of integers in the range [0, 1]</p>
`}}),ge=new G({props:{name:"create_token_type_ids_from_sequences",anchor:"transformers.CodeLlamaTokenizerFast.create_token_type_ids_from_sequences",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizerFast.create_token_type_ids_from_sequences.token_ids_0",description:"<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014; The first tokenized sequence.",name:"token_ids_0"},{anchor:"transformers.CodeLlamaTokenizerFast.create_token_type_ids_from_sequences.token_ids_1",description:"<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014; The second tokenized sequence.",name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/tokenization_utils_base.py#L3388",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The token type ids.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),be=new G({props:{name:"update_post_processor",anchor:"transformers.CodeLlamaTokenizerFast.update_post_processor",parameters:[],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/code_llama/tokenization_code_llama_fast.py#L176"}}),ke=new G({props:{name:"save_vocabulary",anchor:"transformers.CodeLlamaTokenizerFast.save_vocabulary",parameters:[{name:"save_directory",val:": str"},{name:"filename_prefix",val:": typing.Optional[str] = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/code_llama/tokenization_code_llama_fast.py#L330"}}),Me=new An({props:{source:"https://github.com/huggingface/transformers/blob/main/docs/source/en/model_doc/code_llama.md"}}),{c(){n=i("meta"),m=s(),o=i("p"),M=s(),T=i("div"),T.innerHTML=v,p=s(),u(w.$$.fragment),D=s(),q=i("p"),q.innerHTML=Qe,Oe=s(),P=i("p"),P.innerHTML=on,Ke=s(),u(E.$$.fragment),et=s(),O=i("p"),O.innerHTML=sn,tt=s(),u(R.$$.fragment),nt=s(),K=i("p"),K.innerHTML=an,ot=s(),ee=i("p"),ee.innerHTML=ln,st=s(),u(te.$$.fragment),at=s(),ne=i("p"),ne.innerHTML=rn,lt=s(),u(oe.$$.fragment),rt=s(),V=i("div"),V.innerHTML=dn,it=s(),u(se.$$.fragment),dt=s(),$=i("ul"),Te=i("li"),Te.innerHTML=cn,Tt=s(),ae=i("li"),ve=i("p"),ve.innerHTML=pn,vt=s(),u(le.$$.fragment),Jt=s(),Je=i("li"),Je.innerHTML=mn,wt=s(),we=i("li"),we.innerHTML=fn,xt=s(),xe=i("li"),xe.innerHTML=un,ct=s(),u(re.$$.fragment),pt=s(),x=i("div"),u(ie.$$.fragment),Ct=s(),Ce=i("p"),Ce.textContent=hn,jt=s(),je=i("p"),je.innerHTML=_n,Lt=s(),Le=i("div"),u(de.$$.fragment),$t=s(),B=i("div"),u(ce.$$.fragment),Ut=s(),$e=i("p"),$e.innerHTML=gn,zt=s(),Z=i("div"),u(pe.$$.fragment),It=s(),Ue=i("p"),Ue.textContent=bn,Zt=s(),u(A.$$.fragment),Gt=s(),ze=i("p"),ze.textContent=kn,qt=s(),N=i("div"),u(me.$$.fragment),Wt=s(),Ie=i("p"),Ie.textContent=Mn,mt=s(),u(fe.$$.fragment),ft=s(),y=i("div"),u(ue.$$.fragment),Ft=s(),Ze=i("p"),Ze.textContent=yn,Et=s(),Ge=i("p"),Ge.textContent=Tn,Rt=s(),u(X.$$.fragment),Vt=s(),qe=i("p"),qe.innerHTML=vn,Bt=s(),We=i("p"),We.innerHTML=Jn,At=s(),U=i("div"),u(he.$$.fragment),Nt=s(),Fe=i("p"),Fe.textContent=wn,Xt=s(),Ee=i("p"),Ee.innerHTML=xn,Ht=s(),Re=i("ul"),Re.innerHTML=Cn,Yt=s(),Ve=i("p"),Ve.textContent=jn,St=s(),H=i("div"),u(_e.$$.fragment),Qt=s(),Be=i("p"),Be.innerHTML=Ln,Dt=s(),W=i("div"),u(ge.$$.fragment),Pt=s(),Ae=i("p"),Ae.innerHTML=$n,Ot=s(),Ne=i("p"),Ne.textContent=Un,Kt=s(),Y=i("div"),u(be.$$.fragment),en=s(),Xe=i("p"),Xe.innerHTML=zn,tn=s(),He=i("div"),u(ke.$$.fragment),ut=s(),u(Me.$$.fragment),ht=s(),De=i("p"),this.h()},l(e){const r=En("svelte-u9bgzb",document.head);n=d(r,"META",{name:!0,content:!0}),r.forEach(l),m=a(e),o=d(e,"P",{}),L(o).forEach(l),M=a(e),T=d(e,"DIV",{style:!0,"data-svelte-h":!0}),f(T)!=="svelte-f1we31"&&(T.innerHTML=v),p=a(e),h(w.$$.fragment,e),D=a(e),q=d(e,"P",{"data-svelte-h":!0}),f(q)!=="svelte-1yo8bzu"&&(q.innerHTML=Qe),Oe=a(e),P=d(e,"P",{"data-svelte-h":!0}),f(P)!=="svelte-fa552u"&&(P.innerHTML=on),Ke=a(e),h(E.$$.fragment,e),et=a(e),O=d(e,"P",{"data-svelte-h":!0}),f(O)!=="svelte-1hdf1tf"&&(O.innerHTML=sn),tt=a(e),h(R.$$.fragment,e),nt=a(e),K=d(e,"P",{"data-svelte-h":!0}),f(K)!=="svelte-nf5ooi"&&(K.innerHTML=an),ot=a(e),ee=d(e,"P",{"data-svelte-h":!0}),f(ee)!=="svelte-60nsd0"&&(ee.innerHTML=ln),st=a(e),h(te.$$.fragment,e),at=a(e),ne=d(e,"P",{"data-svelte-h":!0}),f(ne)!=="svelte-w3z5ks"&&(ne.innerHTML=rn),lt=a(e),h(oe.$$.fragment,e),rt=a(e),V=d(e,"DIV",{class:!0,"data-svelte-h":!0}),f(V)!=="svelte-1dvtm5s"&&(V.innerHTML=dn),it=a(e),h(se.$$.fragment,e),dt=a(e),$=d(e,"UL",{});var I=L($);Te=d(I,"LI",{"data-svelte-h":!0}),f(Te)!=="svelte-n8bjng"&&(Te.innerHTML=cn),Tt=a(I),ae=d(I,"LI",{});var ye=L(ae);ve=d(ye,"P",{"data-svelte-h":!0}),f(ve)!=="svelte-xmombd"&&(ve.innerHTML=pn),vt=a(ye),h(le.$$.fragment,ye),ye.forEach(l),Jt=a(I),Je=d(I,"LI",{"data-svelte-h":!0}),f(Je)!=="svelte-2ql863"&&(Je.innerHTML=mn),wt=a(I),we=d(I,"LI",{"data-svelte-h":!0}),f(we)!=="svelte-1qqi6e7"&&(we.innerHTML=fn),xt=a(I),xe=d(I,"LI",{"data-svelte-h":!0}),f(xe)!=="svelte-4gdjt6"&&(xe.innerHTML=un),I.forEach(l),ct=a(e),h(re.$$.fragment,e),pt=a(e),x=d(e,"DIV",{class:!0});var C=L(x);h(ie.$$.fragment,C),Ct=a(C),Ce=d(C,"P",{"data-svelte-h":!0}),f(Ce)!=="svelte-1vmwvh5"&&(Ce.textContent=hn),jt=a(C),je=d(C,"P",{"data-svelte-h":!0}),f(je)!=="svelte-1a4a72p"&&(je.innerHTML=_n),Lt=a(C),Le=d(C,"DIV",{class:!0});var Pe=L(Le);h(de.$$.fragment,Pe),Pe.forEach(l),$t=a(C),B=d(C,"DIV",{class:!0});var gt=L(B);h(ce.$$.fragment,gt),Ut=a(gt),$e=d(gt,"P",{"data-svelte-h":!0}),f($e)!=="svelte-1f4f5kp"&&($e.innerHTML=gn),gt.forEach(l),zt=a(C),Z=d(C,"DIV",{class:!0});var S=L(Z);h(pe.$$.fragment,S),It=a(S),Ue=d(S,"P",{"data-svelte-h":!0}),f(Ue)!=="svelte-13bfd60"&&(Ue.textContent=bn),Zt=a(S),h(A.$$.fragment,S),Gt=a(S),ze=d(S,"P",{"data-svelte-h":!0}),f(ze)!=="svelte-wtrslu"&&(ze.textContent=kn),S.forEach(l),qt=a(C),N=d(C,"DIV",{class:!0});var bt=L(N);h(me.$$.fragment,bt),Wt=a(bt),Ie=d(bt,"P",{"data-svelte-h":!0}),f(Ie)!=="svelte-1slb66l"&&(Ie.textContent=Mn),bt.forEach(l),C.forEach(l),mt=a(e),h(fe.$$.fragment,e),ft=a(e),y=d(e,"DIV",{class:!0});var J=L(y);h(ue.$$.fragment,J),Ft=a(J),Ze=d(J,"P",{"data-svelte-h":!0}),f(Ze)!=="svelte-15tdcz8"&&(Ze.textContent=yn),Et=a(J),Ge=d(J,"P",{"data-svelte-h":!0}),f(Ge)!=="svelte-llhmpa"&&(Ge.textContent=Tn),Rt=a(J),h(X.$$.fragment,J),Vt=a(J),qe=d(J,"P",{"data-svelte-h":!0}),f(qe)!=="svelte-cnb6q1"&&(qe.innerHTML=vn),Bt=a(J),We=d(J,"P",{"data-svelte-h":!0}),f(We)!=="svelte-a387ws"&&(We.innerHTML=Jn),At=a(J),U=d(J,"DIV",{class:!0});var F=L(U);h(he.$$.fragment,F),Nt=a(F),Fe=d(F,"P",{"data-svelte-h":!0}),f(Fe)!=="svelte-1vll0v2"&&(Fe.textContent=wn),Xt=a(F),Ee=d(F,"P",{"data-svelte-h":!0}),f(Ee)!=="svelte-90np8u"&&(Ee.innerHTML=xn),Ht=a(F),Re=d(F,"UL",{"data-svelte-h":!0}),f(Re)!=="svelte-mlrsks"&&(Re.innerHTML=Cn),Yt=a(F),Ve=d(F,"P",{"data-svelte-h":!0}),f(Ve)!=="svelte-46aam0"&&(Ve.textContent=jn),F.forEach(l),St=a(J),H=d(J,"DIV",{class:!0});var kt=L(H);h(_e.$$.fragment,kt),Qt=a(kt),Be=d(kt,"P",{"data-svelte-h":!0}),f(Be)!=="svelte-1wmjg8a"&&(Be.innerHTML=Ln),kt.forEach(l),Dt=a(J),W=d(J,"DIV",{class:!0});var Ye=L(W);h(ge.$$.fragment,Ye),Pt=a(Ye),Ae=d(Ye,"P",{"data-svelte-h":!0}),f(Ae)!=="svelte-zj1vf1"&&(Ae.innerHTML=$n),Ot=a(Ye),Ne=d(Ye,"P",{"data-svelte-h":!0}),f(Ne)!=="svelte-9vptpw"&&(Ne.textContent=Un),Ye.forEach(l),Kt=a(J),Y=d(J,"DIV",{class:!0});var Mt=L(Y);h(be.$$.fragment,Mt),en=a(Mt),Xe=d(Mt,"P",{"data-svelte-h":!0}),f(Xe)!=="svelte-nfci2w"&&(Xe.innerHTML=zn),Mt.forEach(l),tn=a(J),He=d(J,"DIV",{class:!0});var In=L(He);h(ke.$$.fragment,In),In.forEach(l),J.forEach(l),ut=a(e),h(Me.$$.fragment,e),ht=a(e),De=d(e,"P",{}),L(De).forEach(l),this.h()},h(){z(n,"name","hf:doc:metadata"),z(n,"content",On),Rn(T,"float","right"),z(V,"class","flex justify-center"),z(Le,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),z(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),z(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),z(N,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),z(x,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),z(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),z(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),z(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),z(Y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),z(He,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),z(y,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,r){t(document.head,n),c(e,m,r),c(e,o,r),c(e,M,r),c(e,T,r),c(e,p,r),_(w,e,r),c(e,D,r),c(e,q,r),c(e,Oe,r),c(e,P,r),c(e,Ke,r),_(E,e,r),c(e,et,r),c(e,O,r),c(e,tt,r),_(R,e,r),c(e,nt,r),c(e,K,r),c(e,ot,r),c(e,ee,r),c(e,st,r),_(te,e,r),c(e,at,r),c(e,ne,r),c(e,lt,r),_(oe,e,r),c(e,rt,r),c(e,V,r),c(e,it,r),_(se,e,r),c(e,dt,r),c(e,$,r),t($,Te),t($,Tt),t($,ae),t(ae,ve),t(ae,vt),_(le,ae,null),t($,Jt),t($,Je),t($,wt),t($,we),t($,xt),t($,xe),c(e,ct,r),_(re,e,r),c(e,pt,r),c(e,x,r),_(ie,x,null),t(x,Ct),t(x,Ce),t(x,jt),t(x,je),t(x,Lt),t(x,Le),_(de,Le,null),t(x,$t),t(x,B),_(ce,B,null),t(B,Ut),t(B,$e),t(x,zt),t(x,Z),_(pe,Z,null),t(Z,It),t(Z,Ue),t(Z,Zt),_(A,Z,null),t(Z,Gt),t(Z,ze),t(x,qt),t(x,N),_(me,N,null),t(N,Wt),t(N,Ie),c(e,mt,r),_(fe,e,r),c(e,ft,r),c(e,y,r),_(ue,y,null),t(y,Ft),t(y,Ze),t(y,Et),t(y,Ge),t(y,Rt),_(X,y,null),t(y,Vt),t(y,qe),t(y,Bt),t(y,We),t(y,At),t(y,U),_(he,U,null),t(U,Nt),t(U,Fe),t(U,Xt),t(U,Ee),t(U,Ht),t(U,Re),t(U,Yt),t(U,Ve),t(y,St),t(y,H),_(_e,H,null),t(H,Qt),t(H,Be),t(y,Dt),t(y,W),_(ge,W,null),t(W,Pt),t(W,Ae),t(W,Ot),t(W,Ne),t(y,Kt),t(y,Y),_(be,Y,null),t(Y,en),t(Y,Xe),t(y,tn),t(y,He),_(ke,He,null),c(e,ut,r),_(Me,e,r),c(e,ht,r),c(e,De,r),_t=!0},p(e,[r]){const I={};r&2&&(I.$$scope={dirty:r,ctx:e}),E.$set(I);const ye={};r&2&&(ye.$$scope={dirty:r,ctx:e}),R.$set(ye);const C={};r&2&&(C.$$scope={dirty:r,ctx:e}),A.$set(C);const Pe={};r&2&&(Pe.$$scope={dirty:r,ctx:e}),X.$set(Pe)},i(e){_t||(g(w.$$.fragment,e),g(E.$$.fragment,e),g(R.$$.fragment,e),g(te.$$.fragment,e),g(oe.$$.fragment,e),g(se.$$.fragment,e),g(le.$$.fragment,e),g(re.$$.fragment,e),g(ie.$$.fragment,e),g(de.$$.fragment,e),g(ce.$$.fragment,e),g(pe.$$.fragment,e),g(A.$$.fragment,e),g(me.$$.fragment,e),g(fe.$$.fragment,e),g(ue.$$.fragment,e),g(X.$$.fragment,e),g(he.$$.fragment,e),g(_e.$$.fragment,e),g(ge.$$.fragment,e),g(be.$$.fragment,e),g(ke.$$.fragment,e),g(Me.$$.fragment,e),_t=!0)},o(e){b(w.$$.fragment,e),b(E.$$.fragment,e),b(R.$$.fragment,e),b(te.$$.fragment,e),b(oe.$$.fragment,e),b(se.$$.fragment,e),b(le.$$.fragment,e),b(re.$$.fragment,e),b(ie.$$.fragment,e),b(de.$$.fragment,e),b(ce.$$.fragment,e),b(pe.$$.fragment,e),b(A.$$.fragment,e),b(me.$$.fragment,e),b(fe.$$.fragment,e),b(ue.$$.fragment,e),b(X.$$.fragment,e),b(he.$$.fragment,e),b(_e.$$.fragment,e),b(ge.$$.fragment,e),b(be.$$.fragment,e),b(ke.$$.fragment,e),b(Me.$$.fragment,e),_t=!1},d(e){e&&(l(m),l(o),l(M),l(T),l(p),l(D),l(q),l(Oe),l(P),l(Ke),l(et),l(O),l(tt),l(nt),l(K),l(ot),l(ee),l(st),l(at),l(ne),l(lt),l(rt),l(V),l(it),l(dt),l($),l(ct),l(pt),l(x),l(mt),l(ft),l(y),l(ut),l(ht),l(De)),l(n),k(w,e),k(E,e),k(R,e),k(te,e),k(oe,e),k(se,e),k(le),k(re,e),k(ie),k(de),k(ce),k(pe),k(A),k(me),k(fe,e),k(ue),k(X),k(he),k(_e),k(ge),k(be),k(ke),k(Me,e)}}}const On='{"title":"CodeLlama","local":"codellama","sections":[{"title":"Notes","local":"notes","sections":[],"depth":2},{"title":"CodeLlamaTokenizer","local":"transformers.CodeLlamaTokenizer","sections":[],"depth":2},{"title":"CodeLlamaTokenizerFast","local":"transformers.CodeLlamaTokenizerFast","sections":[],"depth":2}],"depth":1}';function Kn(j){return qn(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class co extends Wn{constructor(n){super(),Fn(this,n,Kn,Pn,Gn,{})}}export{co as component};
