import{s as Co,o as wo,n as Nt}from"../chunks/scheduler.9bc65507.js";import{S as $o,i as zo,g as r,s,r as u,A as jo,h as l,f as t,c as a,j as z,u as h,x as d,k as j,y as n,a as i,v as _,d as g,t as k,w as b}from"../chunks/index.707bf1b6.js";import{T as qo}from"../chunks/Tip.c2ecdbf4.js";import{D as F}from"../chunks/Docstring.17db21ae.js";import{C as We}from"../chunks/CodeBlock.54a9f38d.js";import{E as Mo}from"../chunks/ExampleCodeBlock.4f515aa9.js";import{H as _t}from"../chunks/Heading.342b1fa6.js";function Fo(J){let c,C="The <code>Llama2</code> family models, on which Code Llama is based, were trained using <code>bfloat16</code>, but the original inference uses <code>float16</code>. Let’s look at the different precisions:",p,y,w="<li><code>float32</code>: PyTorch convention on model initialization is to load models in <code>float32</code>, no matter with which <code>dtype</code> the model weights were stored. <code>transformers</code> also follows this convention for consistency with PyTorch. This will be picked by default. If you want the <code>AutoModel</code> API to cast the load the checkpoints with the storage weights type, you must specify <code>torch_dtype=&quot;auto&quot;</code>, e.g. <code>model = AutoModelForCausalLM.from_pretrained(&quot;path&quot;, torch_dtype = &quot;auto&quot;)</code>.</li> <li><code>bfloat16</code>: Code Llama was trained with this precision, so we recommend using it for further training or fine-tuning.</li> <li><code>float16</code>: We recommend running inference using this precision, as it’s usually faster than <code>bfloat16</code>, and evaluation metrics show no discernible degradation with respect to <code>bfloat16</code>. You can also run inference using <code>bfloat16</code>, and we recommend you check inference results with both <code>float16</code> and <code>bfloat16</code> after fine-tuning.</li>",f,x,G="As mentioned above, the <code>dtype</code> of the storage weights is mostly irrelevant unless you are using <code>torch_dtype=&quot;auto&quot;</code> when initializing a model using. The reason is that the model will first be downloaded (using the <code>dtype</code> of the checkpoints online) and then will be casted to the default <code>dtype</code> of <code>torch</code> (becomes <code>torch.float32</code>). If there is a specified <code>torch_dtype</code>, it will be used instead.";return{c(){c=r("p"),c.innerHTML=C,p=s(),y=r("ul"),y.innerHTML=w,f=s(),x=r("p"),x.innerHTML=G},l(T){c=l(T,"P",{"data-svelte-h":!0}),d(c)!=="svelte-8yf686"&&(c.innerHTML=C),p=a(T),y=l(T,"UL",{"data-svelte-h":!0}),d(y)!=="svelte-17c64tm"&&(y.innerHTML=w),f=a(T),x=l(T,"P",{"data-svelte-h":!0}),d(x)!=="svelte-1bzil5i"&&(x.innerHTML=G)},m(T,E){i(T,c,E),i(T,p,E),i(T,y,E),i(T,f,E),i(T,x,E)},p:Nt,d(T){T&&(t(c),t(p),t(y),t(f),t(x))}}}function Uo(J){let c,C="sequence pair mask has the following format:",p,y,w;return y=new We({props:{code:"MCUyMDAlMjAwJTIwMCUyMDAlMjAwJTIwMCUyMDAlMjAwJTIwMCUyMDAlMjAxJTIwMSUyMDElMjAxJTIwMSUyMDElMjAxJTIwMSUyMDElMEElN0MlMjBmaXJzdCUyMHNlcXVlbmNlJTIwJTIwJTIwJTIwJTdDJTIwc2Vjb25kJTIwc2VxdWVuY2UlMjAlN0M=",highlighted:`0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 0 </span>0<span class="hljs-number"> 1 </span>1<span class="hljs-number"> 1 </span>1<span class="hljs-number"> 1 </span>1<span class="hljs-number"> 1 </span>1 1
| first sequence    | second sequence |`,wrap:!1}}),{c(){c=r("p"),c.textContent=C,p=s(),u(y.$$.fragment)},l(f){c=l(f,"P",{"data-svelte-h":!0}),d(c)!=="svelte-16klr56"&&(c.textContent=C),p=a(f),h(y.$$.fragment,f)},m(f,x){i(f,c,x),i(f,p,x),_(y,f,x),w=!0},p:Nt,i(f){w||(g(y.$$.fragment,f),w=!0)},o(f){k(y.$$.fragment,f),w=!1},d(f){f&&(t(c),t(p)),b(y,f)}}}function Io(J){let c,C;return c=new We({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMENvZGVMbGFtYVRva2VuaXplckZhc3QlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBDb2RlTGxhbWFUb2tlbml6ZXJGYXN0LmZyb21fcHJldHJhaW5lZCglMjJoZi1pbnRlcm5hbC10ZXN0aW5nJTJGbGxhbWEtdG9rZW5pemVyJTIyKSUwQXRva2VuaXplci5lbmNvZGUoJTIySGVsbG8lMjB0aGlzJTIwaXMlMjBhJTIwdGVzdCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> CodeLlamaTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = CodeLlamaTokenizerFast.from_pretrained(<span class="hljs-string">&quot;hf-internal-testing/llama-tokenizer&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.encode(<span class="hljs-string">&quot;Hello this is a test&quot;</span>)
[<span class="hljs-number">1</span>, <span class="hljs-number">15043</span>, <span class="hljs-number">445</span>, <span class="hljs-number">338</span>, <span class="hljs-number">263</span>, <span class="hljs-number">1243</span>]`,wrap:!1}}),{c(){u(c.$$.fragment)},l(p){h(c.$$.fragment,p)},m(p,y){_(c,p,y),C=!0},p:Nt,i(p){C||(g(c.$$.fragment,p),C=!0)},o(p){k(c.$$.fragment,p),C=!1},d(p){b(c,p)}}}function Jo(J){let c,C,p,y,w,f,x,G,T,E='The Code Llama model was proposed in <a href="https://ai.meta.com/research/publications/code-llama-open-foundation-models-for-code/" rel="nofollow">Code Llama: Open Foundation Models for Code</a> by Baptiste Rozière, Jonas Gehring, Fabian Gloeckle, Sten Sootla, Itai Gat, Xiaoqing Ellen Tan, Yossi Adi, Jingyu Liu, Tal Remez, Jérémy Rapin, Artyom Kozhevnikov, Ivan Evtimov, Joanna Bitton, Manish Bhatt, Cristian Canton Ferrer, Aaron Grattafiori, Wenhan Xiong, Alexandre Défossez, Jade Copet, Faisal Azhar, Hugo Touvron, Louis Martin, Nicolas Usunier, Thomas Scialom, Gabriel Synnaeve.',Ge,N,Vt="The abstract from the paper is the following:",Ne,V,St="<em>We release Code Llama, a family of large language models for code based on Llama 2 providing state-of-the-art performance among open models, infilling capabilities, support for large input contexts, and zero-shot instruction following ability for programming tasks. We provide multiple flavors to cover a wide range of applications: foundation models (Code Llama), Python specializations (Code Llama - Python), and instruction-following models (Code Llama - Instruct) with 7B, 13B and 34B parameters each. All models are trained on sequences of 16k tokens and show improvements on inputs with up to 100k tokens. 7B and 13B Code Llama and Code Llama - Instruct variants support infilling based on surrounding content. Code Llama reaches state-of-the-art performance among open models on several code benchmarks, with scores of up to 53% and 55% on HumanEval and MBPP, respectively. Notably, Code Llama - Python 7B outperforms Llama 2 70B on HumanEval and MBPP, and all our models outperform every other publicly available model on MultiPL-E. We release Code Llama under a permissive license that allows for both research and commercial use.</em>",Ve,S,Xt='Check out all Code Llama models <a href="https://huggingface.co/models?search=code_llama" rel="nofollow">here</a> and the officially released ones in the <a href="https://huggingface.co/codellama" rel="nofollow">codellama org</a>.',Se,P,Xe,X,Yt="Tips:",Ye,Y,Qt="<li>These models have the same architecture as the <code>Llama2</code> models</li> <li>The infilling task is supported out of the box. You should be using the <code>tokenizer.fill_token</code> where you want your input to be filled.</li> <li>The model conversion script is the same as for the <code>Llama2</code> family:</li>",Qe,Q,Ot="Here is a sample usage",Oe,O,Ke,K,Kt=`Note that executing the script requires enough CPU RAM to host the whole model in float16 precision (even if the biggest versions
come in several checkpoints they each contain a part of each weight of the model, so we need to load them all in RAM).`,et,ee,eo="<li>After conversion, the model and tokenizer can be loaded via:</li>",tt,te,ot,oe,to="If you only want the infilled part:",nt,ne,st,se,oo='Under the hood, the tokenizer <a href="https://huggingface.co/docs/transformers/main/model_doc/code_llama#transformers.CodeLlamaTokenizer.fill_token" rel="nofollow">automatically splits by <code>&lt;FILL_ME&gt;</code></a> to create a formatted input string that follows <a href="https://github.com/facebookresearch/codellama/blob/cb51c14ec761370ba2e2bc351374a79265d0465e/llama/generation.py#L402" rel="nofollow">the original training pattern</a>. This is more robust than preparing the pattern yourself: it avoids pitfalls, such as token glueing, that are very hard to debug.  To see how much CPU and GPU memory you need for this model or others, try <a href="https://huggingface.co/spaces/hf-accelerate/model-memory-usage" rel="nofollow">this calculator</a> which can help determine that value.',at,ae,no='<li>The LLaMA tokenizer is a BPE model based on <a href="https://github.com/google/sentencepiece" rel="nofollow">sentencepiece</a>. One quirk of sentencepiece is that when decoding a sequence, if the first token is the start of the word (e.g. “Banana”), the tokenizer does not prepend the prefix space to the string.</li>',rt,re,so='This model was contributed by <a href="https://huggingface.co/ArthurZ" rel="nofollow">ArthurZucker</a>. The original code of the authors can be found <a href="https://github.com/facebookresearch/llama" rel="nofollow">here</a>.',lt,le,it,L,ie,gt,ve,ao=`Construct a CodeLlama tokenizer. Based on byte-level Byte-Pair-Encoding. The default padding token is unset as
there is no padding token in the original model.`,kt,Te,ro=`The default configuration match that of
<a href="https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf/blob/main/tokenizer_config.json" rel="nofollow">codellama/CodeLlama-7b-Instruct-hf</a>
which supports prompt infilling.`,bt,xe,de,yt,H,ce,vt,Le,lo=`Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer <code>prepare_for_model</code> method.`,Tt,q,me,xt,Me,io="Creates a mask from the two sequences passed to be used in a sequence-pair classification task. An ALBERT",Lt,R,Mt,Ce,co="if token_ids_1 is None, only returns the first portion of the mask (0s).",Ct,Z,pe,wt,we,mo="Save the vocabulary and special tokens file to a directory.",dt,fe,ct,m,ue,$t,$e,po="Construct a Llama tokenizer. Based on byte-level Byte-Pair-Encoding.",zt,ze,fo="This uses notably ByteFallback and no normalization.",jt,A,qt,je,uo=`If you want to change the <code>bos_token</code> or the <code>eos_token</code>, make sure to specify them when initializing the model, or
call <code>tokenizer.update_post_processor()</code> to make sure that the post-processing is correctly done (otherwise the
values of the first token and final token of an encoded sequence will not be correct). For more details, checkout
[post-processors] (<a href="https://huggingface.co/docs/tokenizers/api/post-processors" rel="nofollow">https://huggingface.co/docs/tokenizers/api/post-processors</a>) documentation.`,Ft,qe,ho=`This tokenizer inherits from <a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizerFast">PreTrainedTokenizerFast</a> which contains most of the main methods. Users should
refer to this superclass for more information regarding those methods. The default configuration match that of
<a href="https://huggingface.co/codellama/CodeLlama-7b-Instruct-hf/blob/main/tokenizer_config.json" rel="nofollow">codellama/CodeLlama-7b-Instruct-hf</a>
which supports prompt infilling.`,Ut,$,he,It,Fe,_o=`Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
adding special tokens. The special tokens depend on calling set_lang.`,Jt,Ue,go="An NLLB sequence has the following format, where <code>X</code> represents the sequence:",Et,Ie,ko="<li><code>input_ids</code> (for encoder) <code>X [eos, src_lang_code]</code></li> <li><code>decoder_input_ids</code>: (for decoder) <code>X [eos, tgt_lang_code]</code></li>",Pt,Je,bo=`BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a
separator.`,Ht,B,_e,Rt,Ee,yo=`Retrieves sequence ids from a token list that has no special tokens added. This method is called when adding
special tokens using the tokenizer <code>prepare_for_model</code> or <code>encode_plus</code> methods.`,Zt,U,ge,At,Pe,vo=`Create the token type IDs corresponding to the sequences passed. <a href="../glossary#token-type-ids">What are token type
IDs?</a>`,Bt,He,To="Should be overridden in a subclass if the model has a special way of building those.",Dt,D,ke,Wt,Re,xo="Updates the underlying post processor with the current <code>bos_token</code> and <code>eos_token</code>.",Gt,Ze,be,mt,Be,pt;return w=new _t({props:{title:"CodeLlama",local:"codellama",headingTag:"h1"}}),x=new _t({props:{title:"Overview",local:"overview",headingTag:"h2"}}),P=new qo({props:{warning:!0,$$slots:{default:[Fo]},$$scope:{ctx:J}}}),O=new We({props:{code:"cHl0aG9uJTIwc3JjJTJGdHJhbnNmb3JtZXJzJTJGbW9kZWxzJTJGbGxhbWElMkZjb252ZXJ0X2xsYW1hX3dlaWdodHNfdG9faGYucHklMjAlNUMlMEElMjAlMjAlMjAlMjAtLWlucHV0X2RpciUyMCUyRnBhdGglMkZ0byUyRmRvd25sb2FkZWQlMkZsbGFtYSUyRndlaWdodHMlMjAtLW1vZGVsX3NpemUlMjA3QiUyMC0tb3V0cHV0X2RpciUyMCUyRm91dHB1dCUyRnBhdGg=",highlighted:`python src/transformers/models/llama/convert_llama_weights_to_hf.py \\
    --input_dir /path/to/downloaded/llama/weights --model_size 7B --output_dir /output/path`,wrap:!1}}),te=new We({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMExsYW1hRm9yQ2F1c2FsTE0lMkMlMjBDb2RlTGxhbWFUb2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBDb2RlTGxhbWFUb2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMmNvZGVsbGFtYSUyRkNvZGVMbGFtYS03Yi1oZiUyMiklMEFtb2RlbCUyMCUzRCUyMExsYW1hRm9yQ2F1c2FsTE0uZnJvbV9wcmV0cmFpbmVkKCUyMmNvZGVsbGFtYSUyRkNvZGVMbGFtYS03Yi1oZiUyMiklMEFQUk9NUFQlMjAlM0QlMjAnJydkZWYlMjByZW1vdmVfbm9uX2FzY2lpKHMlM0ElMjBzdHIpJTIwLSUzRSUyMHN0ciUzQSUwQWlucHV0X2lkcyUyMCUzRCUyMHRva2VuaXplcihQUk9NUFQlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSU1QiUyMmlucHV0X2lkcyUyMiU1RCUwQWdlbmVyYXRlZF9pZHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZShpbnB1dF9pZHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDEyOCklMEElMEFmaWxsaW5nJTIwJTNEJTIwdG9rZW5pemVyLmJhdGNoX2RlY29kZShnZW5lcmF0ZWRfaWRzJTVCJTNBJTJDJTIwaW5wdXRfaWRzLnNoYXBlJTVCMSU1RCUzQSU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlMjAlM0QlMjBUcnVlKSU1QjAlNUQlMEFwcmludChQUk9NUFQucmVwbGFjZSglMjIlM0NGSUxMX01FJTNFJTIyJTJDJTIwZmlsbGluZykpJTBBJTBB",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> LlamaForCausalLM, CodeLlamaTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = CodeLlamaTokenizer.from_pretrained(<span class="hljs-string">&quot;codellama/CodeLlama-7b-hf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = LlamaForCausalLM.from_pretrained(<span class="hljs-string">&quot;codellama/CodeLlama-7b-hf&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>PROMPT = <span class="hljs-string">&#x27;&#x27;&#x27;def remove_non_ascii(s: str) -&gt; str:
    &quot;&quot;&quot; &lt;FILL_ME&gt;
    return result
&#x27;&#x27;&#x27;</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = tokenizer(PROMPT, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>generated_ids = model.generate(input_ids, max_new_tokens=<span class="hljs-number">128</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>filling = tokenizer.batch_decode(generated_ids[:, input_ids.shape[<span class="hljs-number">1</span>]:], skip_special_tokens = <span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(PROMPT.replace(<span class="hljs-string">&quot;&lt;FILL_ME&gt;&quot;</span>, filling))
<span class="hljs-keyword">def</span> <span class="hljs-title function_">remove_non_ascii</span>(<span class="hljs-params">s: <span class="hljs-built_in">str</span></span>) -&gt; <span class="hljs-built_in">str</span>:
    <span class="hljs-string">&quot;&quot;&quot; Remove non-ASCII characters from a string.

    Args:
        s: The string to remove non-ASCII characters from.

    Returns:
        The string with non-ASCII characters removed.
    &quot;&quot;&quot;</span>
    result = <span class="hljs-string">&quot;&quot;</span>
    <span class="hljs-keyword">for</span> c <span class="hljs-keyword">in</span> s:
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">ord</span>(c) &lt; <span class="hljs-number">128</span>:
            result += c
    <span class="hljs-keyword">return</span> result`,wrap:!1}}),ne=new We({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBaW1wb3J0JTIwdG9yY2glMEElMEFnZW5lcmF0b3IlMjAlM0QlMjBwaXBlbGluZSglMjJ0ZXh0LWdlbmVyYXRpb24lMjIlMkNtb2RlbCUzRCUyMmNvZGVsbGFtYSUyRkNvZGVMbGFtYS03Yi1oZiUyMiUyQ3RvcmNoX2R0eXBlJTNEdG9yY2guZmxvYXQxNiUyQyUyMGRldmljZV9tYXAlM0QlMjJhdXRvJTIyKSUwQWdlbmVyYXRvcignZGVmJTIwcmVtb3ZlX25vbl9hc2NpaShzJTNBJTIwc3RyKSUyMC0lM0UlMjBzdHIlM0ElNUNuJTIwJTIwJTIwJTIwJTIyJTIyJTIyJTIwJTNDRklMTF9NRSUzRSU1Q24lMjAlMjAlMjAlMjByZXR1cm4lMjByZXN1bHQnJTJDJTIwbWF4X25ld190b2tlbnMlMjAlM0QlMjAxMjglMkMlMjByZXR1cm5fdHlwZSUyMCUzRCUyMDEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> torch

<span class="hljs-meta">&gt;&gt;&gt; </span>generator = pipeline(<span class="hljs-string">&quot;text-generation&quot;</span>,model=<span class="hljs-string">&quot;codellama/CodeLlama-7b-hf&quot;</span>,torch_dtype=torch.float16, device_map=<span class="hljs-string">&quot;auto&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>generator(<span class="hljs-string">&#x27;def remove_non_ascii(s: str) -&gt; str:\\n    &quot;&quot;&quot; &lt;FILL_ME&gt;\\n    return result&#x27;</span>, max_new_tokens = <span class="hljs-number">128</span>, return_type = <span class="hljs-number">1</span>)`,wrap:!1}}),le=new _t({props:{title:"CodeLlamaTokenizer",local:"transformers.CodeLlamaTokenizer",headingTag:"h2"}}),ie=new F({props:{name:"class transformers.CodeLlamaTokenizer",anchor:"transformers.CodeLlamaTokenizer",parameters:[{name:"vocab_file",val:""},{name:"unk_token",val:" = '<unk>'"},{name:"bos_token",val:" = '<s>'"},{name:"eos_token",val:" = '</s>'"},{name:"prefix_token",val:" = '▁<PRE>'"},{name:"middle_token",val:" = '▁<MID>'"},{name:"suffix_token",val:" = '▁<SUF>'"},{name:"eot_token",val:" = '▁<EOT>'"},{name:"fill_token",val:" = '<FILL_ME>'"},{name:"suffix_first",val:" = False"},{name:"sp_model_kwargs",val:": typing.Union[typing.Dict[str, typing.Any], NoneType] = None"},{name:"add_bos_token",val:" = True"},{name:"add_eos_token",val:" = False"},{name:"clean_up_tokenization_spaces",val:" = False"},{name:"additional_special_tokens",val:" = None"},{name:"use_default_system_prompt",val:" = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizer.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
Path to the vocabulary file.`,name:"vocab_file"},{anchor:"transformers.CodeLlamaTokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;unk&gt;&quot;</code>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead.`,name:"unk_token"},{anchor:"transformers.CodeLlamaTokenizer.bos_token",description:`<strong>bos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;s&gt;&quot;</code>) &#x2014;
The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.`,name:"bos_token"},{anchor:"transformers.CodeLlamaTokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;/s&gt;&quot;</code>) &#x2014;
The end of sequence token.</p>
<div class="course-tip  bg-gradient-to-br dark:bg-gradient-to-r before:border-green-500 dark:before:border-green-800 from-green-50 dark:from-gray-900 to-white dark:to-gray-950 border border-green-50 text-green-700 dark:text-gray-400">
						
<p>When building a sequence using special tokens, this is not the token that is used for the end of sequence.
The token used is the <code>sep_token</code>.</p>

					</div>`,name:"eos_token"},{anchor:"transformers.CodeLlamaTokenizer.prefix_token",description:`<strong>prefix_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;PRE&gt;&quot;</code>) &#x2014;
Prefix token used for infilling.`,name:"prefix_token"},{anchor:"transformers.CodeLlamaTokenizer.middle_token",description:`<strong>middle_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;MID&gt;&quot;</code>) &#x2014;
Middle token used for infilling.`,name:"middle_token"},{anchor:"transformers.CodeLlamaTokenizer.suffix_token",description:`<strong>suffix_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;SUF&gt;&quot;</code>) &#x2014;
Suffix token used for infilling.`,name:"suffix_token"},{anchor:"transformers.CodeLlamaTokenizer.eot_token",description:`<strong>eot_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;EOT&gt;&quot;</code>) &#x2014;
End of text token used for infilling.`,name:"eot_token"},{anchor:"transformers.CodeLlamaTokenizer.fill_token",description:`<strong>fill_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;FILL_ME&gt;&quot;</code>) &#x2014;
The token used to split the input between the prefix and suffix.`,name:"fill_token"},{anchor:"transformers.CodeLlamaTokenizer.suffix_first",description:`<strong>suffix_first</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether the input prompt and suffix should be formatted with the suffix first.`,name:"suffix_first"},{anchor:"transformers.CodeLlamaTokenizer.sp_model_kwargs",description:`<strong>sp_model_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Will be passed to the <code>SentencePieceProcessor.__init__()</code> method. The <a href="https://github.com/google/sentencepiece/tree/master/python" rel="nofollow">Python wrapper for
SentencePiece</a> can be used, among other things,
to set:</p>
<ul>
<li>
<p><code>enable_sampling</code>: Enable subword regularization.</p>
</li>
<li>
<p><code>nbest_size</code>: Sampling parameters for unigram. Invalid for BPE-Dropout.</p>
<ul>
<li><code>nbest_size = {0,1}</code>: No sampling is performed.</li>
<li><code>nbest_size &gt; 1</code>: samples from the nbest_size results.</li>
<li><code>nbest_size &lt; 0</code>: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
using forward-filtering-and-backward-sampling algorithm.</li>
</ul>
</li>
<li>
<p><code>alpha</code>: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
BPE-dropout.</p>
</li>
</ul>`,name:"sp_model_kwargs"},{anchor:"transformers.CodeLlamaTokenizer.add_bos_token",description:`<strong>add_bos_token</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether to add a beginning of sequence token at the start of sequences.`,name:"add_bos_token"},{anchor:"transformers.CodeLlamaTokenizer.add_eos_token",description:`<strong>add_eos_token</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether to add an end of sequence token at the end of sequences.`,name:"add_eos_token"},{anchor:"transformers.CodeLlamaTokenizer.clean_up_tokenization_spaces",description:`<strong>clean_up_tokenization_spaces</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to clean up the tokenization spaces.`,name:"clean_up_tokenization_spaces"},{anchor:"transformers.CodeLlamaTokenizer.additional_special_tokens",description:`<strong>additional_special_tokens</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
Additional special tokens used by the tokenizer.`,name:"additional_special_tokens"},{anchor:"transformers.CodeLlamaTokenizer.use_default_system_prompt",description:`<strong>use_default_system_prompt</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the default system prompt for Llama should be used.`,name:"use_default_system_prompt"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/code_llama/tokenization_code_llama.py#L59"}}),de=new F({props:{name:"build_inputs_with_special_tokens",anchor:"transformers.CodeLlamaTokenizer.build_inputs_with_special_tokens",parameters:[{name:"token_ids_0",val:""},{name:"token_ids_1",val:" = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/code_llama/tokenization_code_llama.py#L369"}}),ce=new F({props:{name:"get_special_tokens_mask",anchor:"transformers.CodeLlamaTokenizer.get_special_tokens_mask",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"},{name:"already_has_special_tokens",val:": bool = False"}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizer.get_special_tokens_mask.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs.`,name:"token_ids_0"},{anchor:"transformers.CodeLlamaTokenizer.get_special_tokens_mask.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"},{anchor:"transformers.CodeLlamaTokenizer.get_special_tokens_mask.already_has_special_tokens",description:`<strong>already_has_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the token list is already formatted with special tokens for the model.`,name:"already_has_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/code_llama/tokenization_code_llama.py#L381",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),me=new F({props:{name:"create_token_type_ids_from_sequences",anchor:"transformers.CodeLlamaTokenizer.create_token_type_ids_from_sequences",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizer.create_token_type_ids_from_sequences.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of ids.`,name:"token_ids_0"},{anchor:"transformers.CodeLlamaTokenizer.create_token_type_ids_from_sequences.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/code_llama/tokenization_code_llama.py#L419",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>List of <a href="../glossary#token-type-ids">token type IDs</a> according to the given sequence(s).</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),R=new Mo({props:{anchor:"transformers.CodeLlamaTokenizer.create_token_type_ids_from_sequences.example",$$slots:{default:[Uo]},$$scope:{ctx:J}}}),pe=new F({props:{name:"save_vocabulary",anchor:"transformers.CodeLlamaTokenizer.save_vocabulary",parameters:[{name:"save_directory",val:""},{name:"filename_prefix",val:": typing.Optional[str] = None"}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizer.save_vocabulary.save_directory",description:`<strong>save_directory</strong> (<code>str</code>) &#x2014;
The directory in which to save the vocabulary.`,name:"save_directory"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/code_llama/tokenization_code_llama.py#L341",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>Paths to the files saved.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>Tuple(str)</code></p>
`}}),fe=new _t({props:{title:"CodeLlamaTokenizerFast",local:"transformers.CodeLlamaTokenizerFast",headingTag:"h2"}}),ue=new F({props:{name:"class transformers.CodeLlamaTokenizerFast",anchor:"transformers.CodeLlamaTokenizerFast",parameters:[{name:"vocab_file",val:" = None"},{name:"tokenizer_file",val:" = None"},{name:"clean_up_tokenization_spaces",val:" = False"},{name:"unk_token",val:" = '<unk>'"},{name:"bos_token",val:" = '<s>'"},{name:"eos_token",val:" = '</s>'"},{name:"prefix_token",val:" = '▁<PRE>'"},{name:"middle_token",val:" = '▁<MID>'"},{name:"suffix_token",val:" = '▁<SUF>'"},{name:"eot_token",val:" = '▁<EOT>'"},{name:"fill_token",val:" = '<FILL_ME>'"},{name:"additional_special_tokens",val:" = None"},{name:"add_bos_token",val:" = True"},{name:"add_eos_token",val:" = False"},{name:"use_default_system_prompt",val:" = False"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizerFast.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
<a href="https://github.com/google/sentencepiece" rel="nofollow">SentencePiece</a> file (generally has a .model extension) that
contains the vocabulary necessary to instantiate a tokenizer.`,name:"vocab_file"},{anchor:"transformers.CodeLlamaTokenizerFast.tokenizer_file",description:`<strong>tokenizer_file</strong> (<code>str</code>) &#x2014;
<a href="https://github.com/huggingface/tokenizers" rel="nofollow">tokenizers</a> file (generally has a .json extension) that
contains everything needed to load the tokenizer.`,name:"tokenizer_file"},{anchor:"transformers.CodeLlamaTokenizerFast.clean_up_tokenization_spaces",description:`<strong>clean_up_tokenization_spaces</strong> (<code>str</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Wether to cleanup spaces after decoding, cleanup consists in removing potential artifacts like extra
spaces.`,name:"clean_up_tokenization_spaces"},{anchor:"transformers.CodeLlamaTokenizerFast.bos_token",description:`<strong>bos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;s&gt;&quot;</code>) &#x2014;
The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.`,name:"bos_token"},{anchor:"transformers.CodeLlamaTokenizerFast.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;/s&gt;&quot;</code>) &#x2014;
The end of sequence token.`,name:"eos_token"},{anchor:"transformers.CodeLlamaTokenizerFast.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;unk&gt;&quot;</code>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead.`,name:"unk_token"},{anchor:"transformers.CodeLlamaTokenizerFast.prefix_token",description:`<strong>prefix_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;PRE&gt;&quot;</code>) &#x2014;
Prefix token used for infilling.`,name:"prefix_token"},{anchor:"transformers.CodeLlamaTokenizerFast.suffix_token",description:`<strong>suffix_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;SUF&gt;&quot;</code>) &#x2014;
Suffix token used for infilling.`,name:"suffix_token"},{anchor:"transformers.CodeLlamaTokenizerFast.middle_token",description:`<strong>middle_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;MID&gt;&quot;</code>) &#x2014;
Middle token used for infilling.`,name:"middle_token"},{anchor:"transformers.CodeLlamaTokenizerFast.eot_token",description:`<strong>eot_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&#x2581;&lt;EOT&gt;&quot;</code>) &#x2014;
End of text token used for infilling.`,name:"eot_token"},{anchor:"transformers.CodeLlamaTokenizerFast.fill_token",description:`<strong>fill_token</strong> (<code>str</code>, <em>optional</em>, defaults to <code>&quot;&lt;FILL_ME&gt;&quot;</code>) &#x2014;
The token used to split the input between the prefix and suffix.`,name:"fill_token"},{anchor:"transformers.CodeLlamaTokenizerFast.suffix_first",description:`<strong>suffix_first</strong> (<code>bool</code>, <em>optional</em>, default to <code>False</code>) &#x2014;
Whether the input prompt and suffix should be formatted with the suffix first.`,name:"suffix_first"},{anchor:"transformers.CodeLlamaTokenizerFast.additional_special_tokens",description:`<strong>additional_special_tokens</strong> (<code>List[str]</code>, <em>optional</em>) &#x2014;
Additional special tokens used by the tokenizer.`,name:"additional_special_tokens"},{anchor:"transformers.CodeLlamaTokenizerFast.use_default_system_prompt",description:`<strong>use_default_system_prompt</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>True</code>) &#x2014;
Whether or not the default system prompt for Llama should be used.`,name:"use_default_system_prompt"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/code_llama/tokenization_code_llama_fast.py#L52"}}),A=new Mo({props:{anchor:"transformers.CodeLlamaTokenizerFast.example",$$slots:{default:[Io]},$$scope:{ctx:J}}}),he=new F({props:{name:"build_inputs_with_special_tokens",anchor:"transformers.CodeLlamaTokenizerFast.build_inputs_with_special_tokens",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizerFast.build_inputs_with_special_tokens.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of IDs to which the special tokens will be added.`,name:"token_ids_0"},{anchor:"transformers.CodeLlamaTokenizerFast.build_inputs_with_special_tokens.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
Optional second list of IDs for sequence pairs.`,name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/code_llama/tokenization_code_llama_fast.py#L400",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>list of <a href="../glossary#input-ids">input IDs</a> with the appropriate special tokens.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),_e=new F({props:{name:"get_special_tokens_mask",anchor:"transformers.CodeLlamaTokenizerFast.get_special_tokens_mask",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"},{name:"already_has_special_tokens",val:": bool = False"}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizerFast.get_special_tokens_mask.token_ids_0",description:`<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014;
List of ids of the first sequence.`,name:"token_ids_0"},{anchor:"transformers.CodeLlamaTokenizerFast.get_special_tokens_mask.token_ids_1",description:`<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014;
List of ids of the second sequence.`,name:"token_ids_1"},{anchor:"transformers.CodeLlamaTokenizerFast.get_special_tokens_mask.already_has_special_tokens",description:`<strong>already_has_special_tokens</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not the token list is already formatted with special tokens for the model.`,name:"already_has_special_tokens"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/tokenization_utils_base.py#L3780",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>1 for a special token, 0 for a sequence token.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p>A list of integers in the range [0, 1]</p>
`}}),ge=new F({props:{name:"create_token_type_ids_from_sequences",anchor:"transformers.CodeLlamaTokenizerFast.create_token_type_ids_from_sequences",parameters:[{name:"token_ids_0",val:": typing.List[int]"},{name:"token_ids_1",val:": typing.Optional[typing.List[int]] = None"}],parametersDescription:[{anchor:"transformers.CodeLlamaTokenizerFast.create_token_type_ids_from_sequences.token_ids_0",description:"<strong>token_ids_0</strong> (<code>List[int]</code>) &#x2014; The first tokenized sequence.",name:"token_ids_0"},{anchor:"transformers.CodeLlamaTokenizerFast.create_token_type_ids_from_sequences.token_ids_1",description:"<strong>token_ids_1</strong> (<code>List[int]</code>, <em>optional</em>) &#x2014; The second tokenized sequence.",name:"token_ids_1"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/tokenization_utils_base.py#L3315",returnDescription:`<script context="module">export const metadata = 'undefined';<\/script>


<p>The token type ids.</p>
`,returnType:`<script context="module">export const metadata = 'undefined';<\/script>


<p><code>List[int]</code></p>
`}}),ke=new F({props:{name:"update_post_processor",anchor:"transformers.CodeLlamaTokenizerFast.update_post_processor",parameters:[],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/code_llama/tokenization_code_llama_fast.py#L175"}}),be=new F({props:{name:"save_vocabulary",anchor:"transformers.CodeLlamaTokenizerFast.save_vocabulary",parameters:[{name:"save_directory",val:": str"},{name:"filename_prefix",val:": typing.Optional[str] = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/code_llama/tokenization_code_llama_fast.py#L329"}}),{c(){c=r("meta"),C=s(),p=r("p"),y=s(),u(w.$$.fragment),f=s(),u(x.$$.fragment),G=s(),T=r("p"),T.innerHTML=E,Ge=s(),N=r("p"),N.textContent=Vt,Ne=s(),V=r("p"),V.innerHTML=St,Ve=s(),S=r("p"),S.innerHTML=Xt,Se=s(),u(P.$$.fragment),Xe=s(),X=r("p"),X.textContent=Yt,Ye=s(),Y=r("ul"),Y.innerHTML=Qt,Qe=s(),Q=r("p"),Q.textContent=Ot,Oe=s(),u(O.$$.fragment),Ke=s(),K=r("p"),K.textContent=Kt,et=s(),ee=r("ul"),ee.innerHTML=eo,tt=s(),u(te.$$.fragment),ot=s(),oe=r("p"),oe.textContent=to,nt=s(),u(ne.$$.fragment),st=s(),se=r("p"),se.innerHTML=oo,at=s(),ae=r("ul"),ae.innerHTML=no,rt=s(),re=r("p"),re.innerHTML=so,lt=s(),u(le.$$.fragment),it=s(),L=r("div"),u(ie.$$.fragment),gt=s(),ve=r("p"),ve.textContent=ao,kt=s(),Te=r("p"),Te.innerHTML=ro,bt=s(),xe=r("div"),u(de.$$.fragment),yt=s(),H=r("div"),u(ce.$$.fragment),vt=s(),Le=r("p"),Le.innerHTML=lo,Tt=s(),q=r("div"),u(me.$$.fragment),xt=s(),Me=r("p"),Me.textContent=io,Lt=s(),u(R.$$.fragment),Mt=s(),Ce=r("p"),Ce.textContent=co,Ct=s(),Z=r("div"),u(pe.$$.fragment),wt=s(),we=r("p"),we.textContent=mo,dt=s(),u(fe.$$.fragment),ct=s(),m=r("div"),u(ue.$$.fragment),$t=s(),$e=r("p"),$e.textContent=po,zt=s(),ze=r("p"),ze.textContent=fo,jt=s(),u(A.$$.fragment),qt=s(),je=r("p"),je.innerHTML=uo,Ft=s(),qe=r("p"),qe.innerHTML=ho,Ut=s(),$=r("div"),u(he.$$.fragment),It=s(),Fe=r("p"),Fe.textContent=_o,Jt=s(),Ue=r("p"),Ue.innerHTML=go,Et=s(),Ie=r("ul"),Ie.innerHTML=ko,Pt=s(),Je=r("p"),Je.textContent=bo,Ht=s(),B=r("div"),u(_e.$$.fragment),Rt=s(),Ee=r("p"),Ee.innerHTML=yo,Zt=s(),U=r("div"),u(ge.$$.fragment),At=s(),Pe=r("p"),Pe.innerHTML=vo,Bt=s(),He=r("p"),He.textContent=To,Dt=s(),D=r("div"),u(ke.$$.fragment),Wt=s(),Re=r("p"),Re.innerHTML=xo,Gt=s(),Ze=r("div"),u(be.$$.fragment),mt=s(),Be=r("p"),this.h()},l(e){const o=jo("svelte-u9bgzb",document.head);c=l(o,"META",{name:!0,content:!0}),o.forEach(t),C=a(e),p=l(e,"P",{}),z(p).forEach(t),y=a(e),h(w.$$.fragment,e),f=a(e),h(x.$$.fragment,e),G=a(e),T=l(e,"P",{"data-svelte-h":!0}),d(T)!=="svelte-1s05gbs"&&(T.innerHTML=E),Ge=a(e),N=l(e,"P",{"data-svelte-h":!0}),d(N)!=="svelte-vfdo9a"&&(N.textContent=Vt),Ne=a(e),V=l(e,"P",{"data-svelte-h":!0}),d(V)!=="svelte-mbr49z"&&(V.innerHTML=St),Ve=a(e),S=l(e,"P",{"data-svelte-h":!0}),d(S)!=="svelte-h8tmfu"&&(S.innerHTML=Xt),Se=a(e),h(P.$$.fragment,e),Xe=a(e),X=l(e,"P",{"data-svelte-h":!0}),d(X)!=="svelte-axv494"&&(X.textContent=Yt),Ye=a(e),Y=l(e,"UL",{"data-svelte-h":!0}),d(Y)!=="svelte-1molplc"&&(Y.innerHTML=Qt),Qe=a(e),Q=l(e,"P",{"data-svelte-h":!0}),d(Q)!=="svelte-eidtfm"&&(Q.textContent=Ot),Oe=a(e),h(O.$$.fragment,e),Ke=a(e),K=l(e,"P",{"data-svelte-h":!0}),d(K)!=="svelte-1ovdf29"&&(K.textContent=Kt),et=a(e),ee=l(e,"UL",{"data-svelte-h":!0}),d(ee)!=="svelte-yjbkch"&&(ee.innerHTML=eo),tt=a(e),h(te.$$.fragment,e),ot=a(e),oe=l(e,"P",{"data-svelte-h":!0}),d(oe)!=="svelte-cm0lfz"&&(oe.textContent=to),nt=a(e),h(ne.$$.fragment,e),st=a(e),se=l(e,"P",{"data-svelte-h":!0}),d(se)!=="svelte-xefpq6"&&(se.innerHTML=oo),at=a(e),ae=l(e,"UL",{"data-svelte-h":!0}),d(ae)!=="svelte-1xmggxn"&&(ae.innerHTML=no),rt=a(e),re=l(e,"P",{"data-svelte-h":!0}),d(re)!=="svelte-z5ta87"&&(re.innerHTML=so),lt=a(e),h(le.$$.fragment,e),it=a(e),L=l(e,"DIV",{class:!0});var M=z(L);h(ie.$$.fragment,M),gt=a(M),ve=l(M,"P",{"data-svelte-h":!0}),d(ve)!=="svelte-1vmwvh5"&&(ve.textContent=ao),kt=a(M),Te=l(M,"P",{"data-svelte-h":!0}),d(Te)!=="svelte-1thmlao"&&(Te.innerHTML=ro),bt=a(M),xe=l(M,"DIV",{class:!0});var De=z(xe);h(de.$$.fragment,De),De.forEach(t),yt=a(M),H=l(M,"DIV",{class:!0});var ye=z(H);h(ce.$$.fragment,ye),vt=a(ye),Le=l(ye,"P",{"data-svelte-h":!0}),d(Le)!=="svelte-1f4f5kp"&&(Le.innerHTML=lo),ye.forEach(t),Tt=a(M),q=l(M,"DIV",{class:!0});var W=z(q);h(me.$$.fragment,W),xt=a(W),Me=l(W,"P",{"data-svelte-h":!0}),d(Me)!=="svelte-13bfd60"&&(Me.textContent=io),Lt=a(W),h(R.$$.fragment,W),Mt=a(W),Ce=l(W,"P",{"data-svelte-h":!0}),d(Ce)!=="svelte-wtrslu"&&(Ce.textContent=co),W.forEach(t),Ct=a(M),Z=l(M,"DIV",{class:!0});var ft=z(Z);h(pe.$$.fragment,ft),wt=a(ft),we=l(ft,"P",{"data-svelte-h":!0}),d(we)!=="svelte-1slb66l"&&(we.textContent=mo),ft.forEach(t),M.forEach(t),dt=a(e),h(fe.$$.fragment,e),ct=a(e),m=l(e,"DIV",{class:!0});var v=z(m);h(ue.$$.fragment,v),$t=a(v),$e=l(v,"P",{"data-svelte-h":!0}),d($e)!=="svelte-15tdcz8"&&($e.textContent=po),zt=a(v),ze=l(v,"P",{"data-svelte-h":!0}),d(ze)!=="svelte-llhmpa"&&(ze.textContent=fo),jt=a(v),h(A.$$.fragment,v),qt=a(v),je=l(v,"P",{"data-svelte-h":!0}),d(je)!=="svelte-cnb6q1"&&(je.innerHTML=uo),Ft=a(v),qe=l(v,"P",{"data-svelte-h":!0}),d(qe)!=="svelte-6feuay"&&(qe.innerHTML=ho),Ut=a(v),$=l(v,"DIV",{class:!0});var I=z($);h(he.$$.fragment,I),It=a(I),Fe=l(I,"P",{"data-svelte-h":!0}),d(Fe)!=="svelte-1vll0v2"&&(Fe.textContent=_o),Jt=a(I),Ue=l(I,"P",{"data-svelte-h":!0}),d(Ue)!=="svelte-90np8u"&&(Ue.innerHTML=go),Et=a(I),Ie=l(I,"UL",{"data-svelte-h":!0}),d(Ie)!=="svelte-mlrsks"&&(Ie.innerHTML=ko),Pt=a(I),Je=l(I,"P",{"data-svelte-h":!0}),d(Je)!=="svelte-46aam0"&&(Je.textContent=bo),I.forEach(t),Ht=a(v),B=l(v,"DIV",{class:!0});var ut=z(B);h(_e.$$.fragment,ut),Rt=a(ut),Ee=l(ut,"P",{"data-svelte-h":!0}),d(Ee)!=="svelte-1wmjg8a"&&(Ee.innerHTML=yo),ut.forEach(t),Zt=a(v),U=l(v,"DIV",{class:!0});var Ae=z(U);h(ge.$$.fragment,Ae),At=a(Ae),Pe=l(Ae,"P",{"data-svelte-h":!0}),d(Pe)!=="svelte-zj1vf1"&&(Pe.innerHTML=vo),Bt=a(Ae),He=l(Ae,"P",{"data-svelte-h":!0}),d(He)!=="svelte-9vptpw"&&(He.textContent=To),Ae.forEach(t),Dt=a(v),D=l(v,"DIV",{class:!0});var ht=z(D);h(ke.$$.fragment,ht),Wt=a(ht),Re=l(ht,"P",{"data-svelte-h":!0}),d(Re)!=="svelte-nfci2w"&&(Re.innerHTML=xo),ht.forEach(t),Gt=a(v),Ze=l(v,"DIV",{class:!0});var Lo=z(Ze);h(be.$$.fragment,Lo),Lo.forEach(t),v.forEach(t),mt=a(e),Be=l(e,"P",{}),z(Be).forEach(t),this.h()},h(){j(c,"name","hf:doc:metadata"),j(c,"content",Eo),j(xe,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(H,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(q,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(Z,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(L,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j($,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(B,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(U,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(D,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(Ze,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),j(m,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,o){n(document.head,c),i(e,C,o),i(e,p,o),i(e,y,o),_(w,e,o),i(e,f,o),_(x,e,o),i(e,G,o),i(e,T,o),i(e,Ge,o),i(e,N,o),i(e,Ne,o),i(e,V,o),i(e,Ve,o),i(e,S,o),i(e,Se,o),_(P,e,o),i(e,Xe,o),i(e,X,o),i(e,Ye,o),i(e,Y,o),i(e,Qe,o),i(e,Q,o),i(e,Oe,o),_(O,e,o),i(e,Ke,o),i(e,K,o),i(e,et,o),i(e,ee,o),i(e,tt,o),_(te,e,o),i(e,ot,o),i(e,oe,o),i(e,nt,o),_(ne,e,o),i(e,st,o),i(e,se,o),i(e,at,o),i(e,ae,o),i(e,rt,o),i(e,re,o),i(e,lt,o),_(le,e,o),i(e,it,o),i(e,L,o),_(ie,L,null),n(L,gt),n(L,ve),n(L,kt),n(L,Te),n(L,bt),n(L,xe),_(de,xe,null),n(L,yt),n(L,H),_(ce,H,null),n(H,vt),n(H,Le),n(L,Tt),n(L,q),_(me,q,null),n(q,xt),n(q,Me),n(q,Lt),_(R,q,null),n(q,Mt),n(q,Ce),n(L,Ct),n(L,Z),_(pe,Z,null),n(Z,wt),n(Z,we),i(e,dt,o),_(fe,e,o),i(e,ct,o),i(e,m,o),_(ue,m,null),n(m,$t),n(m,$e),n(m,zt),n(m,ze),n(m,jt),_(A,m,null),n(m,qt),n(m,je),n(m,Ft),n(m,qe),n(m,Ut),n(m,$),_(he,$,null),n($,It),n($,Fe),n($,Jt),n($,Ue),n($,Et),n($,Ie),n($,Pt),n($,Je),n(m,Ht),n(m,B),_(_e,B,null),n(B,Rt),n(B,Ee),n(m,Zt),n(m,U),_(ge,U,null),n(U,At),n(U,Pe),n(U,Bt),n(U,He),n(m,Dt),n(m,D),_(ke,D,null),n(D,Wt),n(D,Re),n(m,Gt),n(m,Ze),_(be,Ze,null),i(e,mt,o),i(e,Be,o),pt=!0},p(e,[o]){const M={};o&2&&(M.$$scope={dirty:o,ctx:e}),P.$set(M);const De={};o&2&&(De.$$scope={dirty:o,ctx:e}),R.$set(De);const ye={};o&2&&(ye.$$scope={dirty:o,ctx:e}),A.$set(ye)},i(e){pt||(g(w.$$.fragment,e),g(x.$$.fragment,e),g(P.$$.fragment,e),g(O.$$.fragment,e),g(te.$$.fragment,e),g(ne.$$.fragment,e),g(le.$$.fragment,e),g(ie.$$.fragment,e),g(de.$$.fragment,e),g(ce.$$.fragment,e),g(me.$$.fragment,e),g(R.$$.fragment,e),g(pe.$$.fragment,e),g(fe.$$.fragment,e),g(ue.$$.fragment,e),g(A.$$.fragment,e),g(he.$$.fragment,e),g(_e.$$.fragment,e),g(ge.$$.fragment,e),g(ke.$$.fragment,e),g(be.$$.fragment,e),pt=!0)},o(e){k(w.$$.fragment,e),k(x.$$.fragment,e),k(P.$$.fragment,e),k(O.$$.fragment,e),k(te.$$.fragment,e),k(ne.$$.fragment,e),k(le.$$.fragment,e),k(ie.$$.fragment,e),k(de.$$.fragment,e),k(ce.$$.fragment,e),k(me.$$.fragment,e),k(R.$$.fragment,e),k(pe.$$.fragment,e),k(fe.$$.fragment,e),k(ue.$$.fragment,e),k(A.$$.fragment,e),k(he.$$.fragment,e),k(_e.$$.fragment,e),k(ge.$$.fragment,e),k(ke.$$.fragment,e),k(be.$$.fragment,e),pt=!1},d(e){e&&(t(C),t(p),t(y),t(f),t(G),t(T),t(Ge),t(N),t(Ne),t(V),t(Ve),t(S),t(Se),t(Xe),t(X),t(Ye),t(Y),t(Qe),t(Q),t(Oe),t(Ke),t(K),t(et),t(ee),t(tt),t(ot),t(oe),t(nt),t(st),t(se),t(at),t(ae),t(rt),t(re),t(lt),t(it),t(L),t(dt),t(ct),t(m),t(mt),t(Be)),t(c),b(w,e),b(x,e),b(P,e),b(O,e),b(te,e),b(ne,e),b(le,e),b(ie),b(de),b(ce),b(me),b(R),b(pe),b(fe,e),b(ue),b(A),b(he),b(_e),b(ge),b(ke),b(be)}}}const Eo='{"title":"CodeLlama","local":"codellama","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"CodeLlamaTokenizer","local":"transformers.CodeLlamaTokenizer","sections":[],"depth":2},{"title":"CodeLlamaTokenizerFast","local":"transformers.CodeLlamaTokenizerFast","sections":[],"depth":2}],"depth":1}';function Po(J){return wo(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Go extends $o{constructor(c){super(),zo(this,c,Po,Jo,Co,{})}}export{Go as component};
