import{s as K,n as O,o as Q}from"../chunks/scheduler.9bc65507.js";import{S as V,i as W,g as o,s as a,r as X,A as Z,h as i,f as n,c,j as D,u as ee,x as l,k as J,y as te,a as d,v as ne,d as de,t as oe,w as ae}from"../chunks/index.707bf1b6.js";import{H as ie}from"../chunks/Heading.342b1fa6.js";function ce($){let r,y,b,k,s,w,h,I="Batched inputs are often different lengths, so they can’t be converted to fixed-size tensors. Padding and truncation are strategies for dealing with this problem, to create rectangular tensors from batches of varying lengths. Padding adds a special <strong>padding token</strong> to ensure shorter sequences will have the same length as either the longest sequence in a batch or the maximum length accepted by the model. Truncation works in the other direction by truncating long sequences.",z,u,R="In most cases, padding your batch to the length of the longest sequence and truncating to the maximum length a model can accept works pretty well. However, the API supports more strategies if you need them. The three arguments you need to are: <code>padding</code>, <code>truncation</code> and <code>max_length</code>.",L,p,G="The <code>padding</code> argument controls padding. It can be a boolean or a string:",H,m,N=`<li><code>True</code> or <code>&#39;longest&#39;</code>: pad to the longest sequence in the batch (no padding is applied if you only provide
a single sequence).</li> <li><code>&#39;max_length&#39;</code>: pad to a length specified by the <code>max_length</code> argument or the maximum length accepted
by the model if no <code>max_length</code> is provided (<code>max_length=None</code>). Padding will still be applied if you only provide a single sequence.</li> <li><code>False</code> or <code>&#39;do_not_pad&#39;</code>: no padding is applied. This is the default behavior.</li>`,M,g,Y="The <code>truncation</code> argument controls truncation. It can be a boolean or a string:",q,_,U=`<li><code>True</code> or <code>&#39;longest_first&#39;</code>: truncate to a maximum length specified by the <code>max_length</code> argument or
the maximum length accepted by the model if no <code>max_length</code> is provided (<code>max_length=None</code>). This will
truncate token by token, removing a token from the longest sequence in the pair until the proper length is
reached.</li> <li><code>&#39;only_second&#39;</code>: truncate to a maximum length specified by the <code>max_length</code> argument or the maximum
length accepted by the model if no <code>max_length</code> is provided (<code>max_length=None</code>). This will only truncate
the second sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.</li> <li><code>&#39;only_first&#39;</code>: truncate to a maximum length specified by the <code>max_length</code> argument or the maximum
length accepted by the model if no <code>max_length</code> is provided (<code>max_length=None</code>). This will only truncate
the first sentence of a pair if a pair of sequences (or a batch of pairs of sequences) is provided.</li> <li><code>False</code> or <code>&#39;do_not_truncate&#39;</code>: no truncation is applied. This is the default behavior.</li>`,P,f,j="The <code>max_length</code> argument controls the length of the padding and truncation. It can be an integer or <code>None</code>, in which case it will default to the maximum length the model can accept. If the model has no specific maximum input length, truncation or padding to <code>max_length</code> is deactivated.",E,x,B=`The following table summarizes the recommended way to setup padding and truncation. If you use pairs of input sequences in any of the following examples, you can replace <code>truncation=True</code> by a <code>STRATEGY</code> selected in
<code>[&#39;only_first&#39;, &#39;only_second&#39;, &#39;longest_first&#39;]</code>, i.e. <code>truncation=&#39;only_second&#39;</code> or <code>truncation=&#39;longest_first&#39;</code> to control how both sequences in the pair are truncated as detailed before.`,A,T,F="<thead><tr><th>Truncation</th> <th>Padding</th> <th>Instruction</th></tr></thead> <tbody><tr><td>no truncation</td> <td>no padding</td> <td><code>tokenizer(batch_sentences)</code></td></tr> <tr><td></td> <td>padding to max sequence in batch</td> <td><code>tokenizer(batch_sentences, padding=True)</code> or</td></tr> <tr><td></td> <td></td> <td><code>tokenizer(batch_sentences, padding=&#39;longest&#39;)</code></td></tr> <tr><td></td> <td>padding to max model input length</td> <td><code>tokenizer(batch_sentences, padding=&#39;max_length&#39;)</code></td></tr> <tr><td></td> <td>padding to specific length</td> <td><code>tokenizer(batch_sentences, padding=&#39;max_length&#39;, max_length=42)</code></td></tr> <tr><td></td> <td>padding to a multiple of a value</td> <td>`tokenizer(batch_sentences, padding=True, pad_to_multiple_of=8)</td></tr> <tr><td>truncation to max model input length</td> <td>no padding</td> <td><code>tokenizer(batch_sentences, truncation=True)</code> or</td></tr> <tr><td></td> <td></td> <td><code>tokenizer(batch_sentences, truncation=STRATEGY)</code></td></tr> <tr><td></td> <td>padding to max sequence in batch</td> <td><code>tokenizer(batch_sentences, padding=True, truncation=True)</code> or</td></tr> <tr><td></td> <td></td> <td><code>tokenizer(batch_sentences, padding=True, truncation=STRATEGY)</code></td></tr> <tr><td></td> <td>padding to max model input length</td> <td><code>tokenizer(batch_sentences, padding=&#39;max_length&#39;, truncation=True)</code> or</td></tr> <tr><td></td> <td></td> <td><code>tokenizer(batch_sentences, padding=&#39;max_length&#39;, truncation=STRATEGY)</code></td></tr> <tr><td></td> <td>padding to specific length</td> <td>Not possible</td></tr> <tr><td>truncation to specific length</td> <td>no padding</td> <td><code>tokenizer(batch_sentences, truncation=True, max_length=42)</code> or</td></tr> <tr><td></td> <td></td> <td><code>tokenizer(batch_sentences, truncation=STRATEGY, max_length=42)</code></td></tr> <tr><td></td> <td>padding to max sequence in batch</td> <td><code>tokenizer(batch_sentences, padding=True, truncation=True, max_length=42)</code> or</td></tr> <tr><td></td> <td></td> <td><code>tokenizer(batch_sentences, padding=True, truncation=STRATEGY, max_length=42)</code></td></tr> <tr><td></td> <td>padding to max model input length</td> <td>Not possible</td></tr> <tr><td></td> <td>padding to specific length</td> <td><code>tokenizer(batch_sentences, padding=&#39;max_length&#39;, truncation=True, max_length=42)</code> or</td></tr> <tr><td></td> <td></td> <td><code>tokenizer(batch_sentences, padding=&#39;max_length&#39;, truncation=STRATEGY, max_length=42)</code></td></tr></tbody>",C,v,S;return s=new ie({props:{title:"Padding and truncation",local:"padding-and-truncation",headingTag:"h1"}}),{c(){r=o("meta"),y=a(),b=o("p"),k=a(),X(s.$$.fragment),w=a(),h=o("p"),h.innerHTML=I,z=a(),u=o("p"),u.innerHTML=R,L=a(),p=o("p"),p.innerHTML=G,H=a(),m=o("ul"),m.innerHTML=N,M=a(),g=o("p"),g.innerHTML=Y,q=a(),_=o("ul"),_.innerHTML=U,P=a(),f=o("p"),f.innerHTML=j,E=a(),x=o("p"),x.innerHTML=B,A=a(),T=o("table"),T.innerHTML=F,C=a(),v=o("p"),this.h()},l(e){const t=Z("svelte-u9bgzb",document.head);r=i(t,"META",{name:!0,content:!0}),t.forEach(n),y=c(e),b=i(e,"P",{}),D(b).forEach(n),k=c(e),ee(s.$$.fragment,e),w=c(e),h=i(e,"P",{"data-svelte-h":!0}),l(h)!=="svelte-18l3nu8"&&(h.innerHTML=I),z=c(e),u=i(e,"P",{"data-svelte-h":!0}),l(u)!=="svelte-4dmk3p"&&(u.innerHTML=R),L=c(e),p=i(e,"P",{"data-svelte-h":!0}),l(p)!=="svelte-1xxctal"&&(p.innerHTML=G),H=c(e),m=i(e,"UL",{"data-svelte-h":!0}),l(m)!=="svelte-1pqvfv8"&&(m.innerHTML=N),M=c(e),g=i(e,"P",{"data-svelte-h":!0}),l(g)!=="svelte-14kdvi5"&&(g.innerHTML=Y),q=c(e),_=i(e,"UL",{"data-svelte-h":!0}),l(_)!=="svelte-1eic0m4"&&(_.innerHTML=U),P=c(e),f=i(e,"P",{"data-svelte-h":!0}),l(f)!=="svelte-n95phd"&&(f.innerHTML=j),E=c(e),x=i(e,"P",{"data-svelte-h":!0}),l(x)!=="svelte-1rrzmkp"&&(x.innerHTML=B),A=c(e),T=i(e,"TABLE",{"data-svelte-h":!0}),l(T)!=="svelte-1btfijn"&&(T.innerHTML=F),C=c(e),v=i(e,"P",{}),D(v).forEach(n),this.h()},h(){J(r,"name","hf:doc:metadata"),J(r,"content",re)},m(e,t){te(document.head,r),d(e,y,t),d(e,b,t),d(e,k,t),ne(s,e,t),d(e,w,t),d(e,h,t),d(e,z,t),d(e,u,t),d(e,L,t),d(e,p,t),d(e,H,t),d(e,m,t),d(e,M,t),d(e,g,t),d(e,q,t),d(e,_,t),d(e,P,t),d(e,f,t),d(e,E,t),d(e,x,t),d(e,A,t),d(e,T,t),d(e,C,t),d(e,v,t),S=!0},p:O,i(e){S||(de(s.$$.fragment,e),S=!0)},o(e){oe(s.$$.fragment,e),S=!1},d(e){e&&(n(y),n(b),n(k),n(w),n(h),n(z),n(u),n(L),n(p),n(H),n(m),n(M),n(g),n(q),n(_),n(P),n(f),n(E),n(x),n(A),n(T),n(C),n(v)),n(r),ae(s,e)}}}const re='{"title":"Padding and truncation","local":"padding-and-truncation","sections":[],"depth":1}';function le($){return Q(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class pe extends V{constructor(r){super(),W(this,r,le,ce,K,{})}}export{pe as component};
