import{s as be,o as ve,n as ye}from"../chunks/scheduler.9bc65507.js";import{S as Me,i as $e,g as p,s as o,r as w,A as ze,h as c,f as n,c as a,j as ne,u as k,x as h,k as se,y as M,a as s,v as T,d as _,t as b,w as v}from"../chunks/index.707bf1b6.js";import{D as Te}from"../chunks/Docstring.17db21ae.js";import{C as _e}from"../chunks/CodeBlock.54a9f38d.js";import{E as Se}from"../chunks/ExampleCodeBlock.4f515aa9.js";import{H as oe}from"../chunks/Heading.342b1fa6.js";function xe(L){let i,$="Example usage:",u,d,g;return d=new _e({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEdQVFN3M1Rva2VuaXplciUwQSUwQXRva2VuaXplciUyMCUzRCUyMEdQVFN3M1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIyQUktU3dlZGVuJTJGZ3B0LXN3My0xMjZtJTIyKSUwQXRva2VuaXplciglMjJTdmVuc2thJTIwJUMzJUE0ciUyMGt1bCElMjIpJTVCJTIyaW5wdXRfaWRzJTIyJTVE",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> GPTSw3Tokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = GPTSw3Tokenizer.from_pretrained(<span class="hljs-string">&quot;AI-Sweden/gpt-sw3-126m&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer(<span class="hljs-string">&quot;Svenska är kul!&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]
[<span class="hljs-number">1814</span>, <span class="hljs-number">377</span>, <span class="hljs-number">3617</span>, <span class="hljs-number">63504</span>]`,wrap:!1}}),{c(){i=p("p"),i.textContent=$,u=o(),w(d.$$.fragment)},l(r){i=c(r,"P",{"data-svelte-h":!0}),h(i)!=="svelte-1ni337v"&&(i.textContent=$),u=a(r),k(d.$$.fragment,r)},m(r,f){s(r,i,f),s(r,u,f),T(d,r,f),g=!0},p:ye,i(r){g||(_(d.$$.fragment,r),g=!0)},o(r){b(d.$$.fragment,r),g=!1},d(r){r&&(n(i),n(u)),v(d,r)}}}function Pe(L){let i,$,u,d,g,r,f,H,z,pe=`The GPT-Sw3 model was first proposed in
<a href="http://www.lrec-conf.org/proceedings/lrec2022/pdf/2022.lrec-1.376.pdf" rel="nofollow">Lessons Learned from GPT-SW3: Building the First Large-Scale Generative Language Model for Swedish</a>
by Ariel Ekgren, Amaru Cuba Gyllensten, Evangelia Gogoulou, Alice Heiman, Severine Verlinden, Joey Öhman,
Fredrik Carlsson, Magnus Sahlgren.`,N,S,ce="Since that first paper the authors have extended their work and trained new models on their new 1.2TB corpora named The Nordic Pile.",q,x,me=`GPT-Sw3 is a collection of large decoder-only pretrained transformer language models that were developed by AI Sweden
in collaboration with RISE and the WASP WARA for Media and Language. GPT-Sw3 has been trained on a dataset containing
320B tokens in Swedish, Norwegian, Danish, Icelandic, English, and programming code. The model was pretrained using a
causal language modeling (CLM) objective utilizing the NeMo Megatron GPT implementation.`,F,P,de='This model was contributed by <a href="https://huggingface.co/AI-Sweden" rel="nofollow">AI Sweden</a>.',A,G,ge=`The implementation uses the <a href="https://huggingface.co/docs/transformers/model_doc/gpt2" rel="nofollow">GPT2Model</a> coupled
with our <code>GPTSw3Tokenizer</code>. This means that <code>AutoTokenizer</code> and <code>AutoModelForCausalLM</code> map to our tokenizer
implementation and the corresponding GPT2 model implementation respectively.
<em>Note that sentencepiece is required to use our tokenizer and can be installed with:</em> <code>pip install transformers[sentencepiece]</code> or <code>pip install sentencepiece</code>`,X,C,fe="Example usage:",D,U,Q,j,Y,J,ue='<li><a href="../tasks/sequence_classification">Text classification task guide</a></li> <li><a href="../tasks/token_classification">Token classification task guide</a></li> <li><a href="../tasks/language_modeling">Causal language modeling task guide</a></li>',K,I,O,l,R,ae,Z,he='Construct an GPTSw3 tokenizer. Based on <a href="https://github.com/google/sentencepiece" rel="nofollow">SentencePiece</a>.',re,E,we=`This tokenizer inherits from <a href="/docs/transformers/main/en/main_classes/tokenizer#transformers.PreTrainedTokenizer">PreTrainedTokenizer</a> which contains most of the main methods. Users should refer to
this superclass for more information regarding those methods.`,ie,y,le,W,V,ee,B,te;return g=new oe({props:{title:"GPT-Sw3",local:"gpt-sw3",headingTag:"h1"}}),f=new oe({props:{title:"Overview",local:"overview",headingTag:"h2"}}),U=new _e({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMkMlMjBBdXRvTW9kZWxGb3JDYXVzYWxMTSUwQSUwQXRva2VuaXplciUyMCUzRCUyMEF1dG9Ub2tlbml6ZXIuZnJvbV9wcmV0cmFpbmVkKCUyMkFJLVN3ZWRlbiUyRmdwdC1zdzMtMzU2bSUyMiklMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvckNhdXNhbExNLmZyb21fcHJldHJhaW5lZCglMjJBSS1Td2VkZW4lMkZncHQtc3czLTM1Nm0lMjIpJTBBJTBBaW5wdXRfaWRzJTIwJTNEJTIwdG9rZW5pemVyKCUyMlRyJUMzJUE0ZCUyMCVDMyVBNHIlMjBmaW5hJTIwZiVDMyVCNnIlMjBhdHQlMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSU1QiUyMmlucHV0X2lkcyUyMiU1RCUwQSUwQWdlbmVyYXRlZF90b2tlbl9pZHMlMjAlM0QlMjBtb2RlbC5nZW5lcmF0ZShpbnB1dHMlM0RpbnB1dF9pZHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDEwJTJDJTIwZG9fc2FtcGxlJTNEVHJ1ZSklNUIwJTVEJTBBJTBBcHJpbnQodG9rZW5pemVyLmRlY29kZShnZW5lcmF0ZWRfdG9rZW5faWRzKSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer, AutoModelForCausalLM

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;AI-Sweden/gpt-sw3-356m&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForCausalLM.from_pretrained(<span class="hljs-string">&quot;AI-Sweden/gpt-sw3-356m&quot;</span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>input_ids = tokenizer(<span class="hljs-string">&quot;Träd är fina för att&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)[<span class="hljs-string">&quot;input_ids&quot;</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span>generated_token_ids = model.generate(inputs=input_ids, max_new_tokens=<span class="hljs-number">10</span>, do_sample=<span class="hljs-literal">True</span>)[<span class="hljs-number">0</span>]

<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-built_in">print</span>(tokenizer.decode(generated_token_ids))
Träd är fina för att de är färgstarka. Men ibland är det fint`,wrap:!1}}),j=new oe({props:{title:"Documentation resources",local:"documentation-resources",headingTag:"h2"}}),I=new oe({props:{title:"GPTSw3Tokenizer",local:"transformers.GPTSw3Tokenizer",headingTag:"h2"}}),R=new Te({props:{name:"class transformers.GPTSw3Tokenizer",anchor:"transformers.GPTSw3Tokenizer",parameters:[{name:"vocab_file",val:""},{name:"do_lower_case",val:" = False"},{name:"remove_space",val:" = False"},{name:"keep_accents",val:" = False"},{name:"pad_token",val:" = None"},{name:"unk_token",val:" = None"},{name:"eos_token",val:" = None"},{name:"bos_token",val:" = None"},{name:"sp_model_kwargs",val:": typing.Union[typing.Dict[str, typing.Any], NoneType] = None"},{name:"**kwargs",val:""}],parametersDescription:[{anchor:"transformers.GPTSw3Tokenizer.vocab_file",description:`<strong>vocab_file</strong> (<code>str</code>) &#x2014;
<a href="https://github.com/google/sentencepiece" rel="nofollow">SentencePiece</a> file (generally has a <em>.spm</em> extension) that
contains the vocabulary necessary to instantiate a tokenizer.`,name:"vocab_file"},{anchor:"transformers.GPTSw3Tokenizer.do_lower_case",description:`<strong>do_lower_case</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to lowercase the input when tokenizing.`,name:"do_lower_case"},{anchor:"transformers.GPTSw3Tokenizer.remove_space",description:`<strong>remove_space</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to strip the text when tokenizing (removing excess spaces before and after the string).`,name:"remove_space"},{anchor:"transformers.GPTSw3Tokenizer.keep_accents",description:`<strong>keep_accents</strong> (<code>bool</code>, <em>optional</em>, defaults to <code>False</code>) &#x2014;
Whether or not to keep accents when tokenizing.`,name:"keep_accents"},{anchor:"transformers.GPTSw3Tokenizer.pad_token",description:`<strong>pad_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The token used for padding, for example when batching sequences of different lengths. If not provided, will
default to &#x2019;<pad>&#x2019; or &#x2019;<unk>&#x2019; depending on model size.</unk></pad>`,name:"pad_token"},{anchor:"transformers.GPTSw3Tokenizer.unk_token",description:`<strong>unk_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
token instead. If not provided, will default to &#x2019;<unk>&#x2018;.</unk>`,name:"unk_token"},{anchor:"transformers.GPTSw3Tokenizer.eos_token",description:`<strong>eos_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The end of sequence token seen during pretraining. If not provided, will default to &#x2019;&lt;|endoftext|&gt;&#x2019;`,name:"eos_token"},{anchor:"transformers.GPTSw3Tokenizer.bos_token",description:`<strong>bos_token</strong> (<code>str</code>, <em>optional</em>) &#x2014;
The beginning of sequence token that can be used for downstream task, was not seen during pretraining. If
not provided, will default to &#x2019;<s>&#x2019; or &#x2019;&lt;|endoftext|&gt;&#x2019;, depending on model size.</s>`,name:"bos_token"},{anchor:"transformers.GPTSw3Tokenizer.sp_model_kwargs",description:`<strong>sp_model_kwargs</strong> (<code>dict</code>, <em>optional</em>) &#x2014;
Will be passed to the <code>SentencePieceProcessor.__init__()</code> method. The <a href="https://github.com/google/sentencepiece/tree/master/python" rel="nofollow">Python wrapper for
SentencePiece</a> can be used, among other things,
to set:</p>
<ul>
<li>
<p><code>enable_sampling</code>: Enable subword regularization.</p>
</li>
<li>
<p><code>nbest_size</code>: Sampling parameters for unigram. Invalid for BPE-Dropout.</p>
<ul>
<li><code>nbest_size = {0,1}</code>: No sampling is performed.</li>
<li><code>nbest_size &gt; 1</code>: samples from the nbest_size results.</li>
<li><code>nbest_size &lt; 0</code>: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
using forward-filtering-and-backward-sampling algorithm.</li>
</ul>
</li>
<li>
<p><code>alpha</code>: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
BPE-dropout.</p>
</li>
</ul>`,name:"sp_model_kwargs"},{anchor:"transformers.GPTSw3Tokenizer.sp_model",description:`<strong>sp_model</strong> (<code>SentencePieceProcessor</code>) &#x2014;
The <em>SentencePiece</em> processor that is used for every conversion (string, tokens and IDs).`,name:"sp_model"},{anchor:"transformers.GPTSw3Tokenizer.whitespaces",description:`<strong>whitespaces</strong> (<code>set</code>) &#x2014;
The whitespaces that are replaced in the whitespace normalization in preprocessing.`,name:"whitespaces"},{anchor:"transformers.GPTSw3Tokenizer.non_printing_characters_re",description:`<strong>non_printing_characters_re</strong> (<code>Pattern</code>) &#x2014;
The compiled regular expression to remove non-printing characters in preprocessing.`,name:"non_printing_characters_re"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/gpt_sw3/tokenization_gpt_sw3.py#L41"}}),y=new Se({props:{anchor:"transformers.GPTSw3Tokenizer.example",$$slots:{default:[xe]},$$scope:{ctx:L}}}),V=new Te({props:{name:"save_vocabulary",anchor:"transformers.GPTSw3Tokenizer.save_vocabulary",parameters:[{name:"save_directory",val:": str"},{name:"filename_prefix",val:": typing.Optional[str] = None"}],source:"https://github.com/huggingface/transformers/blob/main/src/transformers/models/gpt_sw3/tokenization_gpt_sw3.py#L254"}}),{c(){i=p("meta"),$=o(),u=p("p"),d=o(),w(g.$$.fragment),r=o(),w(f.$$.fragment),H=o(),z=p("p"),z.innerHTML=pe,N=o(),S=p("p"),S.textContent=ce,q=o(),x=p("p"),x.textContent=me,F=o(),P=p("p"),P.innerHTML=de,A=o(),G=p("p"),G.innerHTML=ge,X=o(),C=p("p"),C.textContent=fe,D=o(),w(U.$$.fragment),Q=o(),w(j.$$.fragment),Y=o(),J=p("ul"),J.innerHTML=ue,K=o(),w(I.$$.fragment),O=o(),l=p("div"),w(R.$$.fragment),ae=o(),Z=p("p"),Z.innerHTML=he,re=o(),E=p("p"),E.innerHTML=we,ie=o(),w(y.$$.fragment),le=o(),W=p("div"),w(V.$$.fragment),ee=o(),B=p("p"),this.h()},l(e){const t=ze("svelte-u9bgzb",document.head);i=c(t,"META",{name:!0,content:!0}),t.forEach(n),$=a(e),u=c(e,"P",{}),ne(u).forEach(n),d=a(e),k(g.$$.fragment,e),r=a(e),k(f.$$.fragment,e),H=a(e),z=c(e,"P",{"data-svelte-h":!0}),h(z)!=="svelte-1qucz45"&&(z.innerHTML=pe),N=a(e),S=c(e,"P",{"data-svelte-h":!0}),h(S)!=="svelte-1uddber"&&(S.textContent=ce),q=a(e),x=c(e,"P",{"data-svelte-h":!0}),h(x)!=="svelte-19zkpwq"&&(x.textContent=me),F=a(e),P=c(e,"P",{"data-svelte-h":!0}),h(P)!=="svelte-1u6kw5m"&&(P.innerHTML=de),A=a(e),G=c(e,"P",{"data-svelte-h":!0}),h(G)!=="svelte-4mp7uz"&&(G.innerHTML=ge),X=a(e),C=c(e,"P",{"data-svelte-h":!0}),h(C)!=="svelte-1ni337v"&&(C.textContent=fe),D=a(e),k(U.$$.fragment,e),Q=a(e),k(j.$$.fragment,e),Y=a(e),J=c(e,"UL",{"data-svelte-h":!0}),h(J)!=="svelte-v6nfyh"&&(J.innerHTML=ue),K=a(e),k(I.$$.fragment,e),O=a(e),l=c(e,"DIV",{class:!0});var m=ne(l);k(R.$$.fragment,m),ae=a(m),Z=c(m,"P",{"data-svelte-h":!0}),h(Z)!=="svelte-19f98fg"&&(Z.innerHTML=he),re=a(m),E=c(m,"P",{"data-svelte-h":!0}),h(E)!=="svelte-1c3t5ty"&&(E.innerHTML=we),ie=a(m),k(y.$$.fragment,m),le=a(m),W=c(m,"DIV",{class:!0});var ke=ne(W);k(V.$$.fragment,ke),ke.forEach(n),m.forEach(n),ee=a(e),B=c(e,"P",{}),ne(B).forEach(n),this.h()},h(){se(i,"name","hf:doc:metadata"),se(i,"content",Ge),se(W,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"),se(l,"class","docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8")},m(e,t){M(document.head,i),s(e,$,t),s(e,u,t),s(e,d,t),T(g,e,t),s(e,r,t),T(f,e,t),s(e,H,t),s(e,z,t),s(e,N,t),s(e,S,t),s(e,q,t),s(e,x,t),s(e,F,t),s(e,P,t),s(e,A,t),s(e,G,t),s(e,X,t),s(e,C,t),s(e,D,t),T(U,e,t),s(e,Q,t),T(j,e,t),s(e,Y,t),s(e,J,t),s(e,K,t),T(I,e,t),s(e,O,t),s(e,l,t),T(R,l,null),M(l,ae),M(l,Z),M(l,re),M(l,E),M(l,ie),T(y,l,null),M(l,le),M(l,W),T(V,W,null),s(e,ee,t),s(e,B,t),te=!0},p(e,[t]){const m={};t&2&&(m.$$scope={dirty:t,ctx:e}),y.$set(m)},i(e){te||(_(g.$$.fragment,e),_(f.$$.fragment,e),_(U.$$.fragment,e),_(j.$$.fragment,e),_(I.$$.fragment,e),_(R.$$.fragment,e),_(y.$$.fragment,e),_(V.$$.fragment,e),te=!0)},o(e){b(g.$$.fragment,e),b(f.$$.fragment,e),b(U.$$.fragment,e),b(j.$$.fragment,e),b(I.$$.fragment,e),b(R.$$.fragment,e),b(y.$$.fragment,e),b(V.$$.fragment,e),te=!1},d(e){e&&(n($),n(u),n(d),n(r),n(H),n(z),n(N),n(S),n(q),n(x),n(F),n(P),n(A),n(G),n(X),n(C),n(D),n(Q),n(Y),n(J),n(K),n(O),n(l),n(ee),n(B)),n(i),v(g,e),v(f,e),v(U,e),v(j,e),v(I,e),v(R),v(y),v(V)}}}const Ge='{"title":"GPT-Sw3","local":"gpt-sw3","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Documentation resources","local":"documentation-resources","sections":[],"depth":2},{"title":"GPTSw3Tokenizer","local":"transformers.GPTSw3Tokenizer","sections":[],"depth":2}],"depth":1}';function Ce(L){return ve(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ze extends Me{constructor(i){super(),$e(this,i,Ce,Pe,be,{})}}export{Ze as component};
