import{s as oe,n as ce,o as fe}from"../chunks/scheduler.56730f09.js";import{S as Me,i as de,g as r,s as n,r as U,A as ye,h as p,f as s,c as a,j as ie,u as Z,x as m,k as me,y as ke,a as l,v as J,d as w,t as V,w as _}from"../chunks/index.1f144517.js";import{C as K}from"../chunks/CodeBlock.738eeccb.js";import{H as O}from"../chunks/Heading.57d46534.js";function ge(D){let i,C,B,F,o,N,c,ee=`<code>PreTrainedTokenizerFast</code>는 <a href="https://huggingface.co/docs/tokenizers" rel="nofollow">🤗 Tokenizers</a> 라이브러리에 기반합니다. 🤗 Tokenizers 라이브러리의 토크나이저는
🤗 Transformers로 매우 간단하게 불러올 수 있습니다.`,I,f,te="구체적인 내용에 들어가기 전에, 몇 줄의 코드로 더미 토크나이저를 만들어 보겠습니다:",P,M,Q,d,se="우리가 정의한 파일을 통해 이제 학습된 토크나이저를 갖게 되었습니다. 이 런타임에서 계속 사용하거나 JSON 파일로 저장하여 나중에 사용할 수 있습니다.",W,y,X,k,le=`🤗 Transformers 라이브러리에서 이 토크나이저 객체를 활용하는 방법을 살펴보겠습니다.
<code>PreTrainedTokenizerFast</code> 클래스는 인스턴스화된 <em>토크나이저</em> 객체를 인수로 받아 쉽게 인스턴스화할 수 있습니다:`,x,g,G,j,ne='이제 <code>fast_tokenizer</code> 객체는 🤗 Transformers 토크나이저에서 공유하는 모든 메소드와 함께 사용할 수 있습니다! 자세한 내용은 <a href="main_classes/tokenizer">토크나이저 페이지</a>를 참조하세요.',E,h,R,u,ae="JSON 파일에서 토크나이저를 불러오기 위해, 먼저 토크나이저를 저장해 보겠습니다:",S,T,q,$,re="JSON 파일을 저장한 경로는 <code>tokenizer_file</code> 매개변수를 사용하여 <code>PreTrainedTokenizerFast</code> 초기화 메소드에 전달할 수 있습니다:",H,z,L,b,pe='이제 <code>fast_tokenizer</code> 객체는 🤗 Transformers 토크나이저에서 공유하는 모든 메소드와 함께 사용할 수 있습니다! 자세한 내용은 <a href="main_classes/tokenizer">토크나이저 페이지</a>를 참조하세요.',A,v,Y;return o=new O({props:{title:"🤗 Tokenizers 라이브러리의 토크나이저 사용하기",local:"use-tokenizers-from-tokenizers",headingTag:"h1"}}),M=new K({props:{code:"ZnJvbSUyMHRva2VuaXplcnMlMjBpbXBvcnQlMjBUb2tlbml6ZXIlMEFmcm9tJTIwdG9rZW5pemVycy5tb2RlbHMlMjBpbXBvcnQlMjBCUEUlMEFmcm9tJTIwdG9rZW5pemVycy50cmFpbmVycyUyMGltcG9ydCUyMEJwZVRyYWluZXIlMEFmcm9tJTIwdG9rZW5pemVycy5wcmVfdG9rZW5pemVycyUyMGltcG9ydCUyMFdoaXRlc3BhY2UlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBUb2tlbml6ZXIoQlBFKHVua190b2tlbiUzRCUyMiU1QlVOSyU1RCUyMikpJTBBdHJhaW5lciUyMCUzRCUyMEJwZVRyYWluZXIoc3BlY2lhbF90b2tlbnMlM0QlNUIlMjIlNUJVTkslNUQlMjIlMkMlMjAlMjIlNUJDTFMlNUQlMjIlMkMlMjAlMjIlNUJTRVAlNUQlMjIlMkMlMjAlMjIlNUJQQUQlNUQlMjIlMkMlMjAlMjIlNUJNQVNLJTVEJTIyJTVEKSUwQSUwQXRva2VuaXplci5wcmVfdG9rZW5pemVyJTIwJTNEJTIwV2hpdGVzcGFjZSgpJTBBZmlsZXMlMjAlM0QlMjAlNUIuLi4lNUQlMEF0b2tlbml6ZXIudHJhaW4oZmlsZXMlMkMlMjB0cmFpbmVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers <span class="hljs-keyword">import</span> Tokenizer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.models <span class="hljs-keyword">import</span> BPE
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.trainers <span class="hljs-keyword">import</span> BpeTrainer
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> tokenizers.pre_tokenizers <span class="hljs-keyword">import</span> Whitespace

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = Tokenizer(BPE(unk_token=<span class="hljs-string">&quot;[UNK]&quot;</span>))
<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = BpeTrainer(special_tokens=[<span class="hljs-string">&quot;[UNK]&quot;</span>, <span class="hljs-string">&quot;[CLS]&quot;</span>, <span class="hljs-string">&quot;[SEP]&quot;</span>, <span class="hljs-string">&quot;[PAD]&quot;</span>, <span class="hljs-string">&quot;[MASK]&quot;</span>])

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.pre_tokenizer = Whitespace()
<span class="hljs-meta">&gt;&gt;&gt; </span>files = [...]
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.train(files, trainer)`,wrap:!1}}),y=new O({props:{title:"토크나이저 객체로부터 직접 불러오기",local:"loading-directly-from-the-tokenizer-object",headingTag:"h2"}}),g=new K({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfb2JqZWN0JTNEdG9rZW5pemVyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_object=tokenizer)`,wrap:!1}}),h=new O({props:{title:"JSON 파일에서 불러오기",local:"loading-from-a-JSON-file",headingTag:"h2"}}),T=new K({props:{code:"dG9rZW5pemVyLnNhdmUoJTIydG9rZW5pemVyLmpzb24lMjIp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer.save(<span class="hljs-string">&quot;tokenizer.json&quot;</span>)',wrap:!1}}),z=new K({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFByZVRyYWluZWRUb2tlbml6ZXJGYXN0JTBBJTBBZmFzdF90b2tlbml6ZXIlMjAlM0QlMjBQcmVUcmFpbmVkVG9rZW5pemVyRmFzdCh0b2tlbml6ZXJfZmlsZSUzRCUyMnRva2VuaXplci5qc29uJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> PreTrainedTokenizerFast

<span class="hljs-meta">&gt;&gt;&gt; </span>fast_tokenizer = PreTrainedTokenizerFast(tokenizer_file=<span class="hljs-string">&quot;tokenizer.json&quot;</span>)`,wrap:!1}}),{c(){i=r("meta"),C=n(),B=r("p"),F=n(),U(o.$$.fragment),N=n(),c=r("p"),c.innerHTML=ee,I=n(),f=r("p"),f.textContent=te,P=n(),U(M.$$.fragment),Q=n(),d=r("p"),d.textContent=se,W=n(),U(y.$$.fragment),X=n(),k=r("p"),k.innerHTML=le,x=n(),U(g.$$.fragment),G=n(),j=r("p"),j.innerHTML=ne,E=n(),U(h.$$.fragment),R=n(),u=r("p"),u.textContent=ae,S=n(),U(T.$$.fragment),q=n(),$=r("p"),$.innerHTML=re,H=n(),U(z.$$.fragment),L=n(),b=r("p"),b.innerHTML=pe,A=n(),v=r("p"),this.h()},l(e){const t=ye("svelte-u9bgzb",document.head);i=p(t,"META",{name:!0,content:!0}),t.forEach(s),C=a(e),B=p(e,"P",{}),ie(B).forEach(s),F=a(e),Z(o.$$.fragment,e),N=a(e),c=p(e,"P",{"data-svelte-h":!0}),m(c)!=="svelte-1ilwdv2"&&(c.innerHTML=ee),I=a(e),f=p(e,"P",{"data-svelte-h":!0}),m(f)!=="svelte-nobjxu"&&(f.textContent=te),P=a(e),Z(M.$$.fragment,e),Q=a(e),d=p(e,"P",{"data-svelte-h":!0}),m(d)!=="svelte-vmdasx"&&(d.textContent=se),W=a(e),Z(y.$$.fragment,e),X=a(e),k=p(e,"P",{"data-svelte-h":!0}),m(k)!=="svelte-10u78cn"&&(k.innerHTML=le),x=a(e),Z(g.$$.fragment,e),G=a(e),j=p(e,"P",{"data-svelte-h":!0}),m(j)!=="svelte-tdf1x7"&&(j.innerHTML=ne),E=a(e),Z(h.$$.fragment,e),R=a(e),u=p(e,"P",{"data-svelte-h":!0}),m(u)!=="svelte-16yucd6"&&(u.textContent=ae),S=a(e),Z(T.$$.fragment,e),q=a(e),$=p(e,"P",{"data-svelte-h":!0}),m($)!=="svelte-126md19"&&($.innerHTML=re),H=a(e),Z(z.$$.fragment,e),L=a(e),b=p(e,"P",{"data-svelte-h":!0}),m(b)!=="svelte-tdf1x7"&&(b.innerHTML=pe),A=a(e),v=p(e,"P",{}),ie(v).forEach(s),this.h()},h(){me(i,"name","hf:doc:metadata"),me(i,"content",je)},m(e,t){ke(document.head,i),l(e,C,t),l(e,B,t),l(e,F,t),J(o,e,t),l(e,N,t),l(e,c,t),l(e,I,t),l(e,f,t),l(e,P,t),J(M,e,t),l(e,Q,t),l(e,d,t),l(e,W,t),J(y,e,t),l(e,X,t),l(e,k,t),l(e,x,t),J(g,e,t),l(e,G,t),l(e,j,t),l(e,E,t),J(h,e,t),l(e,R,t),l(e,u,t),l(e,S,t),J(T,e,t),l(e,q,t),l(e,$,t),l(e,H,t),J(z,e,t),l(e,L,t),l(e,b,t),l(e,A,t),l(e,v,t),Y=!0},p:ce,i(e){Y||(w(o.$$.fragment,e),w(M.$$.fragment,e),w(y.$$.fragment,e),w(g.$$.fragment,e),w(h.$$.fragment,e),w(T.$$.fragment,e),w(z.$$.fragment,e),Y=!0)},o(e){V(o.$$.fragment,e),V(M.$$.fragment,e),V(y.$$.fragment,e),V(g.$$.fragment,e),V(h.$$.fragment,e),V(T.$$.fragment,e),V(z.$$.fragment,e),Y=!1},d(e){e&&(s(C),s(B),s(F),s(N),s(c),s(I),s(f),s(P),s(Q),s(d),s(W),s(X),s(k),s(x),s(G),s(j),s(E),s(R),s(u),s(S),s(q),s($),s(H),s(L),s(b),s(A),s(v)),s(i),_(o,e),_(M,e),_(y,e),_(g,e),_(h,e),_(T,e),_(z,e)}}}const je='{"title":"🤗 Tokenizers 라이브러리의 토크나이저 사용하기","local":"use-tokenizers-from-tokenizers","sections":[{"title":"토크나이저 객체로부터 직접 불러오기","local":"loading-directly-from-the-tokenizer-object","sections":[],"depth":2},{"title":"JSON 파일에서 불러오기","local":"loading-from-a-JSON-file","sections":[],"depth":2}],"depth":1}';function he(D){return fe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class be extends Me{constructor(i){super(),de(this,i,he,ge,oe,{})}}export{be as component};
