import{s as ra,o as ca,n as is}from"../chunks/scheduler.56730f09.js";import{S as ia,i as oa,g,s as c,r as j,A as ma,h as w,f as a,c as i,j as aa,u as h,x as J,k as ta,y as Ma,a as e,v as d,d as u,t as y,w as f,m as na,n as pa}from"../chunks/index.1f144517.js";import{T as vl}from"../chunks/Tip.41e845e5.js";import{Y as ea}from"../chunks/Youtube.62e0f062.js";import{C as _}from"../chunks/CodeBlock.738eeccb.js";import{D as ja}from"../chunks/DocNotebookDropdown.b2e82107.js";import{F as Bl,M as Ss}from"../chunks/Markdown.c541024b.js";import{H as Ys}from"../chunks/Heading.57d46534.js";function ha(k){let t,o,l='<a href="../model_doc/albert">ALBERT</a>, <a href="../model_doc/bert">BERT</a>, <a href="../model_doc/big_bird">BigBird</a>, <a href="../model_doc/biogpt">BioGpt</a>, <a href="../model_doc/bloom">BLOOM</a>, <a href="../model_doc/camembert">CamemBERT</a>, <a href="../model_doc/canine">CANINE</a>, <a href="../model_doc/convbert">ConvBERT</a>, <a href="../model_doc/data2vec-text">Data2VecText</a>, <a href="../model_doc/deberta">DeBERTa</a>, <a href="../model_doc/deberta-v2">DeBERTa-v2</a>, <a href="../model_doc/distilbert">DistilBERT</a>, <a href="../model_doc/electra">ELECTRA</a>, <a href="../model_doc/ernie">ERNIE</a>, <a href="../model_doc/ernie_m">ErnieM</a>, <a href="../model_doc/esm">ESM</a>, <a href="../model_doc/flaubert">FlauBERT</a>, <a href="../model_doc/fnet">FNet</a>, <a href="../model_doc/funnel">Funnel Transformer</a>, <a href="../model_doc/gpt-sw3">GPT-Sw3</a>, <a href="../model_doc/gpt2">OpenAI GPT-2</a>, <a href="../model_doc/gpt_bigcode">GPTBigCode</a>, <a href="../model_doc/ibert">I-BERT</a>, <a href="../model_doc/layoutlm">LayoutLM</a>, <a href="../model_doc/layoutlmv2">LayoutLMv2</a>, <a href="../model_doc/layoutlmv3">LayoutLMv3</a>, <a href="../model_doc/lilt">LiLT</a>, <a href="../model_doc/longformer">Longformer</a>, <a href="../model_doc/luke">LUKE</a>, <a href="../model_doc/markuplm">MarkupLM</a>, <a href="../model_doc/mega">MEGA</a>, <a href="../model_doc/megatron-bert">Megatron-BERT</a>, <a href="../model_doc/mobilebert">MobileBERT</a>, <a href="../model_doc/mpnet">MPNet</a>, <a href="../model_doc/nezha">Nezha</a>, <a href="../model_doc/nystromformer">Nyströmformer</a>, <a href="../model_doc/qdqbert">QDQBert</a>, <a href="../model_doc/rembert">RemBERT</a>, <a href="../model_doc/roberta">RoBERTa</a>, <a href="../model_doc/roberta-prelayernorm">RoBERTa-PreLayerNorm</a>, <a href="../model_doc/roc_bert">RoCBert</a>, <a href="../model_doc/roformer">RoFormer</a>, <a href="../model_doc/squeezebert">SqueezeBERT</a>, <a href="../model_doc/xlm">XLM</a>, <a href="../model_doc/xlm-roberta">XLM-RoBERTa</a>, <a href="../model_doc/xlm-roberta-xl">XLM-RoBERTa-XL</a>, <a href="../model_doc/xlnet">XLNet</a>, <a href="../model_doc/xmod">X-MOD</a>, <a href="../model_doc/yoso">YOSO</a>';return{c(){t=na(`이 튜토리얼에서 설명하는 작업은 다음 모델 아키텍처에 의해 지원됩니다:

`),o=g("p"),o.innerHTML=l},l(m){t=pa(m,`이 튜토리얼에서 설명하는 작업은 다음 모델 아키텍처에 의해 지원됩니다:

`),o=w(m,"P",{"data-svelte-h":!0}),J(o)!=="svelte-17zgief"&&(o.innerHTML=l)},m(m,b){e(m,t,b),e(m,o,b)},p:is,d(m){m&&(a(t),a(o))}}}function da(k){let t,o;return t=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvclRva2VuQ2xhc3NpZmljYXRpb24lMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGF0YUNvbGxhdG9yRm9yVG9rZW5DbGFzc2lmaWNhdGlvbih0b2tlbml6ZXIlM0R0b2tlbml6ZXIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer)`,wrap:!1}}),{c(){j(t.$$.fragment)},l(l){h(t.$$.fragment,l)},m(l,m){d(t,l,m),o=!0},p:is,i(l){o||(u(t.$$.fragment,l),o=!0)},o(l){y(t.$$.fragment,l),o=!1},d(l){f(t,l)}}}function ua(k){let t,o;return t=new Ss({props:{$$slots:{default:[da]},$$scope:{ctx:k}}}),{c(){j(t.$$.fragment)},l(l){h(t.$$.fragment,l)},m(l,m){d(t,l,m),o=!0},p(l,m){const b={};m&2&&(b.$$scope={dirty:m,ctx:l}),t.$set(b)},i(l){o||(u(t.$$.fragment,l),o=!0)},o(l){y(t.$$.fragment,l),o=!1},d(l){f(t,l)}}}function ya(k){let t,o;return t=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMERhdGFDb2xsYXRvckZvclRva2VuQ2xhc3NpZmljYXRpb24lMEElMEFkYXRhX2NvbGxhdG9yJTIwJTNEJTIwRGF0YUNvbGxhdG9yRm9yVG9rZW5DbGFzc2lmaWNhdGlvbih0b2tlbml6ZXIlM0R0b2tlbml6ZXIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnRmJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DataCollatorForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`,wrap:!1}}),{c(){j(t.$$.fragment)},l(l){h(t.$$.fragment,l)},m(l,m){d(t,l,m),o=!0},p:is,i(l){o||(u(t.$$.fragment,l),o=!0)},o(l){y(t.$$.fragment,l),o=!1},d(l){f(t,l)}}}function fa(k){let t,o;return t=new Ss({props:{$$slots:{default:[ya]},$$scope:{ctx:k}}}),{c(){j(t.$$.fragment)},l(l){h(t.$$.fragment,l)},m(l,m){d(t,l,m),o=!0},p(l,m){const b={};m&2&&(b.$$scope={dirty:m,ctx:l}),t.$set(b)},i(l){o||(u(t.$$.fragment,l),o=!0)},o(l){y(t.$$.fragment,l),o=!1},d(l){f(t,l)}}}function ba(k){let t,o='<code>Trainer</code>를 사용하여 모델을 파인 튜닝하는 방법에 익숙하지 않은 경우, <a href="../training#train-with-pytorch-trainer">여기</a>에서 기본 튜토리얼을 확인하세요!';return{c(){t=g("p"),t.innerHTML=o},l(l){t=w(l,"P",{"data-svelte-h":!0}),J(t)!=="svelte-j16k5v"&&(t.innerHTML=o)},m(l,m){e(l,t,m)},p:is,d(l){l&&a(t)}}}function ga(k){let t,o,l,m="이제 모델을 훈련시킬 준비가 되었습니다! <code>AutoModelForSequenceClassification</code>로 DistilBERT를 가져오고 예상되는 레이블 수와 레이블 매핑을 지정하세요:",b,C,W,v,I="이제 세 단계만 거치면 끝입니다:",B,x,E="<li><code>TrainingArguments</code>에서 하이퍼파라미터를 정의하세요. <code>output_dir</code>는 모델을 저장할 위치를 지정하는 유일한 매개변수입니다. 이 모델을 허브에 업로드하기 위해 <code>push_to_hub=True</code>를 설정합니다(모델을 업로드하기 위해 Hugging Face에 로그인해야합니다.) 각 에폭이 끝날 때마다, <code>Trainer</code>는 seqeval 점수를 평가하고 훈련 체크포인트를 저장합니다.</li> <li><code>Trainer</code>에 훈련 인수와 모델, 데이터 세트, 토크나이저, 데이터 콜레이터 및 <code>compute_metrics</code> 함수를 전달하세요.</li> <li><code>train()</code>를 호출하여 모델을 파인 튜닝하세요.</li>",G,$,R,r,U="훈련이 완료되면, <code>push_to_hub()</code> 메소드를 사용하여 모델을 허브에 공유할 수 있습니다.",V,X,A;return t=new vl({props:{$$slots:{default:[ba]},$$scope:{ctx:k}}}),C=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24lMkMlMjBUcmFpbmluZ0FyZ3VtZW50cyUyQyUyMFRyYWluZXIlMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUwQSUyMCUyMCUyMCUyMCUyMmRpc3RpbGJlcnQtYmFzZS11bmNhc2VkJTIyJTJDJTIwbnVtX2xhYmVscyUzRDEzJTJDJTIwaWQybGFiZWwlM0RpZDJsYWJlbCUyQyUyMGxhYmVsMmlkJTNEbGFiZWwyaWQlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForTokenClassification, TrainingArguments, Trainer

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForTokenClassification.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">13</span>, id2label=id2label, label2id=label2id
<span class="hljs-meta">... </span>)`,wrap:!1}}),$=new _({props:{code:"dHJhaW5pbmdfYXJncyUyMCUzRCUyMFRyYWluaW5nQXJndW1lbnRzKCUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0QlMjJteV9hd2Vzb21lX3dudXRfbW9kZWwlMjIlMkMlMEElMjAlMjAlMjAlMjBsZWFybmluZ19yYXRlJTNEMmUtNSUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDE2JTJDJTBBJTIwJTIwJTIwJTIwcGVyX2RldmljZV9ldmFsX2JhdGNoX3NpemUlM0QxNiUyQyUwQSUyMCUyMCUyMCUyMG51bV90cmFpbl9lcG9jaHMlM0QyJTJDJTBBJTIwJTIwJTIwJTIwd2VpZ2h0X2RlY2F5JTNEMC4wMSUyQyUwQSUyMCUyMCUyMCUyMGV2YWx1YXRpb25fc3RyYXRlZ3klM0QlMjJlcG9jaCUyMiUyQyUwQSUyMCUyMCUyMCUyMHNhdmVfc3RyYXRlZ3klM0QlMjJlcG9jaCUyMiUyQyUwQSUyMCUyMCUyMCUyMGxvYWRfYmVzdF9tb2RlbF9hdF9lbmQlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwcHVzaF90b19odWIlM0RUcnVlJTJDJTBBKSUwQSUwQXRyYWluZXIlMjAlM0QlMjBUcmFpbmVyKCUwQSUyMCUyMCUyMCUyMG1vZGVsJTNEbW9kZWwlMkMlMEElMjAlMjAlMjAlMjBhcmdzJTNEdHJhaW5pbmdfYXJncyUyQyUwQSUyMCUyMCUyMCUyMHRyYWluX2RhdGFzZXQlM0R0b2tlbml6ZWRfd251dCU1QiUyMnRyYWluJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZXZhbF9kYXRhc2V0JTNEdG9rZW5pemVkX3dudXQlNUIlMjJ0ZXN0JTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwdG9rZW5pemVyJTNEdG9rZW5pemVyJTJDJTBBJTIwJTIwJTIwJTIwZGF0YV9jb2xsYXRvciUzRGRhdGFfY29sbGF0b3IlMkMlMEElMjAlMjAlMjAlMjBjb21wdXRlX21ldHJpY3MlM0Rjb21wdXRlX21ldHJpY3MlMkMlMEEpJTBBJTBBdHJhaW5lci50cmFpbigp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>training_args = TrainingArguments(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;my_awesome_wnut_model&quot;</span>,
<span class="hljs-meta">... </span>    learning_rate=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    per_device_train_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    per_device_eval_batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    num_train_epochs=<span class="hljs-number">2</span>,
<span class="hljs-meta">... </span>    weight_decay=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>    evaluation_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    save_strategy=<span class="hljs-string">&quot;epoch&quot;</span>,
<span class="hljs-meta">... </span>    load_best_model_at_end=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    push_to_hub=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer = Trainer(
<span class="hljs-meta">... </span>    model=model,
<span class="hljs-meta">... </span>    args=training_args,
<span class="hljs-meta">... </span>    train_dataset=tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    eval_dataset=tokenized_wnut[<span class="hljs-string">&quot;test&quot;</span>],
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>    data_collator=data_collator,
<span class="hljs-meta">... </span>    compute_metrics=compute_metrics,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.train()`,wrap:!1}}),X=new _({props:{code:"dHJhaW5lci5wdXNoX3RvX2h1Yigp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>trainer.push_to_hub()',wrap:!1}}),{c(){j(t.$$.fragment),o=c(),l=g("p"),l.innerHTML=m,b=c(),j(C.$$.fragment),W=c(),v=g("p"),v.textContent=I,B=c(),x=g("ol"),x.innerHTML=E,G=c(),j($.$$.fragment),R=c(),r=g("p"),r.innerHTML=U,V=c(),j(X.$$.fragment)},l(M){h(t.$$.fragment,M),o=i(M),l=w(M,"P",{"data-svelte-h":!0}),J(l)!=="svelte-155uoos"&&(l.innerHTML=m),b=i(M),h(C.$$.fragment,M),W=i(M),v=w(M,"P",{"data-svelte-h":!0}),J(v)!=="svelte-14zzcxs"&&(v.textContent=I),B=i(M),x=w(M,"OL",{"data-svelte-h":!0}),J(x)!=="svelte-yqkkyp"&&(x.innerHTML=E),G=i(M),h($.$$.fragment,M),R=i(M),r=w(M,"P",{"data-svelte-h":!0}),J(r)!=="svelte-ydcgmn"&&(r.innerHTML=U),V=i(M),h(X.$$.fragment,M)},m(M,Z){d(t,M,Z),e(M,o,Z),e(M,l,Z),e(M,b,Z),d(C,M,Z),e(M,W,Z),e(M,v,Z),e(M,B,Z),e(M,x,Z),e(M,G,Z),d($,M,Z),e(M,R,Z),e(M,r,Z),e(M,V,Z),d(X,M,Z),A=!0},p(M,Z){const F={};Z&2&&(F.$$scope={dirty:Z,ctx:M}),t.$set(F)},i(M){A||(u(t.$$.fragment,M),u(C.$$.fragment,M),u($.$$.fragment,M),u(X.$$.fragment,M),A=!0)},o(M){y(t.$$.fragment,M),y(C.$$.fragment,M),y($.$$.fragment,M),y(X.$$.fragment,M),A=!1},d(M){M&&(a(o),a(l),a(b),a(W),a(v),a(B),a(x),a(G),a(R),a(r),a(V)),f(t,M),f(C,M),f($,M),f(X,M)}}}function wa(k){let t,o;return t=new Ss({props:{$$slots:{default:[ga]},$$scope:{ctx:k}}}),{c(){j(t.$$.fragment)},l(l){h(t.$$.fragment,l)},m(l,m){d(t,l,m),o=!0},p(l,m){const b={};m&2&&(b.$$scope={dirty:m,ctx:l}),t.$set(b)},i(l){o||(u(t.$$.fragment,l),o=!0)},o(l){y(t.$$.fragment,l),o=!1},d(l){f(t,l)}}}function Ja(k){let t,o='Keras를 사용하여 모델을 파인 튜닝하는 방법에 익숙하지 않은 경우, <a href="../training#train-a-tensorflow-model-with-keras">여기</a>의 기본 튜토리얼을 확인하세요!';return{c(){t=g("p"),t.innerHTML=o},l(l){t=w(l,"P",{"data-svelte-h":!0}),J(t)!=="svelte-nkj0lu"&&(t.innerHTML=o)},m(l,m){e(l,t,m)},p:is,d(l){l&&a(t)}}}function Ta(k){let t,o,l,m,b,C="그런 다음 <code>TFAutoModelForSequenceClassification</code>을 사용하여 DistilBERT를 가져오고, 예상되는 레이블 수와 레이블 매핑을 지정합니다:",W,v,I,B,x="<code>prepare_tf_dataset()</code>을 사용하여 데이터 세트를 <code>tf.data.Dataset</code> 형식으로 변환합니다:",E,G,$,R,r='<a href="https://keras.io/api/models/model_training_apis/#compile-method" rel="nofollow"><code>compile</code></a>를 사용하여 훈련할 모델을 구성합니다:',U,V,X,A,M='훈련을 시작하기 전에 설정해야할 마지막 두 가지는 예측에서 seqeval 점수를 계산하고, 모델을 허브에 업로드할 방법을 제공하는 것입니다. 모두 <a href="../main_classes/keras_callbacks">Keras callbacks</a>를 사용하여 수행됩니다.',Z,F,os="<code>KerasMetricCallback</code>에 <code>compute_metrics</code> 함수를 전달하세요:",H,N,z,ls,ms="<code>PushToHubCallback</code>에서 모델과 토크나이저를 업로드할 위치를 지정합니다:",Q,q,Y,S,as="그런 다음 콜백을 함께 묶습니다:",Ms,L,D,P,ts='드디어, 모델 훈련을 시작할 준비가 되었습니다! <a href="https://keras.io/api/models/model_training_apis/#fit-method" rel="nofollow"><code>fit</code></a>에 훈련 데이터 세트, 검증 데이터 세트, 에폭의 수 및 콜백을 전달하여 파인 튜닝합니다:',js,K,O,ss,es="훈련이 완료되면, 모델이 자동으로 허브에 업로드되어 누구나 사용할 수 있습니다!",hs;return t=new vl({props:{$$slots:{default:[Ja]},$$scope:{ctx:k}}}),l=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMGNyZWF0ZV9vcHRpbWl6ZXIlMEElMEFiYXRjaF9zaXplJTIwJTNEJTIwMTYlMEFudW1fdHJhaW5fZXBvY2hzJTIwJTNEJTIwMyUwQW51bV90cmFpbl9zdGVwcyUyMCUzRCUyMChsZW4odG9rZW5pemVkX3dudXQlNUIlMjJ0cmFpbiUyMiU1RCklMjAlMkYlMkYlMjBiYXRjaF9zaXplKSUyMColMjBudW1fdHJhaW5fZXBvY2hzJTBBb3B0aW1pemVyJTJDJTIwbHJfc2NoZWR1bGUlMjAlM0QlMjBjcmVhdGVfb3B0aW1pemVyKCUwQSUyMCUyMCUyMCUyMGluaXRfbHIlM0QyZS01JTJDJTBBJTIwJTIwJTIwJTIwbnVtX3RyYWluX3N0ZXBzJTNEbnVtX3RyYWluX3N0ZXBzJTJDJTBBJTIwJTIwJTIwJTIwd2VpZ2h0X2RlY2F5X3JhdGUlM0QwLjAxJTJDJTBBJTIwJTIwJTIwJTIwbnVtX3dhcm11cF9zdGVwcyUzRDAlMkMlMEEp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> create_optimizer

<span class="hljs-meta">&gt;&gt;&gt; </span>batch_size = <span class="hljs-number">16</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_train_epochs = <span class="hljs-number">3</span>
<span class="hljs-meta">&gt;&gt;&gt; </span>num_train_steps = (<span class="hljs-built_in">len</span>(tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>]) // batch_size) * num_train_epochs
<span class="hljs-meta">&gt;&gt;&gt; </span>optimizer, lr_schedule = create_optimizer(
<span class="hljs-meta">... </span>    init_lr=<span class="hljs-number">2e-5</span>,
<span class="hljs-meta">... </span>    num_train_steps=num_train_steps,
<span class="hljs-meta">... </span>    weight_decay_rate=<span class="hljs-number">0.01</span>,
<span class="hljs-meta">... </span>    num_warmup_steps=<span class="hljs-number">0</span>,
<span class="hljs-meta">... </span>)`,wrap:!1}}),v=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsRm9yVG9rZW5DbGFzc2lmaWNhdGlvbiUwQSUwQW1vZGVsJTIwJTNEJTIwVEZBdXRvTW9kZWxGb3JUb2tlbkNsYXNzaWZpY2F0aW9uLmZyb21fcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZCUyMiUyQyUyMG51bV9sYWJlbHMlM0QxMyUyQyUyMGlkMmxhYmVsJTNEaWQybGFiZWwlMkMlMjBsYWJlbDJpZCUzRGxhYmVsMmlkJTBBKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForTokenClassification.from_pretrained(
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>, num_labels=<span class="hljs-number">13</span>, id2label=id2label, label2id=label2id
<span class="hljs-meta">... </span>)`,wrap:!1}}),G=new _({props:{code:"dGZfdHJhaW5fc2V0JTIwJTNEJTIwbW9kZWwucHJlcGFyZV90Zl9kYXRhc2V0KCUwQSUyMCUyMCUyMCUyMHRva2VuaXplZF93bnV0JTVCJTIydHJhaW4lMjIlNUQlMkMlMEElMjAlMjAlMjAlMjBzaHVmZmxlJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMGJhdGNoX3NpemUlM0QxNiUyQyUwQSUyMCUyMCUyMCUyMGNvbGxhdGVfZm4lM0RkYXRhX2NvbGxhdG9yJTJDJTBBKSUwQSUwQXRmX3ZhbGlkYXRpb25fc2V0JTIwJTNEJTIwbW9kZWwucHJlcGFyZV90Zl9kYXRhc2V0KCUwQSUyMCUyMCUyMCUyMHRva2VuaXplZF93bnV0JTVCJTIydmFsaWRhdGlvbiUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMHNodWZmbGUlM0RGYWxzZSUyQyUwQSUyMCUyMCUyMCUyMGJhdGNoX3NpemUlM0QxNiUyQyUwQSUyMCUyMCUyMCUyMGNvbGxhdGVfZm4lM0RkYXRhX2NvbGxhdG9yJTJDJTBBKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>tf_train_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    tokenized_wnut[<span class="hljs-string">&quot;train&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">True</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)

<span class="hljs-meta">&gt;&gt;&gt; </span>tf_validation_set = model.prepare_tf_dataset(
<span class="hljs-meta">... </span>    tokenized_wnut[<span class="hljs-string">&quot;validation&quot;</span>],
<span class="hljs-meta">... </span>    shuffle=<span class="hljs-literal">False</span>,
<span class="hljs-meta">... </span>    batch_size=<span class="hljs-number">16</span>,
<span class="hljs-meta">... </span>    collate_fn=data_collator,
<span class="hljs-meta">... </span>)`,wrap:!1}}),V=new _({props:{code:"aW1wb3J0JTIwdGVuc29yZmxvdyUyMGFzJTIwdGYlMEElMEFtb2RlbC5jb21waWxlKG9wdGltaXplciUzRG9wdGltaXplcik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> tensorflow <span class="hljs-keyword">as</span> tf

<span class="hljs-meta">&gt;&gt;&gt; </span>model.<span class="hljs-built_in">compile</span>(optimizer=optimizer)`,wrap:!1}}),N=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5rZXJhc19jYWxsYmFja3MlMjBpbXBvcnQlMjBLZXJhc01ldHJpY0NhbGxiYWNrJTBBJTBBbWV0cmljX2NhbGxiYWNrJTIwJTNEJTIwS2VyYXNNZXRyaWNDYWxsYmFjayhtZXRyaWNfZm4lM0Rjb21wdXRlX21ldHJpY3MlMkMlMjBldmFsX2RhdGFzZXQlM0R0Zl92YWxpZGF0aW9uX3NldCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> KerasMetricCallback

<span class="hljs-meta">&gt;&gt;&gt; </span>metric_callback = KerasMetricCallback(metric_fn=compute_metrics, eval_dataset=tf_validation_set)`,wrap:!1}}),q=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5rZXJhc19jYWxsYmFja3MlMjBpbXBvcnQlMjBQdXNoVG9IdWJDYWxsYmFjayUwQSUwQXB1c2hfdG9faHViX2NhbGxiYWNrJTIwJTNEJTIwUHVzaFRvSHViQ2FsbGJhY2soJTBBJTIwJTIwJTIwJTIwb3V0cHV0X2RpciUzRCUyMm15X2F3ZXNvbWVfd251dF9tb2RlbCUyMiUyQyUwQSUyMCUyMCUyMCUyMHRva2VuaXplciUzRHRva2VuaXplciUyQyUwQSk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers.keras_callbacks <span class="hljs-keyword">import</span> PushToHubCallback

<span class="hljs-meta">&gt;&gt;&gt; </span>push_to_hub_callback = PushToHubCallback(
<span class="hljs-meta">... </span>    output_dir=<span class="hljs-string">&quot;my_awesome_wnut_model&quot;</span>,
<span class="hljs-meta">... </span>    tokenizer=tokenizer,
<span class="hljs-meta">... </span>)`,wrap:!1}}),L=new _({props:{code:"Y2FsbGJhY2tzJTIwJTNEJTIwJTVCbWV0cmljX2NhbGxiYWNrJTJDJTIwcHVzaF90b19odWJfY2FsbGJhY2slNUQ=",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>callbacks = [metric_callback, push_to_hub_callback]',wrap:!1}}),K=new _({props:{code:"bW9kZWwuZml0KHglM0R0Zl90cmFpbl9zZXQlMkMlMjB2YWxpZGF0aW9uX2RhdGElM0R0Zl92YWxpZGF0aW9uX3NldCUyQyUyMGVwb2NocyUzRDMlMkMlMjBjYWxsYmFja3MlM0RjYWxsYmFja3Mp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>model.fit(x=tf_train_set, validation_data=tf_validation_set, epochs=<span class="hljs-number">3</span>, callbacks=callbacks)',wrap:!1}}),{c(){j(t.$$.fragment),o=na(`
TensorFlow에서 모델을 파인 튜닝하려면, 먼저 옵티마이저 함수와 학습률 스케쥴, 그리고 일부 훈련 하이퍼파라미터를 설정해야 합니다:

	`),j(l.$$.fragment),m=c(),b=g("p"),b.innerHTML=C,W=c(),j(v.$$.fragment),I=c(),B=g("p"),B.innerHTML=x,E=c(),j(G.$$.fragment),$=c(),R=g("p"),R.innerHTML=r,U=c(),j(V.$$.fragment),X=c(),A=g("p"),A.innerHTML=M,Z=c(),F=g("p"),F.innerHTML=os,H=c(),j(N.$$.fragment),z=c(),ls=g("p"),ls.innerHTML=ms,Q=c(),j(q.$$.fragment),Y=c(),S=g("p"),S.textContent=as,Ms=c(),j(L.$$.fragment),D=c(),P=g("p"),P.innerHTML=ts,js=c(),j(K.$$.fragment),O=c(),ss=g("p"),ss.textContent=es},l(p){h(t.$$.fragment,p),o=pa(p,`
TensorFlow에서 모델을 파인 튜닝하려면, 먼저 옵티마이저 함수와 학습률 스케쥴, 그리고 일부 훈련 하이퍼파라미터를 설정해야 합니다:

	`),h(l.$$.fragment,p),m=i(p),b=w(p,"P",{"data-svelte-h":!0}),J(b)!=="svelte-1opo5t0"&&(b.innerHTML=C),W=i(p),h(v.$$.fragment,p),I=i(p),B=w(p,"P",{"data-svelte-h":!0}),J(B)!=="svelte-txv5rs"&&(B.innerHTML=x),E=i(p),h(G.$$.fragment,p),$=i(p),R=w(p,"P",{"data-svelte-h":!0}),J(R)!=="svelte-qrlpiv"&&(R.innerHTML=r),U=i(p),h(V.$$.fragment,p),X=i(p),A=w(p,"P",{"data-svelte-h":!0}),J(A)!=="svelte-10vlwlw"&&(A.innerHTML=M),Z=i(p),F=w(p,"P",{"data-svelte-h":!0}),J(F)!=="svelte-1fwgit4"&&(F.innerHTML=os),H=i(p),h(N.$$.fragment,p),z=i(p),ls=w(p,"P",{"data-svelte-h":!0}),J(ls)!=="svelte-1ilfuc9"&&(ls.innerHTML=ms),Q=i(p),h(q.$$.fragment,p),Y=i(p),S=w(p,"P",{"data-svelte-h":!0}),J(S)!=="svelte-90s2we"&&(S.textContent=as),Ms=i(p),h(L.$$.fragment,p),D=i(p),P=w(p,"P",{"data-svelte-h":!0}),J(P)!=="svelte-1kbr19n"&&(P.innerHTML=ts),js=i(p),h(K.$$.fragment,p),O=i(p),ss=w(p,"P",{"data-svelte-h":!0}),J(ss)!=="svelte-w14up1"&&(ss.textContent=es)},m(p,T){d(t,p,T),e(p,o,T),d(l,p,T),e(p,m,T),e(p,b,T),e(p,W,T),d(v,p,T),e(p,I,T),e(p,B,T),e(p,E,T),d(G,p,T),e(p,$,T),e(p,R,T),e(p,U,T),d(V,p,T),e(p,X,T),e(p,A,T),e(p,Z,T),e(p,F,T),e(p,H,T),d(N,p,T),e(p,z,T),e(p,ls,T),e(p,Q,T),d(q,p,T),e(p,Y,T),e(p,S,T),e(p,Ms,T),d(L,p,T),e(p,D,T),e(p,P,T),e(p,js,T),d(K,p,T),e(p,O,T),e(p,ss,T),hs=!0},p(p,T){const ds={};T&2&&(ds.$$scope={dirty:T,ctx:p}),t.$set(ds)},i(p){hs||(u(t.$$.fragment,p),u(l.$$.fragment,p),u(v.$$.fragment,p),u(G.$$.fragment,p),u(V.$$.fragment,p),u(N.$$.fragment,p),u(q.$$.fragment,p),u(L.$$.fragment,p),u(K.$$.fragment,p),hs=!0)},o(p){y(t.$$.fragment,p),y(l.$$.fragment,p),y(v.$$.fragment,p),y(G.$$.fragment,p),y(V.$$.fragment,p),y(N.$$.fragment,p),y(q.$$.fragment,p),y(L.$$.fragment,p),y(K.$$.fragment,p),hs=!1},d(p){p&&(a(o),a(m),a(b),a(W),a(I),a(B),a(E),a($),a(R),a(U),a(X),a(A),a(Z),a(F),a(H),a(z),a(ls),a(Q),a(Y),a(S),a(Ms),a(D),a(P),a(js),a(O),a(ss)),f(t,p),f(l,p),f(v,p),f(G,p),f(V,p),f(N,p),f(q,p),f(L,p),f(K,p)}}}function Ua(k){let t,o;return t=new Ss({props:{$$slots:{default:[Ta]},$$scope:{ctx:k}}}),{c(){j(t.$$.fragment)},l(l){h(t.$$.fragment,l)},m(l,m){d(t,l,m),o=!0},p(l,m){const b={};m&2&&(b.$$scope={dirty:m,ctx:l}),t.$set(b)},i(l){o||(u(t.$$.fragment,l),o=!0)},o(l){y(t.$$.fragment,l),o=!1},d(l){f(t,l)}}}function $a(k){let t,o=`토큰 분류를 위한 모델을 파인 튜닝하는 자세한 예제는 다음
<a href="https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification.ipynb" rel="nofollow">PyTorch notebook</a>
또는 <a href="https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/token_classification-tf.ipynb" rel="nofollow">TensorFlow notebook</a>를 참조하세요.`;return{c(){t=g("p"),t.innerHTML=o},l(l){t=w(l,"P",{"data-svelte-h":!0}),J(t)!=="svelte-139pooy"&&(t.innerHTML=o)},m(l,m){e(l,t,m)},p:is,d(l){l&&a(t)}}}function xa(k){let t,o="텍스트를 토큰화하고 PyTorch 텐서를 반환합니다:",l,m,b,C,W="입력을 모델에 전달하고 <code>logits</code>을 반환합니다:",v,I,B,x,E="가장 높은 확률을 가진 클래스를 모델의 <code>id2label</code> 매핑을 사용하여 텍스트 레이블로 변환합니다:",G,$,R;return m=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJzdGV2aGxpdSUyRm15X2F3ZXNvbWVfd251dF9tb2RlbCUyMiklMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIodGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIycHQlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;stevhliu/my_awesome_wnut_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(text, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)`,wrap:!1}}),I=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24lMEElMEFtb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbEZvclRva2VuQ2xhc3NpZmljYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMnN0ZXZobGl1JTJGbXlfYXdlc29tZV93bnV0X21vZGVsJTIyKSUwQXdpdGglMjB0b3JjaC5ub19ncmFkKCklM0ElMEElMjAlMjAlMjAlMjBsb2dpdHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cykubG9naXRz",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoModelForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = AutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;stevhliu/my_awesome_wnut_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">with</span> torch.no_grad():
<span class="hljs-meta">... </span>    logits = model(**inputs).logits`,wrap:!1}}),$=new _({props:{code:"cHJlZGljdGlvbnMlMjAlM0QlMjB0b3JjaC5hcmdtYXgobG9naXRzJTJDJTIwZGltJTNEMiklMEFwcmVkaWN0ZWRfdG9rZW5fY2xhc3MlMjAlM0QlMjAlNUJtb2RlbC5jb25maWcuaWQybGFiZWwlNUJ0Lml0ZW0oKSU1RCUyMGZvciUyMHQlMjBpbiUyMHByZWRpY3Rpb25zJTVCMCU1RCU1RCUwQXByZWRpY3RlZF90b2tlbl9jbGFzcw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>predictions = torch.argmax(logits, dim=<span class="hljs-number">2</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>predicted_token_class = [model.config.id2label[t.item()] <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> predictions[<span class="hljs-number">0</span>]]
<span class="hljs-meta">&gt;&gt;&gt; </span>predicted_token_class
[<span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;I-location&#x27;</span>,
 <span class="hljs-string">&#x27;B-group&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>]`,wrap:!1}}),{c(){t=g("p"),t.textContent=o,l=c(),j(m.$$.fragment),b=c(),C=g("p"),C.innerHTML=W,v=c(),j(I.$$.fragment),B=c(),x=g("p"),x.innerHTML=E,G=c(),j($.$$.fragment)},l(r){t=w(r,"P",{"data-svelte-h":!0}),J(t)!=="svelte-ctuaol"&&(t.textContent=o),l=i(r),h(m.$$.fragment,r),b=i(r),C=w(r,"P",{"data-svelte-h":!0}),J(C)!=="svelte-1hjuppo"&&(C.innerHTML=W),v=i(r),h(I.$$.fragment,r),B=i(r),x=w(r,"P",{"data-svelte-h":!0}),J(x)!=="svelte-1jbp04u"&&(x.innerHTML=E),G=i(r),h($.$$.fragment,r)},m(r,U){e(r,t,U),e(r,l,U),d(m,r,U),e(r,b,U),e(r,C,U),e(r,v,U),d(I,r,U),e(r,B,U),e(r,x,U),e(r,G,U),d($,r,U),R=!0},p:is,i(r){R||(u(m.$$.fragment,r),u(I.$$.fragment,r),u($.$$.fragment,r),R=!0)},o(r){y(m.$$.fragment,r),y(I.$$.fragment,r),y($.$$.fragment,r),R=!1},d(r){r&&(a(t),a(l),a(b),a(C),a(v),a(B),a(x),a(G)),f(m,r),f(I,r),f($,r)}}}function _a(k){let t,o;return t=new Ss({props:{$$slots:{default:[xa]},$$scope:{ctx:k}}}),{c(){j(t.$$.fragment)},l(l){h(t.$$.fragment,l)},m(l,m){d(t,l,m),o=!0},p(l,m){const b={};m&2&&(b.$$scope={dirty:m,ctx:l}),t.$set(b)},i(l){o||(u(t.$$.fragment,l),o=!0)},o(l){y(t.$$.fragment,l),o=!1},d(l){f(t,l)}}}function ka(k){let t,o="텍스트를 토큰화하고 TensorFlow 텐서를 반환합니다:",l,m,b,C,W="입력값을 모델에 전달하고 <code>logits</code>을 반환합니다:",v,I,B,x,E="가장 높은 확률을 가진 클래스를 모델의 <code>id2label</code> 매핑을 사용하여 텍스트 레이블로 변환합니다:",G,$,R;return m=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJzdGV2aGxpdSUyRm15X2F3ZXNvbWVfd251dF9tb2RlbCUyMiklMEFpbnB1dHMlMjAlM0QlMjB0b2tlbml6ZXIodGV4dCUyQyUyMHJldHVybl90ZW5zb3JzJTNEJTIydGYlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;stevhliu/my_awesome_wnut_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>inputs = tokenizer(text, return_tensors=<span class="hljs-string">&quot;tf&quot;</span>)`,wrap:!1}}),I=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMFRGQXV0b01vZGVsRm9yVG9rZW5DbGFzc2lmaWNhdGlvbiUwQSUwQW1vZGVsJTIwJTNEJTIwVEZBdXRvTW9kZWxGb3JUb2tlbkNsYXNzaWZpY2F0aW9uLmZyb21fcHJldHJhaW5lZCglMjJzdGV2aGxpdSUyRm15X2F3ZXNvbWVfd251dF9tb2RlbCUyMiklMEFsb2dpdHMlMjAlM0QlMjBtb2RlbCgqKmlucHV0cykubG9naXRz",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> TFAutoModelForTokenClassification

<span class="hljs-meta">&gt;&gt;&gt; </span>model = TFAutoModelForTokenClassification.from_pretrained(<span class="hljs-string">&quot;stevhliu/my_awesome_wnut_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>logits = model(**inputs).logits`,wrap:!1}}),$=new _({props:{code:"cHJlZGljdGVkX3Rva2VuX2NsYXNzX2lkcyUyMCUzRCUyMHRmLm1hdGguYXJnbWF4KGxvZ2l0cyUyQyUyMGF4aXMlM0QtMSklMEFwcmVkaWN0ZWRfdG9rZW5fY2xhc3MlMjAlM0QlMjAlNUJtb2RlbC5jb25maWcuaWQybGFiZWwlNUJ0JTVEJTIwZm9yJTIwdCUyMGluJTIwcHJlZGljdGVkX3Rva2VuX2NsYXNzX2lkcyU1QjAlNUQubnVtcHkoKS50b2xpc3QoKSU1RCUwQXByZWRpY3RlZF90b2tlbl9jbGFzcw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>predicted_token_class_ids = tf.math.argmax(logits, axis=-<span class="hljs-number">1</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>predicted_token_class = [model.config.id2label[t] <span class="hljs-keyword">for</span> t <span class="hljs-keyword">in</span> predicted_token_class_ids[<span class="hljs-number">0</span>].numpy().tolist()]
<span class="hljs-meta">&gt;&gt;&gt; </span>predicted_token_class
[<span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;I-location&#x27;</span>,
 <span class="hljs-string">&#x27;B-group&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;B-location&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>,
 <span class="hljs-string">&#x27;O&#x27;</span>]`,wrap:!1}}),{c(){t=g("p"),t.textContent=o,l=c(),j(m.$$.fragment),b=c(),C=g("p"),C.innerHTML=W,v=c(),j(I.$$.fragment),B=c(),x=g("p"),x.innerHTML=E,G=c(),j($.$$.fragment)},l(r){t=w(r,"P",{"data-svelte-h":!0}),J(t)!=="svelte-1m12ipr"&&(t.textContent=o),l=i(r),h(m.$$.fragment,r),b=i(r),C=w(r,"P",{"data-svelte-h":!0}),J(C)!=="svelte-tcnp5y"&&(C.innerHTML=W),v=i(r),h(I.$$.fragment,r),B=i(r),x=w(r,"P",{"data-svelte-h":!0}),J(x)!=="svelte-1jbp04u"&&(x.innerHTML=E),G=i(r),h($.$$.fragment,r)},m(r,U){e(r,t,U),e(r,l,U),d(m,r,U),e(r,b,U),e(r,C,U),e(r,v,U),d(I,r,U),e(r,B,U),e(r,x,U),e(r,G,U),d($,r,U),R=!0},p:is,i(r){R||(u(m.$$.fragment,r),u(I.$$.fragment,r),u($.$$.fragment,r),R=!0)},o(r){y(m.$$.fragment,r),y(I.$$.fragment,r),y($.$$.fragment,r),R=!1},d(r){r&&(a(t),a(l),a(b),a(C),a(v),a(B),a(x),a(G)),f(m,r),f(I,r),f($,r)}}}function Ca(k){let t,o;return t=new Ss({props:{$$slots:{default:[ka]},$$scope:{ctx:k}}}),{c(){j(t.$$.fragment)},l(l){h(t.$$.fragment,l)},m(l,m){d(t,l,m),o=!0},p(l,m){const b={};m&2&&(b.$$scope={dirty:m,ctx:l}),t.$set(b)},i(l){o||(u(t.$$.fragment,l),o=!0)},o(l){y(t.$$.fragment,l),o=!1},d(l){f(t,l)}}}function Ia(k){let t,o,l,m,b,C,W,v,I,B,x,E="토큰 분류는 문장의 개별 토큰에 레이블을 할당합니다. 가장 일반적인 토큰 분류 작업 중 하나는 개체명 인식(Named Entity Recognition, NER)입니다. 개체명 인식은 문장에서 사람, 위치 또는 조직과 같은 각 개체의 레이블을 찾으려고 시도합니다.",G,$,R="이 가이드에서 학습할 내용은:",r,U,V='<li><a href="https://huggingface.co/datasets/wnut_17" rel="nofollow">WNUT 17</a> 데이터 세트에서 <a href="https://huggingface.co/distilbert-base-uncased" rel="nofollow">DistilBERT</a>를 파인 튜닝하여 새로운 개체를 탐지합니다.</li> <li>추론을 위해 파인 튜닝 모델을 사용합니다.</li>',X,A,M,Z,F="시작하기 전에, 필요한 모든 라이브러리가 설치되어 있는지 확인하세요:",os,H,N,z,ls="Hugging Face 계정에 로그인하여 모델을 업로드하고 커뮤니티에 공유하는 것을 권장합니다. 메시지가 표시되면, 토큰을 입력하여 로그인하세요:",ms,Q,q,Y,S,as,Ms="먼저 🤗 Datasets 라이브러리에서 WNUT 17 데이터 세트를 가져옵니다:",L,D,P,ts,js="다음 예제를 살펴보세요:",K,O,ss,es,hs="<code>ner_tags</code>의 각 숫자는 개체를 나타냅니다. 숫자를 레이블 이름으로 변환하여 개체가 무엇인지 확인합니다:",p,T,ds,us,Gl="각 <code>ner_tag</code>의 앞에 붙은 문자는 개체의 토큰 위치를 나타냅니다:",Ds,ys,Rl="<li><code>B-</code>는 개체의 시작을 나타냅니다.</li> <li><code>I-</code>는 토큰이 동일한 개체 내부에 포함되어 있음을 나타냅니다(예를 들어 <code>State</code> 토큰은 <code>Empire State Building</code>와 같은 개체의 일부입니다).</li> <li><code>0</code>는 토큰이 어떤 개체에도 해당하지 않음을 나타냅니다.</li>",Ps,fs,Ks,bs,Os,gs,Wl="다음으로 <code>tokens</code> 필드를 전처리하기 위해 DistilBERT 토크나이저를 가져옵니다:",sl,ws,ll,Js,Al="위의 예제 <code>tokens</code> 필드를 보면 입력이 이미 토큰화된 것처럼 보입니다. 그러나 실제로 입력은 아직 토큰화되지 않았으므로 단어를 하위 단어로 토큰화하기 위해 <code>is_split_into_words=True</code>를 설정해야 합니다. 예제로 확인합니다:",al,Ts,tl,Us,Xl="그러나 이로 인해 <code>[CLS]</code>과 <code>[SEP]</code>라는 특수 토큰이 추가되고, 하위 단어 토큰화로 인해 입력과 레이블 간에 불일치가 발생합니다. 하나의 레이블에 해당하는 단일 단어는 이제 두 개의 하위 단어로 분할될 수 있습니다. 토큰과 레이블을 다음과 같이 재정렬해야 합니다:",el,$s,El='<li><a href="https://huggingface.co/docs/transformers/main_classes/tokenizer#transformers.BatchEncoding.word_ids" rel="nofollow"><code>word_ids</code></a> 메소드로 모든 토큰을 해당 단어에 매핑합니다.</li> <li>특수 토큰 <code>[CLS]</code>와 <code>[SEP]</code>에 <code>-100</code> 레이블을 할당하여, PyTorch 손실 함수가 해당 토큰을 무시하도록 합니다.</li> <li>주어진 단어의 첫 번째 토큰에만 레이블을 지정합니다. 같은 단어의 다른 하위 토큰에 <code>-100</code>을 할당합니다.</li>',nl,xs,Vl="다음은 토큰과 레이블을 재정렬하고 DistilBERT의 최대 입력 길이보다 길지 않도록 시퀀스를 잘라내는 함수를 만드는 방법입니다:",pl,_s,rl,ks,Fl="전체 데이터 세트에 전처리 함수를 적용하려면, 🤗 Datasets <code>map</code> 함수를 사용하세요. <code>batched=True</code>로 설정하여 데이터 세트의 여러 요소를 한 번에 처리하면 <code>map</code> 함수의 속도를 높일 수 있습니다:",cl,Cs,il,Is,Hl="이제 <code>DataCollatorWithPadding</code>를 사용하여 예제 배치를 만들어봅시다. 데이터 세트 전체를 최대 길이로 패딩하는 대신, <em>동적 패딩</em>을 사용하여 배치에서 가장 긴 길이에 맞게 문장을 패딩하는 것이 효율적입니다.",ol,ns,ml,Zs,Ml,vs,Nl='훈련 중 모델의 성능을 평가하기 위해 평가 지표를 포함하는 것이 유용합니다. 🤗 <a href="https://huggingface.co/docs/evaluate/index" rel="nofollow">Evaluate</a> 라이브러리를 사용하여 빠르게 평가 방법을 가져올 수 있습니다. 이 작업에서는 <a href="https://huggingface.co/spaces/evaluate-metric/seqeval" rel="nofollow">seqeval</a> 평가 지표를 가져옵니다. (평가 지표를 가져오고 계산하는 방법에 대해서는 🤗 Evaluate <a href="https://huggingface.co/docs/evaluate/a_quick_tour" rel="nofollow">빠른 둘러보기</a>를 참조하세요). Seqeval은 실제로 정밀도, 재현률, F1 및 정확도와 같은 여러 점수를 산출합니다.',jl,Bs,hl,Gs,zl="먼저 NER 레이블을 가져온 다음, <code>compute</code>에 실제 예측과 실제 레이블을 전달하여 점수를 계산하는 함수를 만듭니다:",dl,Rs,ul,Ws,Ql="이제 <code>compute_metrics</code> 함수를 사용할 준비가 되었으며, 훈련을 설정하면 이 함수로 되돌아올 것입니다.",yl,As,fl,Xs,ql="모델을 훈련하기 전에, <code>id2label</code>와 <code>label2id</code>를 사용하여 예상되는 id와 레이블의 맵을 생성하세요:",bl,Es,gl,ps,wl,rs,Jl,Vs,Tl,Fs,Yl="좋아요, 이제 모델을 파인 튜닝했으니 추론에 사용할 수 있습니다!",Ul,Hs,Sl="추론을 수행하고자 하는 텍스트를 가져와봅시다:",$l,Ns,xl,zs,Ll="파인 튜닝된 모델로 추론을 시도하는 가장 간단한 방법은 <code>pipeline()</code>를 사용하는 것입니다. 모델로 NER의 <code>pipeline</code>을 인스턴스화하고, 텍스트를 전달해보세요:",_l,Qs,kl,qs,Dl="원한다면, <code>pipeline</code>의 결과를 수동으로 복제할 수도 있습니다:",Cl,cs,Il,Ls,Zl;return b=new Ys({props:{title:"토큰 분류",local:"token-classification",headingTag:"h1"}}),W=new ja({props:{classNames:"absolute z-10 right-0 top-0",options:[{label:"Mixed",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/ko/token_classification.ipynb"},{label:"PyTorch",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/ko/pytorch/token_classification.ipynb"},{label:"TensorFlow",value:"https://colab.research.google.com/github/huggingface/notebooks/blob/main/transformers_doc/ko/tensorflow/token_classification.ipynb"},{label:"Mixed",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/ko/token_classification.ipynb"},{label:"PyTorch",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/ko/pytorch/token_classification.ipynb"},{label:"TensorFlow",value:"https://studiolab.sagemaker.aws/import/github/huggingface/notebooks/blob/main/transformers_doc/ko/tensorflow/token_classification.ipynb"}]}}),I=new ea({props:{id:"wVHdVlPScxA"}}),A=new vl({props:{$$slots:{default:[ha]},$$scope:{ctx:k}}}),H=new _({props:{code:"cGlwJTIwaW5zdGFsbCUyMHRyYW5zZm9ybWVycyUyMGRhdGFzZXRzJTIwZXZhbHVhdGUlMjBzZXFldmFs",highlighted:"pip install transformers datasets evaluate seqeval",wrap:!1}}),Q=new _({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMG5vdGVib29rX2xvZ2luJTBBJTBBbm90ZWJvb2tfbG9naW4oKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> notebook_login

<span class="hljs-meta">&gt;&gt;&gt; </span>notebook_login()`,wrap:!1}}),Y=new Ys({props:{title:"WNUT 17 데이터 세트 가져오기",local:"load-wnut-17-dataset",headingTag:"h2"}}),D=new _({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBJTBBd251dCUyMCUzRCUyMGxvYWRfZGF0YXNldCglMjJ3bnV0XzE3JTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset

<span class="hljs-meta">&gt;&gt;&gt; </span>wnut = load_dataset(<span class="hljs-string">&quot;wnut_17&quot;</span>)`,wrap:!1}}),O=new _({props:{code:"d251dCU1QiUyMnRyYWluJTIyJTVEJTVCMCU1RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>wnut[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
{<span class="hljs-string">&#x27;id&#x27;</span>: <span class="hljs-string">&#x27;0&#x27;</span>,
 <span class="hljs-string">&#x27;ner_tags&#x27;</span>: [<span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">8</span>, <span class="hljs-number">8</span>, <span class="hljs-number">0</span>, <span class="hljs-number">7</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>, <span class="hljs-number">0</span>],
 <span class="hljs-string">&#x27;tokens&#x27;</span>: [<span class="hljs-string">&#x27;@paulwalk&#x27;</span>, <span class="hljs-string">&#x27;It&#x27;</span>, <span class="hljs-string">&quot;&#x27;s&quot;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;I&#x27;</span>, <span class="hljs-string">&quot;&#x27;m&quot;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Empire&#x27;</span>, <span class="hljs-string">&#x27;State&#x27;</span>, <span class="hljs-string">&#x27;Building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;ESB&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;Pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>]
}`,wrap:!1}}),T=new _({props:{code:"bGFiZWxfbGlzdCUyMCUzRCUyMHdudXQlNUIlMjJ0cmFpbiUyMiU1RC5mZWF0dXJlcyU1QmYlMjJuZXJfdGFncyUyMiU1RC5mZWF0dXJlLm5hbWVzJTBBbGFiZWxfbGlzdA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>label_list = wnut[<span class="hljs-string">&quot;train&quot;</span>].features[<span class="hljs-string">f&quot;ner_tags&quot;</span>].feature.names
<span class="hljs-meta">&gt;&gt;&gt; </span>label_list
[
    <span class="hljs-string">&quot;O&quot;</span>,
    <span class="hljs-string">&quot;B-corporation&quot;</span>,
    <span class="hljs-string">&quot;I-corporation&quot;</span>,
    <span class="hljs-string">&quot;B-creative-work&quot;</span>,
    <span class="hljs-string">&quot;I-creative-work&quot;</span>,
    <span class="hljs-string">&quot;B-group&quot;</span>,
    <span class="hljs-string">&quot;I-group&quot;</span>,
    <span class="hljs-string">&quot;B-location&quot;</span>,
    <span class="hljs-string">&quot;I-location&quot;</span>,
    <span class="hljs-string">&quot;B-person&quot;</span>,
    <span class="hljs-string">&quot;I-person&quot;</span>,
    <span class="hljs-string">&quot;B-product&quot;</span>,
    <span class="hljs-string">&quot;I-product&quot;</span>,
]`,wrap:!1}}),fs=new Ys({props:{title:"전처리",local:"preprocess",headingTag:"h2"}}),bs=new ea({props:{id:"iY2AZYdZAr0"}}),ws=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Ub2tlbml6ZXIlMEElMEF0b2tlbml6ZXIlMjAlM0QlMjBBdXRvVG9rZW5pemVyLmZyb21fcHJldHJhaW5lZCglMjJkaXN0aWxiZXJ0LWJhc2UtdW5jYXNlZCUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoTokenizer

<span class="hljs-meta">&gt;&gt;&gt; </span>tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;distilbert-base-uncased&quot;</span>)`,wrap:!1}}),Ts=new _({props:{code:"ZXhhbXBsZSUyMCUzRCUyMHdudXQlNUIlMjJ0cmFpbiUyMiU1RCU1QjAlNUQlMEF0b2tlbml6ZWRfaW5wdXQlMjAlM0QlMjB0b2tlbml6ZXIoZXhhbXBsZSU1QiUyMnRva2VucyUyMiU1RCUyQyUyMGlzX3NwbGl0X2ludG9fd29yZHMlM0RUcnVlKSUwQXRva2VucyUyMCUzRCUyMHRva2VuaXplci5jb252ZXJ0X2lkc190b190b2tlbnModG9rZW5pemVkX2lucHV0JTVCJTIyaW5wdXRfaWRzJTIyJTVEKSUwQXRva2Vucw==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>example = wnut[<span class="hljs-string">&quot;train&quot;</span>][<span class="hljs-number">0</span>]
<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_input = tokenizer(example[<span class="hljs-string">&quot;tokens&quot;</span>], is_split_into_words=<span class="hljs-literal">True</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens = tokenizer.convert_ids_to_tokens(tokenized_input[<span class="hljs-string">&quot;input_ids&quot;</span>])
<span class="hljs-meta">&gt;&gt;&gt; </span>tokens
[<span class="hljs-string">&#x27;[CLS]&#x27;</span>, <span class="hljs-string">&#x27;@&#x27;</span>, <span class="hljs-string">&#x27;paul&#x27;</span>, <span class="hljs-string">&#x27;##walk&#x27;</span>, <span class="hljs-string">&#x27;it&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;s&#x27;</span>, <span class="hljs-string">&#x27;the&#x27;</span>, <span class="hljs-string">&#x27;view&#x27;</span>, <span class="hljs-string">&#x27;from&#x27;</span>, <span class="hljs-string">&#x27;where&#x27;</span>, <span class="hljs-string">&#x27;i&#x27;</span>, <span class="hljs-string">&quot;&#x27;&quot;</span>, <span class="hljs-string">&#x27;m&#x27;</span>, <span class="hljs-string">&#x27;living&#x27;</span>, <span class="hljs-string">&#x27;for&#x27;</span>, <span class="hljs-string">&#x27;two&#x27;</span>, <span class="hljs-string">&#x27;weeks&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;empire&#x27;</span>, <span class="hljs-string">&#x27;state&#x27;</span>, <span class="hljs-string">&#x27;building&#x27;</span>, <span class="hljs-string">&#x27;=&#x27;</span>, <span class="hljs-string">&#x27;es&#x27;</span>, <span class="hljs-string">&#x27;##b&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;pretty&#x27;</span>, <span class="hljs-string">&#x27;bad&#x27;</span>, <span class="hljs-string">&#x27;storm&#x27;</span>, <span class="hljs-string">&#x27;here&#x27;</span>, <span class="hljs-string">&#x27;last&#x27;</span>, <span class="hljs-string">&#x27;evening&#x27;</span>, <span class="hljs-string">&#x27;.&#x27;</span>, <span class="hljs-string">&#x27;[SEP]&#x27;</span>]`,wrap:!1}}),_s=new _({props:{code:"ZGVmJTIwdG9rZW5pemVfYW5kX2FsaWduX2xhYmVscyhleGFtcGxlcyklM0ElMEElMjAlMjAlMjAlMjB0b2tlbml6ZWRfaW5wdXRzJTIwJTNEJTIwdG9rZW5pemVyKGV4YW1wbGVzJTVCJTIydG9rZW5zJTIyJTVEJTJDJTIwdHJ1bmNhdGlvbiUzRFRydWUlMkMlMjBpc19zcGxpdF9pbnRvX3dvcmRzJTNEVHJ1ZSklMEElMEElMjAlMjAlMjAlMjBsYWJlbHMlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjBmb3IlMjBpJTJDJTIwbGFiZWwlMjBpbiUyMGVudW1lcmF0ZShleGFtcGxlcyU1QmYlMjJuZXJfdGFncyUyMiU1RCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjB3b3JkX2lkcyUyMCUzRCUyMHRva2VuaXplZF9pbnB1dHMud29yZF9pZHMoYmF0Y2hfaW5kZXglM0RpKSUyMCUyMCUyMyUyME1hcCUyMHRva2VucyUyMHRvJTIwdGhlaXIlMjByZXNwZWN0aXZlJTIwd29yZC4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmV2aW91c193b3JkX2lkeCUyMCUzRCUyME5vbmUlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbF9pZHMlMjAlM0QlMjAlNUIlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjB3b3JkX2lkeCUyMGluJTIwd29yZF9pZHMlM0ElMjAlMjAlMjMlMjBTZXQlMjB0aGUlMjBzcGVjaWFsJTIwdG9rZW5zJTIwdG8lMjAtMTAwLiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGlmJTIwd29yZF9pZHglMjBpcyUyME5vbmUlM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbF9pZHMuYXBwZW5kKC0xMDApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxpZiUyMHdvcmRfaWR4JTIwISUzRCUyMHByZXZpb3VzX3dvcmRfaWR4JTNBJTIwJTIwJTIzJTIwT25seSUyMGxhYmVsJTIwdGhlJTIwZmlyc3QlMjB0b2tlbiUyMG9mJTIwYSUyMGdpdmVuJTIwd29yZC4lMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBsYWJlbF9pZHMuYXBwZW5kKGxhYmVsJTVCd29yZF9pZHglNUQpJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZWxzZSUzQSUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGxhYmVsX2lkcy5hcHBlbmQoLTEwMCklMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBwcmV2aW91c193b3JkX2lkeCUyMCUzRCUyMHdvcmRfaWR4JTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbGFiZWxzLmFwcGVuZChsYWJlbF9pZHMpJTBBJTBBJTIwJTIwJTIwJTIwdG9rZW5pemVkX2lucHV0cyU1QiUyMmxhYmVscyUyMiU1RCUyMCUzRCUyMGxhYmVscyUwQSUyMCUyMCUyMCUyMHJldHVybiUyMHRva2VuaXplZF9pbnB1dHM=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">tokenize_and_align_labels</span>(<span class="hljs-params">examples</span>):
<span class="hljs-meta">... </span>    tokenized_inputs = tokenizer(examples[<span class="hljs-string">&quot;tokens&quot;</span>], truncation=<span class="hljs-literal">True</span>, is_split_into_words=<span class="hljs-literal">True</span>)

<span class="hljs-meta">... </span>    labels = []
<span class="hljs-meta">... </span>    <span class="hljs-keyword">for</span> i, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(examples[<span class="hljs-string">f&quot;ner_tags&quot;</span>]):
<span class="hljs-meta">... </span>        word_ids = tokenized_inputs.word_ids(batch_index=i)  <span class="hljs-comment"># Map tokens to their respective word.</span>
<span class="hljs-meta">... </span>        previous_word_idx = <span class="hljs-literal">None</span>
<span class="hljs-meta">... </span>        label_ids = []
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> word_idx <span class="hljs-keyword">in</span> word_ids:  <span class="hljs-comment"># Set the special tokens to -100.</span>
<span class="hljs-meta">... </span>            <span class="hljs-keyword">if</span> word_idx <span class="hljs-keyword">is</span> <span class="hljs-literal">None</span>:
<span class="hljs-meta">... </span>                label_ids.append(-<span class="hljs-number">100</span>)
<span class="hljs-meta">... </span>            <span class="hljs-keyword">elif</span> word_idx != previous_word_idx:  <span class="hljs-comment"># Only label the first token of a given word.</span>
<span class="hljs-meta">... </span>                label_ids.append(label[word_idx])
<span class="hljs-meta">... </span>            <span class="hljs-keyword">else</span>:
<span class="hljs-meta">... </span>                label_ids.append(-<span class="hljs-number">100</span>)
<span class="hljs-meta">... </span>            previous_word_idx = word_idx
<span class="hljs-meta">... </span>        labels.append(label_ids)

<span class="hljs-meta">... </span>    tokenized_inputs[<span class="hljs-string">&quot;labels&quot;</span>] = labels
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> tokenized_inputs`,wrap:!1}}),Cs=new _({props:{code:"dG9rZW5pemVkX3dudXQlMjAlM0QlMjB3bnV0Lm1hcCh0b2tlbml6ZV9hbmRfYWxpZ25fbGFiZWxzJTJDJTIwYmF0Y2hlZCUzRFRydWUp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>tokenized_wnut = wnut.<span class="hljs-built_in">map</span>(tokenize_and_align_labels, batched=<span class="hljs-literal">True</span>)',wrap:!1}}),ns=new Bl({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[fa],pytorch:[ua]},$$scope:{ctx:k}}}),Zs=new Ys({props:{title:"평가",local:"evaluation",headingTag:"h2"}}),Bs=new _({props:{code:"aW1wb3J0JTIwZXZhbHVhdGUlMEElMEFzZXFldmFsJTIwJTNEJTIwZXZhbHVhdGUubG9hZCglMjJzZXFldmFsJTIyKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> evaluate

<span class="hljs-meta">&gt;&gt;&gt; </span>seqeval = evaluate.load(<span class="hljs-string">&quot;seqeval&quot;</span>)`,wrap:!1}}),Rs=new _({props:{code:"aW1wb3J0JTIwbnVtcHklMjBhcyUyMG5wJTBBJTBBbGFiZWxzJTIwJTNEJTIwJTVCbGFiZWxfbGlzdCU1QmklNUQlMjBmb3IlMjBpJTIwaW4lMjBleGFtcGxlJTVCZiUyMm5lcl90YWdzJTIyJTVEJTVEJTBBJTBBJTBBZGVmJTIwY29tcHV0ZV9tZXRyaWNzKHApJTNBJTBBJTIwJTIwJTIwJTIwcHJlZGljdGlvbnMlMkMlMjBsYWJlbHMlMjAlM0QlMjBwJTBBJTIwJTIwJTIwJTIwcHJlZGljdGlvbnMlMjAlM0QlMjBucC5hcmdtYXgocHJlZGljdGlvbnMlMkMlMjBheGlzJTNEMiklMEElMEElMjAlMjAlMjAlMjB0cnVlX3ByZWRpY3Rpb25zJTIwJTNEJTIwJTVCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTVCbGFiZWxfbGlzdCU1QnAlNUQlMjBmb3IlMjAocCUyQyUyMGwpJTIwaW4lMjB6aXAocHJlZGljdGlvbiUyQyUyMGxhYmVsKSUyMGlmJTIwbCUyMCElM0QlMjAtMTAwJTVEJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZm9yJTIwcHJlZGljdGlvbiUyQyUyMGxhYmVsJTIwaW4lMjB6aXAocHJlZGljdGlvbnMlMkMlMjBsYWJlbHMpJTBBJTIwJTIwJTIwJTIwJTVEJTBBJTIwJTIwJTIwJTIwdHJ1ZV9sYWJlbHMlMjAlM0QlMjAlNUIlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlNUJsYWJlbF9saXN0JTVCbCU1RCUyMGZvciUyMChwJTJDJTIwbCklMjBpbiUyMHppcChwcmVkaWN0aW9uJTJDJTIwbGFiZWwpJTIwaWYlMjBsJTIwISUzRCUyMC0xMDAlNUQlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBmb3IlMjBwcmVkaWN0aW9uJTJDJTIwbGFiZWwlMjBpbiUyMHppcChwcmVkaWN0aW9ucyUyQyUyMGxhYmVscyklMEElMjAlMjAlMjAlMjAlNUQlMEElMEElMjAlMjAlMjAlMjByZXN1bHRzJTIwJTNEJTIwc2VxZXZhbC5jb21wdXRlKHByZWRpY3Rpb25zJTNEdHJ1ZV9wcmVkaWN0aW9ucyUyQyUyMHJlZmVyZW5jZXMlM0R0cnVlX2xhYmVscyklMEElMjAlMjAlMjAlMjByZXR1cm4lMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJwcmVjaXNpb24lMjIlM0ElMjByZXN1bHRzJTVCJTIyb3ZlcmFsbF9wcmVjaXNpb24lMjIlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJyZWNhbGwlMjIlM0ElMjByZXN1bHRzJTVCJTIyb3ZlcmFsbF9yZWNhbGwlMjIlNUQlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJmMSUyMiUzQSUyMHJlc3VsdHMlNUIlMjJvdmVyYWxsX2YxJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyYWNjdXJhY3klMjIlM0ElMjByZXN1bHRzJTVCJTIyb3ZlcmFsbF9hY2N1cmFjeSUyMiU1RCUyQyUwQSUyMCUyMCUyMCUyMCU3RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np

<span class="hljs-meta">&gt;&gt;&gt; </span>labels = [label_list[i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> example[<span class="hljs-string">f&quot;ner_tags&quot;</span>]]


<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">def</span> <span class="hljs-title function_">compute_metrics</span>(<span class="hljs-params">p</span>):
<span class="hljs-meta">... </span>    predictions, labels = p
<span class="hljs-meta">... </span>    predictions = np.argmax(predictions, axis=<span class="hljs-number">2</span>)

<span class="hljs-meta">... </span>    true_predictions = [
<span class="hljs-meta">... </span>        [label_list[p] <span class="hljs-keyword">for</span> (p, l) <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(prediction, label) <span class="hljs-keyword">if</span> l != -<span class="hljs-number">100</span>]
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> prediction, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(predictions, labels)
<span class="hljs-meta">... </span>    ]
<span class="hljs-meta">... </span>    true_labels = [
<span class="hljs-meta">... </span>        [label_list[l] <span class="hljs-keyword">for</span> (p, l) <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(prediction, label) <span class="hljs-keyword">if</span> l != -<span class="hljs-number">100</span>]
<span class="hljs-meta">... </span>        <span class="hljs-keyword">for</span> prediction, label <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(predictions, labels)
<span class="hljs-meta">... </span>    ]

<span class="hljs-meta">... </span>    results = seqeval.compute(predictions=true_predictions, references=true_labels)
<span class="hljs-meta">... </span>    <span class="hljs-keyword">return</span> {
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;precision&quot;</span>: results[<span class="hljs-string">&quot;overall_precision&quot;</span>],
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;recall&quot;</span>: results[<span class="hljs-string">&quot;overall_recall&quot;</span>],
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;f1&quot;</span>: results[<span class="hljs-string">&quot;overall_f1&quot;</span>],
<span class="hljs-meta">... </span>        <span class="hljs-string">&quot;accuracy&quot;</span>: results[<span class="hljs-string">&quot;overall_accuracy&quot;</span>],
<span class="hljs-meta">... </span>    }`,wrap:!1}}),As=new Ys({props:{title:"훈련",local:"train",headingTag:"h2"}}),Es=new _({props:{code:"aWQybGFiZWwlMjAlM0QlMjAlN0IlMEElMjAlMjAlMjAlMjAwJTNBJTIwJTIyTyUyMiUyQyUwQSUyMCUyMCUyMCUyMDElM0ElMjAlMjJCLWNvcnBvcmF0aW9uJTIyJTJDJTBBJTIwJTIwJTIwJTIwMiUzQSUyMCUyMkktY29ycG9yYXRpb24lMjIlMkMlMEElMjAlMjAlMjAlMjAzJTNBJTIwJTIyQi1jcmVhdGl2ZS13b3JrJTIyJTJDJTBBJTIwJTIwJTIwJTIwNCUzQSUyMCUyMkktY3JlYXRpdmUtd29yayUyMiUyQyUwQSUyMCUyMCUyMCUyMDUlM0ElMjAlMjJCLWdyb3VwJTIyJTJDJTBBJTIwJTIwJTIwJTIwNiUzQSUyMCUyMkktZ3JvdXAlMjIlMkMlMEElMjAlMjAlMjAlMjA3JTNBJTIwJTIyQi1sb2NhdGlvbiUyMiUyQyUwQSUyMCUyMCUyMCUyMDglM0ElMjAlMjJJLWxvY2F0aW9uJTIyJTJDJTBBJTIwJTIwJTIwJTIwOSUzQSUyMCUyMkItcGVyc29uJTIyJTJDJTBBJTIwJTIwJTIwJTIwMTAlM0ElMjAlMjJJLXBlcnNvbiUyMiUyQyUwQSUyMCUyMCUyMCUyMDExJTNBJTIwJTIyQi1wcm9kdWN0JTIyJTJDJTBBJTIwJTIwJTIwJTIwMTIlM0ElMjAlMjJJLXByb2R1Y3QlMjIlMkMlMEElN0QlMEFsYWJlbDJpZCUyMCUzRCUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMk8lMjIlM0ElMjAwJTJDJTBBJTIwJTIwJTIwJTIwJTIyQi1jb3Jwb3JhdGlvbiUyMiUzQSUyMDElMkMlMEElMjAlMjAlMjAlMjAlMjJJLWNvcnBvcmF0aW9uJTIyJTNBJTIwMiUyQyUwQSUyMCUyMCUyMCUyMCUyMkItY3JlYXRpdmUtd29yayUyMiUzQSUyMDMlMkMlMEElMjAlMjAlMjAlMjAlMjJJLWNyZWF0aXZlLXdvcmslMjIlM0ElMjA0JTJDJTBBJTIwJTIwJTIwJTIwJTIyQi1ncm91cCUyMiUzQSUyMDUlMkMlMEElMjAlMjAlMjAlMjAlMjJJLWdyb3VwJTIyJTNBJTIwNiUyQyUwQSUyMCUyMCUyMCUyMCUyMkItbG9jYXRpb24lMjIlM0ElMjA3JTJDJTBBJTIwJTIwJTIwJTIwJTIySS1sb2NhdGlvbiUyMiUzQSUyMDglMkMlMEElMjAlMjAlMjAlMjAlMjJCLXBlcnNvbiUyMiUzQSUyMDklMkMlMEElMjAlMjAlMjAlMjAlMjJJLXBlcnNvbiUyMiUzQSUyMDEwJTJDJTBBJTIwJTIwJTIwJTIwJTIyQi1wcm9kdWN0JTIyJTNBJTIwMTElMkMlMEElMjAlMjAlMjAlMjAlMjJJLXByb2R1Y3QlMjIlM0ElMjAxMiUyQyUwQSU3RA==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>id2label = {
<span class="hljs-meta">... </span>    <span class="hljs-number">0</span>: <span class="hljs-string">&quot;O&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">1</span>: <span class="hljs-string">&quot;B-corporation&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">2</span>: <span class="hljs-string">&quot;I-corporation&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">3</span>: <span class="hljs-string">&quot;B-creative-work&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">4</span>: <span class="hljs-string">&quot;I-creative-work&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">5</span>: <span class="hljs-string">&quot;B-group&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">6</span>: <span class="hljs-string">&quot;I-group&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">7</span>: <span class="hljs-string">&quot;B-location&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">8</span>: <span class="hljs-string">&quot;I-location&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">9</span>: <span class="hljs-string">&quot;B-person&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">10</span>: <span class="hljs-string">&quot;I-person&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">11</span>: <span class="hljs-string">&quot;B-product&quot;</span>,
<span class="hljs-meta">... </span>    <span class="hljs-number">12</span>: <span class="hljs-string">&quot;I-product&quot;</span>,
<span class="hljs-meta">... </span>}
<span class="hljs-meta">&gt;&gt;&gt; </span>label2id = {
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;O&quot;</span>: <span class="hljs-number">0</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-corporation&quot;</span>: <span class="hljs-number">1</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-corporation&quot;</span>: <span class="hljs-number">2</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-creative-work&quot;</span>: <span class="hljs-number">3</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-creative-work&quot;</span>: <span class="hljs-number">4</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-group&quot;</span>: <span class="hljs-number">5</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-group&quot;</span>: <span class="hljs-number">6</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-location&quot;</span>: <span class="hljs-number">7</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-location&quot;</span>: <span class="hljs-number">8</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-person&quot;</span>: <span class="hljs-number">9</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-person&quot;</span>: <span class="hljs-number">10</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;B-product&quot;</span>: <span class="hljs-number">11</span>,
<span class="hljs-meta">... </span>    <span class="hljs-string">&quot;I-product&quot;</span>: <span class="hljs-number">12</span>,
<span class="hljs-meta">... </span>}`,wrap:!1}}),ps=new Bl({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[Ua],pytorch:[wa]},$$scope:{ctx:k}}}),rs=new vl({props:{$$slots:{default:[$a]},$$scope:{ctx:k}}}),Vs=new Ys({props:{title:"추론",local:"inference",headingTag:"h2"}}),Ns=new _({props:{code:"dGV4dCUyMCUzRCUyMCUyMlRoZSUyMEdvbGRlbiUyMFN0YXRlJTIwV2FycmlvcnMlMjBhcmUlMjBhbiUyMEFtZXJpY2FuJTIwcHJvZmVzc2lvbmFsJTIwYmFza2V0YmFsbCUyMHRlYW0lMjBiYXNlZCUyMGluJTIwU2FuJTIwRnJhbmNpc2NvLiUyMg==",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>text = <span class="hljs-string">&quot;The Golden State Warriors are an American professional basketball team based in San Francisco.&quot;</span>',wrap:!1}}),Qs=new _({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBJTBBY2xhc3NpZmllciUyMCUzRCUyMHBpcGVsaW5lKCUyMm5lciUyMiUyQyUyMG1vZGVsJTNEJTIyc3RldmhsaXUlMkZteV9hd2Vzb21lX3dudXRfbW9kZWwlMjIpJTBBY2xhc3NpZmllcih0ZXh0KQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline

<span class="hljs-meta">&gt;&gt;&gt; </span>classifier = pipeline(<span class="hljs-string">&quot;ner&quot;</span>, model=<span class="hljs-string">&quot;stevhliu/my_awesome_wnut_model&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>classifier(text)
[{<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;B-location&#x27;</span>,
  <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.42658573</span>,
  <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">2</span>,
  <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;golden&#x27;</span>,
  <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">4</span>,
  <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">10</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;I-location&#x27;</span>,
  <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.35856336</span>,
  <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">3</span>,
  <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;state&#x27;</span>,
  <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">11</span>,
  <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">16</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;B-group&#x27;</span>,
  <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.3064001</span>,
  <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">4</span>,
  <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;warriors&#x27;</span>,
  <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">17</span>,
  <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">25</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;B-location&#x27;</span>,
  <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.65523505</span>,
  <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">13</span>,
  <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;san&#x27;</span>,
  <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">80</span>,
  <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">83</span>},
 {<span class="hljs-string">&#x27;entity&#x27;</span>: <span class="hljs-string">&#x27;B-location&#x27;</span>,
  <span class="hljs-string">&#x27;score&#x27;</span>: <span class="hljs-number">0.4668663</span>,
  <span class="hljs-string">&#x27;index&#x27;</span>: <span class="hljs-number">14</span>,
  <span class="hljs-string">&#x27;word&#x27;</span>: <span class="hljs-string">&#x27;francisco&#x27;</span>,
  <span class="hljs-string">&#x27;start&#x27;</span>: <span class="hljs-number">84</span>,
  <span class="hljs-string">&#x27;end&#x27;</span>: <span class="hljs-number">93</span>}]`,wrap:!1}}),cs=new Bl({props:{pytorch:!0,tensorflow:!0,jax:!1,$$slots:{tensorflow:[Ca],pytorch:[_a]},$$scope:{ctx:k}}}),{c(){t=g("meta"),o=c(),l=g("p"),m=c(),j(b.$$.fragment),C=c(),j(W.$$.fragment),v=c(),j(I.$$.fragment),B=c(),x=g("p"),x.textContent=E,G=c(),$=g("p"),$.textContent=R,r=c(),U=g("ol"),U.innerHTML=V,X=c(),j(A.$$.fragment),M=c(),Z=g("p"),Z.textContent=F,os=c(),j(H.$$.fragment),N=c(),z=g("p"),z.textContent=ls,ms=c(),j(Q.$$.fragment),q=c(),j(Y.$$.fragment),S=c(),as=g("p"),as.textContent=Ms,L=c(),j(D.$$.fragment),P=c(),ts=g("p"),ts.textContent=js,K=c(),j(O.$$.fragment),ss=c(),es=g("p"),es.innerHTML=hs,p=c(),j(T.$$.fragment),ds=c(),us=g("p"),us.innerHTML=Gl,Ds=c(),ys=g("ul"),ys.innerHTML=Rl,Ps=c(),j(fs.$$.fragment),Ks=c(),j(bs.$$.fragment),Os=c(),gs=g("p"),gs.innerHTML=Wl,sl=c(),j(ws.$$.fragment),ll=c(),Js=g("p"),Js.innerHTML=Al,al=c(),j(Ts.$$.fragment),tl=c(),Us=g("p"),Us.innerHTML=Xl,el=c(),$s=g("ol"),$s.innerHTML=El,nl=c(),xs=g("p"),xs.textContent=Vl,pl=c(),j(_s.$$.fragment),rl=c(),ks=g("p"),ks.innerHTML=Fl,cl=c(),j(Cs.$$.fragment),il=c(),Is=g("p"),Is.innerHTML=Hl,ol=c(),j(ns.$$.fragment),ml=c(),j(Zs.$$.fragment),Ml=c(),vs=g("p"),vs.innerHTML=Nl,jl=c(),j(Bs.$$.fragment),hl=c(),Gs=g("p"),Gs.innerHTML=zl,dl=c(),j(Rs.$$.fragment),ul=c(),Ws=g("p"),Ws.innerHTML=Ql,yl=c(),j(As.$$.fragment),fl=c(),Xs=g("p"),Xs.innerHTML=ql,bl=c(),j(Es.$$.fragment),gl=c(),j(ps.$$.fragment),wl=c(),j(rs.$$.fragment),Jl=c(),j(Vs.$$.fragment),Tl=c(),Fs=g("p"),Fs.textContent=Yl,Ul=c(),Hs=g("p"),Hs.textContent=Sl,$l=c(),j(Ns.$$.fragment),xl=c(),zs=g("p"),zs.innerHTML=Ll,_l=c(),j(Qs.$$.fragment),kl=c(),qs=g("p"),qs.innerHTML=Dl,Cl=c(),j(cs.$$.fragment),Il=c(),Ls=g("p"),this.h()},l(s){const n=ma("svelte-u9bgzb",document.head);t=w(n,"META",{name:!0,content:!0}),n.forEach(a),o=i(s),l=w(s,"P",{}),aa(l).forEach(a),m=i(s),h(b.$$.fragment,s),C=i(s),h(W.$$.fragment,s),v=i(s),h(I.$$.fragment,s),B=i(s),x=w(s,"P",{"data-svelte-h":!0}),J(x)!=="svelte-h22iel"&&(x.textContent=E),G=i(s),$=w(s,"P",{"data-svelte-h":!0}),J($)!=="svelte-14hiaa1"&&($.textContent=R),r=i(s),U=w(s,"OL",{"data-svelte-h":!0}),J(U)!=="svelte-1hfy8fu"&&(U.innerHTML=V),X=i(s),h(A.$$.fragment,s),M=i(s),Z=w(s,"P",{"data-svelte-h":!0}),J(Z)!=="svelte-1bc8bfk"&&(Z.textContent=F),os=i(s),h(H.$$.fragment,s),N=i(s),z=w(s,"P",{"data-svelte-h":!0}),J(z)!=="svelte-1hhyu5y"&&(z.textContent=ls),ms=i(s),h(Q.$$.fragment,s),q=i(s),h(Y.$$.fragment,s),S=i(s),as=w(s,"P",{"data-svelte-h":!0}),J(as)!=="svelte-1v008tf"&&(as.textContent=Ms),L=i(s),h(D.$$.fragment,s),P=i(s),ts=w(s,"P",{"data-svelte-h":!0}),J(ts)!=="svelte-7y00sm"&&(ts.textContent=js),K=i(s),h(O.$$.fragment,s),ss=i(s),es=w(s,"P",{"data-svelte-h":!0}),J(es)!=="svelte-19kj2kd"&&(es.innerHTML=hs),p=i(s),h(T.$$.fragment,s),ds=i(s),us=w(s,"P",{"data-svelte-h":!0}),J(us)!=="svelte-1jetaks"&&(us.innerHTML=Gl),Ds=i(s),ys=w(s,"UL",{"data-svelte-h":!0}),J(ys)!=="svelte-wzon1c"&&(ys.innerHTML=Rl),Ps=i(s),h(fs.$$.fragment,s),Ks=i(s),h(bs.$$.fragment,s),Os=i(s),gs=w(s,"P",{"data-svelte-h":!0}),J(gs)!=="svelte-xprzxq"&&(gs.innerHTML=Wl),sl=i(s),h(ws.$$.fragment,s),ll=i(s),Js=w(s,"P",{"data-svelte-h":!0}),J(Js)!=="svelte-n9m45o"&&(Js.innerHTML=Al),al=i(s),h(Ts.$$.fragment,s),tl=i(s),Us=w(s,"P",{"data-svelte-h":!0}),J(Us)!=="svelte-1slve6o"&&(Us.innerHTML=Xl),el=i(s),$s=w(s,"OL",{"data-svelte-h":!0}),J($s)!=="svelte-1ynfql3"&&($s.innerHTML=El),nl=i(s),xs=w(s,"P",{"data-svelte-h":!0}),J(xs)!=="svelte-1uatru5"&&(xs.textContent=Vl),pl=i(s),h(_s.$$.fragment,s),rl=i(s),ks=w(s,"P",{"data-svelte-h":!0}),J(ks)!=="svelte-o2pm0z"&&(ks.innerHTML=Fl),cl=i(s),h(Cs.$$.fragment,s),il=i(s),Is=w(s,"P",{"data-svelte-h":!0}),J(Is)!=="svelte-11dkppi"&&(Is.innerHTML=Hl),ol=i(s),h(ns.$$.fragment,s),ml=i(s),h(Zs.$$.fragment,s),Ml=i(s),vs=w(s,"P",{"data-svelte-h":!0}),J(vs)!=="svelte-1xei3jh"&&(vs.innerHTML=Nl),jl=i(s),h(Bs.$$.fragment,s),hl=i(s),Gs=w(s,"P",{"data-svelte-h":!0}),J(Gs)!=="svelte-no1q5"&&(Gs.innerHTML=zl),dl=i(s),h(Rs.$$.fragment,s),ul=i(s),Ws=w(s,"P",{"data-svelte-h":!0}),J(Ws)!=="svelte-1tfmmd0"&&(Ws.innerHTML=Ql),yl=i(s),h(As.$$.fragment,s),fl=i(s),Xs=w(s,"P",{"data-svelte-h":!0}),J(Xs)!=="svelte-84b3vk"&&(Xs.innerHTML=ql),bl=i(s),h(Es.$$.fragment,s),gl=i(s),h(ps.$$.fragment,s),wl=i(s),h(rs.$$.fragment,s),Jl=i(s),h(Vs.$$.fragment,s),Tl=i(s),Fs=w(s,"P",{"data-svelte-h":!0}),J(Fs)!=="svelte-1r6hgyn"&&(Fs.textContent=Yl),Ul=i(s),Hs=w(s,"P",{"data-svelte-h":!0}),J(Hs)!=="svelte-ej1eir"&&(Hs.textContent=Sl),$l=i(s),h(Ns.$$.fragment,s),xl=i(s),zs=w(s,"P",{"data-svelte-h":!0}),J(zs)!=="svelte-bavprk"&&(zs.innerHTML=Ll),_l=i(s),h(Qs.$$.fragment,s),kl=i(s),qs=w(s,"P",{"data-svelte-h":!0}),J(qs)!=="svelte-1smgeha"&&(qs.innerHTML=Dl),Cl=i(s),h(cs.$$.fragment,s),Il=i(s),Ls=w(s,"P",{}),aa(Ls).forEach(a),this.h()},h(){ta(t,"name","hf:doc:metadata"),ta(t,"content",Za)},m(s,n){Ma(document.head,t),e(s,o,n),e(s,l,n),e(s,m,n),d(b,s,n),e(s,C,n),d(W,s,n),e(s,v,n),d(I,s,n),e(s,B,n),e(s,x,n),e(s,G,n),e(s,$,n),e(s,r,n),e(s,U,n),e(s,X,n),d(A,s,n),e(s,M,n),e(s,Z,n),e(s,os,n),d(H,s,n),e(s,N,n),e(s,z,n),e(s,ms,n),d(Q,s,n),e(s,q,n),d(Y,s,n),e(s,S,n),e(s,as,n),e(s,L,n),d(D,s,n),e(s,P,n),e(s,ts,n),e(s,K,n),d(O,s,n),e(s,ss,n),e(s,es,n),e(s,p,n),d(T,s,n),e(s,ds,n),e(s,us,n),e(s,Ds,n),e(s,ys,n),e(s,Ps,n),d(fs,s,n),e(s,Ks,n),d(bs,s,n),e(s,Os,n),e(s,gs,n),e(s,sl,n),d(ws,s,n),e(s,ll,n),e(s,Js,n),e(s,al,n),d(Ts,s,n),e(s,tl,n),e(s,Us,n),e(s,el,n),e(s,$s,n),e(s,nl,n),e(s,xs,n),e(s,pl,n),d(_s,s,n),e(s,rl,n),e(s,ks,n),e(s,cl,n),d(Cs,s,n),e(s,il,n),e(s,Is,n),e(s,ol,n),d(ns,s,n),e(s,ml,n),d(Zs,s,n),e(s,Ml,n),e(s,vs,n),e(s,jl,n),d(Bs,s,n),e(s,hl,n),e(s,Gs,n),e(s,dl,n),d(Rs,s,n),e(s,ul,n),e(s,Ws,n),e(s,yl,n),d(As,s,n),e(s,fl,n),e(s,Xs,n),e(s,bl,n),d(Es,s,n),e(s,gl,n),d(ps,s,n),e(s,wl,n),d(rs,s,n),e(s,Jl,n),d(Vs,s,n),e(s,Tl,n),e(s,Fs,n),e(s,Ul,n),e(s,Hs,n),e(s,$l,n),d(Ns,s,n),e(s,xl,n),e(s,zs,n),e(s,_l,n),d(Qs,s,n),e(s,kl,n),e(s,qs,n),e(s,Cl,n),d(cs,s,n),e(s,Il,n),e(s,Ls,n),Zl=!0},p(s,[n]){const Pl={};n&2&&(Pl.$$scope={dirty:n,ctx:s}),A.$set(Pl);const Kl={};n&2&&(Kl.$$scope={dirty:n,ctx:s}),ns.$set(Kl);const Ol={};n&2&&(Ol.$$scope={dirty:n,ctx:s}),ps.$set(Ol);const sa={};n&2&&(sa.$$scope={dirty:n,ctx:s}),rs.$set(sa);const la={};n&2&&(la.$$scope={dirty:n,ctx:s}),cs.$set(la)},i(s){Zl||(u(b.$$.fragment,s),u(W.$$.fragment,s),u(I.$$.fragment,s),u(A.$$.fragment,s),u(H.$$.fragment,s),u(Q.$$.fragment,s),u(Y.$$.fragment,s),u(D.$$.fragment,s),u(O.$$.fragment,s),u(T.$$.fragment,s),u(fs.$$.fragment,s),u(bs.$$.fragment,s),u(ws.$$.fragment,s),u(Ts.$$.fragment,s),u(_s.$$.fragment,s),u(Cs.$$.fragment,s),u(ns.$$.fragment,s),u(Zs.$$.fragment,s),u(Bs.$$.fragment,s),u(Rs.$$.fragment,s),u(As.$$.fragment,s),u(Es.$$.fragment,s),u(ps.$$.fragment,s),u(rs.$$.fragment,s),u(Vs.$$.fragment,s),u(Ns.$$.fragment,s),u(Qs.$$.fragment,s),u(cs.$$.fragment,s),Zl=!0)},o(s){y(b.$$.fragment,s),y(W.$$.fragment,s),y(I.$$.fragment,s),y(A.$$.fragment,s),y(H.$$.fragment,s),y(Q.$$.fragment,s),y(Y.$$.fragment,s),y(D.$$.fragment,s),y(O.$$.fragment,s),y(T.$$.fragment,s),y(fs.$$.fragment,s),y(bs.$$.fragment,s),y(ws.$$.fragment,s),y(Ts.$$.fragment,s),y(_s.$$.fragment,s),y(Cs.$$.fragment,s),y(ns.$$.fragment,s),y(Zs.$$.fragment,s),y(Bs.$$.fragment,s),y(Rs.$$.fragment,s),y(As.$$.fragment,s),y(Es.$$.fragment,s),y(ps.$$.fragment,s),y(rs.$$.fragment,s),y(Vs.$$.fragment,s),y(Ns.$$.fragment,s),y(Qs.$$.fragment,s),y(cs.$$.fragment,s),Zl=!1},d(s){s&&(a(o),a(l),a(m),a(C),a(v),a(B),a(x),a(G),a($),a(r),a(U),a(X),a(M),a(Z),a(os),a(N),a(z),a(ms),a(q),a(S),a(as),a(L),a(P),a(ts),a(K),a(ss),a(es),a(p),a(ds),a(us),a(Ds),a(ys),a(Ps),a(Ks),a(Os),a(gs),a(sl),a(ll),a(Js),a(al),a(tl),a(Us),a(el),a($s),a(nl),a(xs),a(pl),a(rl),a(ks),a(cl),a(il),a(Is),a(ol),a(ml),a(Ml),a(vs),a(jl),a(hl),a(Gs),a(dl),a(ul),a(Ws),a(yl),a(fl),a(Xs),a(bl),a(gl),a(wl),a(Jl),a(Tl),a(Fs),a(Ul),a(Hs),a($l),a(xl),a(zs),a(_l),a(kl),a(qs),a(Cl),a(Il),a(Ls)),a(t),f(b,s),f(W,s),f(I,s),f(A,s),f(H,s),f(Q,s),f(Y,s),f(D,s),f(O,s),f(T,s),f(fs,s),f(bs,s),f(ws,s),f(Ts,s),f(_s,s),f(Cs,s),f(ns,s),f(Zs,s),f(Bs,s),f(Rs,s),f(As,s),f(Es,s),f(ps,s),f(rs,s),f(Vs,s),f(Ns,s),f(Qs,s),f(cs,s)}}}const Za='{"title":"토큰 분류","local":"token-classification","sections":[{"title":"WNUT 17 데이터 세트 가져오기","local":"load-wnut-17-dataset","sections":[],"depth":2},{"title":"전처리","local":"preprocess","sections":[],"depth":2},{"title":"평가","local":"evaluation","sections":[],"depth":2},{"title":"훈련","local":"train","sections":[],"depth":2},{"title":"추론","local":"inference","sections":[],"depth":2}],"depth":1}';function va(k){return ca(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Fa extends ia{constructor(t){super(),oa(this,t,va,Ia,ra,{})}}export{Fa as component};
