import{s as k,n as S,o as U}from"../chunks/scheduler.9bc65507.js";import{S as D,i as j,g as r,s as o,r as I,A as O,h as i,f as a,c as s,j as $,u as z,x as v,k as G,y as K,a as l,v as N,d as q,t as W,w as F}from"../chunks/index.707bf1b6.js";import{H as J}from"../chunks/Heading.342b1fa6.js";function Q(M){let n,x,u,T,p,b,h,C=`There is a growing field of study concerned with investigating the inner working of large-scale transformers like BERT
(that some call “BERTology”). Some good examples of this field are:`,w,f,R=`<li>BERT Rediscovers the Classical NLP Pipeline by Ian Tenney, Dipanjan Das, Ellie Pavlick:
<a href="https://arxiv.org/abs/1905.05950" rel="nofollow">https://arxiv.org/abs/1905.05950</a></li> <li>Are Sixteen Heads Really Better than One? by Paul Michel, Omer Levy, Graham Neubig: <a href="https://arxiv.org/abs/1905.10650" rel="nofollow">https://arxiv.org/abs/1905.10650</a></li> <li>What Does BERT Look At? An Analysis of BERT’s Attention by Kevin Clark, Urvashi Khandelwal, Omer Levy, Christopher D.
Manning: <a href="https://arxiv.org/abs/1906.04341" rel="nofollow">https://arxiv.org/abs/1906.04341</a></li> <li>CAT-probing: A Metric-based Approach to Interpret How Pre-trained Models for Programming Language Attend Code Structure: <a href="https://arxiv.org/abs/2210.04633" rel="nofollow">https://arxiv.org/abs/2210.04633</a></li>`,y,c,H=`In order to help this new field develop, we have included a few additional features in the BERT/GPT/GPT-2 models to
help people access the inner representations, mainly adapted from the great work of Paul Michel
(<a href="https://arxiv.org/abs/1905.10650" rel="nofollow">https://arxiv.org/abs/1905.10650</a>):`,_,m,A=`<li>accessing all the hidden-states of BERT/GPT/GPT-2,</li> <li>accessing all the attention weights for each head of BERT/GPT/GPT-2,</li> <li>retrieving heads output values and gradients to be able to compute head importance score and prune head as explained
in <a href="https://arxiv.org/abs/1905.10650" rel="nofollow">https://arxiv.org/abs/1905.10650</a>.</li>`,P,d,B=`To help you understand and use these features, we have added a specific example script: <a href="https://github.com/huggingface/transformers/tree/main/examples/research_projects/bertology/run_bertology.py" rel="nofollow">bertology.py</a> while extract information and prune a model pre-trained on
GLUE.`,L,g,E;return p=new J({props:{title:"BERTology",local:"bertology",headingTag:"h1"}}),{c(){n=r("meta"),x=o(),u=r("p"),T=o(),I(p.$$.fragment),b=o(),h=r("p"),h.textContent=C,w=o(),f=r("ul"),f.innerHTML=R,y=o(),c=r("p"),c.innerHTML=H,_=o(),m=r("ul"),m.innerHTML=A,P=o(),d=r("p"),d.innerHTML=B,L=o(),g=r("p"),this.h()},l(e){const t=O("svelte-u9bgzb",document.head);n=i(t,"META",{name:!0,content:!0}),t.forEach(a),x=s(e),u=i(e,"P",{}),$(u).forEach(a),T=s(e),z(p.$$.fragment,e),b=s(e),h=i(e,"P",{"data-svelte-h":!0}),v(h)!=="svelte-e5vxoh"&&(h.textContent=C),w=s(e),f=i(e,"UL",{"data-svelte-h":!0}),v(f)!=="svelte-1e34lw"&&(f.innerHTML=R),y=s(e),c=i(e,"P",{"data-svelte-h":!0}),v(c)!=="svelte-1bavv9a"&&(c.innerHTML=H),_=s(e),m=i(e,"UL",{"data-svelte-h":!0}),v(m)!=="svelte-14erpmf"&&(m.innerHTML=A),P=s(e),d=i(e,"P",{"data-svelte-h":!0}),v(d)!=="svelte-7ztyw0"&&(d.innerHTML=B),L=s(e),g=i(e,"P",{}),$(g).forEach(a),this.h()},h(){G(n,"name","hf:doc:metadata"),G(n,"content",V)},m(e,t){K(document.head,n),l(e,x,t),l(e,u,t),l(e,T,t),N(p,e,t),l(e,b,t),l(e,h,t),l(e,w,t),l(e,f,t),l(e,y,t),l(e,c,t),l(e,_,t),l(e,m,t),l(e,P,t),l(e,d,t),l(e,L,t),l(e,g,t),E=!0},p:S,i(e){E||(q(p.$$.fragment,e),E=!0)},o(e){W(p.$$.fragment,e),E=!1},d(e){e&&(a(x),a(u),a(T),a(b),a(h),a(w),a(f),a(y),a(c),a(_),a(m),a(P),a(d),a(L),a(g)),a(n),F(p,e)}}}const V='{"title":"BERTology","local":"bertology","sections":[],"depth":1}';function X(M){return U(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class te extends D{constructor(n){super(),j(this,n,X,Q,k,{})}}export{te as component};
