import{s as ne,a as se,n as ie,o as pe}from"../chunks/scheduler.3a17fb72.js";import{S as me,i as Me,e as i,s as a,c as M,h as re,a as p,d as l,b as n,f as Gt,g as r,j as u,k as m,l as ue,m as s,n as h,t as c,o,p as y}from"../chunks/index.093f8863.js";import{C as he,H as ae,E as ce}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.4c812468.js";import{C as f}from"../chunks/CodeBlock.34c4e749.js";function oe(_t){let w,tt,P,et,J,lt,k,st,I,Qt=`Pick the right benchmarks with our benchmark finder:
Search by language, task type, dataset name, or keywords.`,at,v,Rt="<p>Not all tasks are compatible with inspect-ai’s API as of yet, we are working on converting all of them !</p>",nt,b,Nt,it,x,Vt="Once you’ve chosen a benchmark, run it with <code>lighteval eval</code>. Below are examples for common setups.",pt,H,mt,S,zt="<li>Evaluate a model via Hugging Face Inference Providers.</li>",Mt,A,rt,d,Et="<li>Run multiple evals at the same time.</li>",ut,G,ht,Z,qt="<li>Compare providers for the same model.</li>",ct,_,ot,g,Xt="<li>Evaluate a vLLM or SGLang model.</li>",yt,Q,ft,$,Yt="<li>See the impact of few-shot on your model.</li>",bt,R,Ut,C,Ft="<li>Optimize custom server connections.</li>",wt,N,vt,T,Bt="<li>Use multiple epochs for more reliable results.</li>",dt,V,Zt,j,Dt="<li>Push to the Hub to share results.</li>",gt,z,$t,E,Ot="Resulting Space:",Ct,U,Pt,Tt,L,Kt="<li>Change model behaviour</li>",jt,q,te="You can use any argument defined in inspect-ai’s API.",Lt,X,Wt,W,ee="<li>Use model-args to use any inference provider specific argument.</li>",Jt,Y,kt,F,It,B,le="LightEval prints a per-model results table:",xt,D,Ht,O,St,K,At;return J=new he({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),k=new ae({props:{title:"Evaluate your model with Inspect-AI",local:"evaluate-your-model-with-inspect-ai",headingTag:"h1"}}),H=new ae({props:{title:"Examples",local:"examples",headingTag:"h3"}}),A=new f({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMCUyMmhmLWluZmVyZW5jZS1wcm92aWRlcnMlMkZvcGVuYWklMkZncHQtb3NzLTIwYiUyMiUyMCUyMmxpZ2h0ZXZhbCU3Q2dwcWElM0FkaWFtb25kJTdDMCUyMg==",highlighted:'lighteval <span class="hljs-built_in">eval</span> <span class="hljs-string">&quot;hf-inference-providers/openai/gpt-oss-20b&quot;</span> <span class="hljs-string">&quot;lighteval|gpqa:diamond|0&quot;</span>',wrap:!1}}),G=new f({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMCUyMmhmLWluZmVyZW5jZS1wcm92aWRlcnMlMkZvcGVuYWklMkZncHQtb3NzLTIwYiUyMiUyMCUyMmxpZ2h0ZXZhbCU3Q2dwcWElM0FkaWFtb25kJTdDMCUyQ2xpZ2h0ZXZhbCU3Q2FpbWUyNSU3QzAlMjI=",highlighted:'lighteval <span class="hljs-built_in">eval</span> <span class="hljs-string">&quot;hf-inference-providers/openai/gpt-oss-20b&quot;</span> <span class="hljs-string">&quot;lighteval|gpqa:diamond|0,lighteval|aime25|0&quot;</span>',wrap:!1}}),_=new f({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMCU1QyUwQSUyMCUyMCUyMCUyMGhmLWluZmVyZW5jZS1wcm92aWRlcnMlMkZvcGVuYWklMkZncHQtb3NzLTIwYiUzQWZpcmV3b3Jrcy1haSUyMCU1QyUwQSUyMCUyMCUyMCUyMGhmLWluZmVyZW5jZS1wcm92aWRlcnMlMkZvcGVuYWklMkZncHQtb3NzLTIwYiUzQXRvZ2V0aGVyJTIwJTVDJTBBJTIwJTIwJTIwJTIwaGYtaW5mZXJlbmNlLXByb3ZpZGVycyUyRm9wZW5haSUyRmdwdC1vc3MtMjBiJTNBbmViaXVzJTIwJTVDJTBBJTIwJTIwJTIwJTIwJTIybGlnaHRldmFsJTdDZ3BxYSUzQWRpYW1vbmQlN0MwJTIy",highlighted:`lighteval <span class="hljs-built_in">eval</span> \\
    hf-inference-providers/openai/gpt-oss-20b:fireworks-ai \\
    hf-inference-providers/openai/gpt-oss-20b:together \\
    hf-inference-providers/openai/gpt-oss-20b:nebius \\
    <span class="hljs-string">&quot;lighteval|gpqa:diamond|0&quot;</span>`,wrap:!1}}),Q=new f({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMHZsbG0lMkZIdWdnaW5nRmFjZVRCJTJGU21vbExNLTEzNU0tSW5zdHJ1Y3QlMjAlMjJsaWdodGV2YWwlN0NncHFhJTNBZGlhbW9uZCU3QzAlMjI=",highlighted:'lighteval <span class="hljs-built_in">eval</span> vllm/HuggingFaceTB/SmolLM-135M-Instruct <span class="hljs-string">&quot;lighteval|gpqa:diamond|0&quot;</span>',wrap:!1}}),R=new f({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMGhmLWluZmVyZW5jZS1wcm92aWRlcnMlMkZvcGVuYWklMkZncHQtb3NzLTIwYiUyMCUyMmxpZ2h0ZXZhbCU3Q2dzbThrJTdDMCUyQ2xpZ2h0ZXZhbCU3Q2dzbThrJTdDNSUyMg==",highlighted:'lighteval <span class="hljs-built_in">eval</span> hf-inference-providers/openai/gpt-oss-20b <span class="hljs-string">&quot;lighteval|gsm8k|0,lighteval|gsm8k|5&quot;</span>',wrap:!1}}),N=new f({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMGhmLWluZmVyZW5jZS1wcm92aWRlcnMlMkZvcGVuYWklMkZncHQtb3NzLTIwYiUyMCUyMmxpZ2h0ZXZhbCU3Q2dzbThrJTdDMCUyMiUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tbWF4LWNvbm5lY3Rpb25zJTIwNTAlMjAlNUMlMEElMjAlMjAlMjAlMjAtLXRpbWVvdXQlMjAzMCUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tcmV0cnktb24tZXJyb3IlMjAxJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tYXgtcmV0cmllcyUyMDElMjAlNUMlMEElMjAlMjAlMjAlMjAtLW1heC1zYW1wbGVzJTIwMTA=",highlighted:`lighteval <span class="hljs-built_in">eval</span> hf-inference-providers/openai/gpt-oss-20b <span class="hljs-string">&quot;lighteval|gsm8k|0&quot;</span> \\
    --max-connections 50 \\
    --<span class="hljs-built_in">timeout</span> 30 \\
    --retry-on-error 1 \\
    --max-retries 1 \\
    --max-samples 10`,wrap:!1}}),V=new f({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMGhmLWluZmVyZW5jZS1wcm92aWRlcnMlMkZvcGVuYWklMkZncHQtb3NzLTIwYiUyMCUyMmxpZ2h0ZXZhbCU3Q2FpbWUyNSU3QzAlMjIlMjAtLWVwb2NocyUyMDE2JTIwLS1lcG9jaHMtcmVkdWNlciUyMCUyMnBhc3NfYXRfNCUyMg==",highlighted:'lighteval <span class="hljs-built_in">eval</span> hf-inference-providers/openai/gpt-oss-20b <span class="hljs-string">&quot;lighteval|aime25|0&quot;</span> --epochs 16 --epochs-reducer <span class="hljs-string">&quot;pass_at_4&quot;</span>',wrap:!1}}),z=new f({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMGhmLWluZmVyZW5jZS1wcm92aWRlcnMlMkZvcGVuYWklMkZncHQtb3NzLTIwYiUyMCUyMmxpZ2h0ZXZhbCU3Q2hsZSU3QzAlMjIlMjAlNUMlMEElMjAlMjAlMjAlMjAtLWJ1bmRsZS1kaXIlMjBncHQtb3NzLWJ1bmRsZSUyMCU1QyUwQSUyMCUyMCUyMCUyMC0tcmVwby1pZCUyME9wZW5FdmFscyUyRmV2YWxzJTIwJTVDJTBBJTIwJTIwJTIwJTIwLS1tYXgtc2FtcGxlcyUyMDEwMA==",highlighted:`lighteval <span class="hljs-built_in">eval</span> hf-inference-providers/openai/gpt-oss-20b <span class="hljs-string">&quot;lighteval|hle|0&quot;</span> \\
    --bundle-dir gpt-oss-bundle \\
    --repo-id OpenEvals/evals \\
    --max-samples 100`,wrap:!1}}),X=new f({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMGhmLWluZmVyZW5jZS1wcm92aWRlcnMlMkZvcGVuYWklMkZncHQtb3NzLTIwYiUyMCUyMmxpZ2h0ZXZhbCU3Q2FpbWUyNSU3QzAlMjIlMjAtLXRlbXBlcmF0dXJlJTIwMC4x",highlighted:'lighteval <span class="hljs-built_in">eval</span> hf-inference-providers/openai/gpt-oss-20b <span class="hljs-string">&quot;lighteval|aime25|0&quot;</span> --temperature 0.1',wrap:!1}}),Y=new f({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMGdvb2dsZSUyRmdlbWluaS0yLjUtcHJvJTIwJTIybGlnaHRldmFsJTdDYWltZTI1JTdDMCUyMiUyMC0tbW9kZWwtYXJncyUyMGxvY2F0aW9uJTNEdXMtZWFzdDU=",highlighted:'lighteval <span class="hljs-built_in">eval</span> google/gemini-2.5-pro <span class="hljs-string">&quot;lighteval|aime25|0&quot;</span> --model-args location=us-east5',wrap:!1}}),F=new f({props:{code:"bGlnaHRldmFsJTIwZXZhbCUyMG9wZW5haSUyRmdwdC00byUyMCUyMmxpZ2h0ZXZhbCU3Q2dwcWElM0FkaWFtb25kJTdDMCUyMiUyMC0tbW9kZWwtYXJncyUyMHNlcnZpY2VfdGllciUzRGZsZXglMkNjbGllbnRfdGltZW91dCUzRDEyMDA=",highlighted:'lighteval <span class="hljs-built_in">eval</span> openai/gpt-4o <span class="hljs-string">&quot;lighteval|gpqa:diamond|0&quot;</span> --model-args service_tier=flex,client_timeout=1200',wrap:!1}}),D=new f({props:{code:"Q29tcGxldGVkJTIwYWxsJTIwdGFza3MlMjBpbiUyMCdsaWdodGV2YWwtbG9ncyclMjBzdWNjZXNzZnVsbHklMEElMEElN0MlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBNb2RlbCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCU3Q2dwcWElN0NncHFhJTNBZGlhbW9uZCU3QyUwQSU3Qy0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLS0tLSU3Qy0tLSUzQSU3Qy0tLS0tLS0tLS0tJTNBJTdDJTBBJTdDdmxsbSUyRkh1Z2dpbmdGYWNlVEIlMkZTbW9sTE0tMTM1TS1JbnN0cnVjdCU3QzAuMDElN0MlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAwLjAxJTdDJTBBJTBBcmVzdWx0cyUyMHNhdmVkJTIwdG8lMjBsaWdodGV2YWwtbG9ncyUwQXJ1biUyMCUyMmluc3BlY3QlMjB2aWV3JTIwLS1sb2ctZGlyJTIwbGlnaHRldmFsLWxvZ3MlMjIlMjB0byUyMHZpZXclMjB0aGUlMjByZXN1bHRz",highlighted:`<span class="hljs-title class_">Completed</span> all tasks <span class="hljs-keyword">in</span> <span class="hljs-string">&#x27;lighteval-logs&#x27;</span> successfully

|                 <span class="hljs-title class_">Model</span>                 |gpqa|<span class="hljs-symbol">gpqa:</span>diamond|
|---------------------------------------|---<span class="hljs-symbol">:|-----------</span><span class="hljs-symbol">:|</span>
|vllm/<span class="hljs-title class_">HuggingFaceTB</span>/<span class="hljs-title class_">SmolLM</span><span class="hljs-number">-135</span>M-<span class="hljs-title class_">Instruct</span>|<span class="hljs-number">0.01</span>|        <span class="hljs-number">0.01</span>|

results saved to lighteval-logs
run <span class="hljs-string">&quot;inspect view --log-dir lighteval-logs&quot;</span> to view the results`,wrap:!1}}),O=new ce({props:{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/inspect-ai.mdx"}}),{c(){w=i("meta"),tt=a(),P=i("p"),et=a(),M(J.$$.fragment),lt=a(),M(k.$$.fragment),st=a(),I=i("p"),I.textContent=Qt,at=a(),v=i("blockquote"),v.innerHTML=Rt,nt=a(),b=i("iframe"),it=a(),x=i("p"),x.innerHTML=Vt,pt=a(),M(H.$$.fragment),mt=a(),S=i("ol"),S.innerHTML=zt,Mt=a(),M(A.$$.fragment),rt=a(),d=i("ol"),d.innerHTML=Et,ut=a(),M(G.$$.fragment),ht=a(),Z=i("ol"),Z.innerHTML=qt,ct=a(),M(_.$$.fragment),ot=a(),g=i("ol"),g.innerHTML=Xt,yt=a(),M(Q.$$.fragment),ft=a(),$=i("ol"),$.innerHTML=Yt,bt=a(),M(R.$$.fragment),Ut=a(),C=i("ol"),C.innerHTML=Ft,wt=a(),M(N.$$.fragment),vt=a(),T=i("ol"),T.innerHTML=Bt,dt=a(),M(V.$$.fragment),Zt=a(),j=i("ol"),j.innerHTML=Dt,gt=a(),M(z.$$.fragment),$t=a(),E=i("p"),E.textContent=Ot,Ct=a(),U=i("iframe"),Tt=a(),L=i("ol"),L.innerHTML=Kt,jt=a(),q=i("p"),q.textContent=te,Lt=a(),M(X.$$.fragment),Wt=a(),W=i("ol"),W.innerHTML=ee,Jt=a(),M(Y.$$.fragment),kt=a(),M(F.$$.fragment),It=a(),B=i("p"),B.textContent=le,xt=a(),M(D.$$.fragment),Ht=a(),M(O.$$.fragment),St=a(),K=i("p"),this.h()},l(t){const e=re("svelte-u9bgzb",document.head);w=p(e,"META",{name:!0,content:!0}),e.forEach(l),tt=n(t),P=p(t,"P",{}),Gt(P).forEach(l),et=n(t),r(J.$$.fragment,t),lt=n(t),r(k.$$.fragment,t),st=n(t),I=p(t,"P",{"data-svelte-h":!0}),u(I)!=="svelte-wvxpw7"&&(I.textContent=Qt),at=n(t),v=p(t,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),u(v)!=="svelte-13z5e58"&&(v.innerHTML=Rt),nt=n(t),b=p(t,"IFRAME",{src:!0,frameborder:!0,width:!0,height:!0}),Gt(b).forEach(l),it=n(t),x=p(t,"P",{"data-svelte-h":!0}),u(x)!=="svelte-1s8wmkx"&&(x.innerHTML=Vt),pt=n(t),r(H.$$.fragment,t),mt=n(t),S=p(t,"OL",{"data-svelte-h":!0}),u(S)!=="svelte-1n0ds6r"&&(S.innerHTML=zt),Mt=n(t),r(A.$$.fragment,t),rt=n(t),d=p(t,"OL",{start:!0,"data-svelte-h":!0}),u(d)!=="svelte-158v33h"&&(d.innerHTML=Et),ut=n(t),r(G.$$.fragment,t),ht=n(t),Z=p(t,"OL",{start:!0,"data-svelte-h":!0}),u(Z)!=="svelte-12sgcy5"&&(Z.innerHTML=qt),ct=n(t),r(_.$$.fragment,t),ot=n(t),g=p(t,"OL",{start:!0,"data-svelte-h":!0}),u(g)!=="svelte-h63d4v"&&(g.innerHTML=Xt),yt=n(t),r(Q.$$.fragment,t),ft=n(t),$=p(t,"OL",{start:!0,"data-svelte-h":!0}),u($)!=="svelte-1lgb0oc"&&($.innerHTML=Yt),bt=n(t),r(R.$$.fragment,t),Ut=n(t),C=p(t,"OL",{start:!0,"data-svelte-h":!0}),u(C)!=="svelte-xic7d6"&&(C.innerHTML=Ft),wt=n(t),r(N.$$.fragment,t),vt=n(t),T=p(t,"OL",{start:!0,"data-svelte-h":!0}),u(T)!=="svelte-z8xpgu"&&(T.innerHTML=Bt),dt=n(t),r(V.$$.fragment,t),Zt=n(t),j=p(t,"OL",{start:!0,"data-svelte-h":!0}),u(j)!=="svelte-bkjaqd"&&(j.innerHTML=Dt),gt=n(t),r(z.$$.fragment,t),$t=n(t),E=p(t,"P",{"data-svelte-h":!0}),u(E)!=="svelte-xmt8gz"&&(E.textContent=Ot),Ct=n(t),U=p(t,"IFRAME",{src:!0,frameborder:!0,width:!0,height:!0}),Gt(U).forEach(l),Tt=n(t),L=p(t,"OL",{start:!0,"data-svelte-h":!0}),u(L)!=="svelte-65smkt"&&(L.innerHTML=Kt),jt=n(t),q=p(t,"P",{"data-svelte-h":!0}),u(q)!=="svelte-1hiqzh0"&&(q.textContent=te),Lt=n(t),r(X.$$.fragment,t),Wt=n(t),W=p(t,"OL",{start:!0,"data-svelte-h":!0}),u(W)!=="svelte-p39luw"&&(W.innerHTML=ee),Jt=n(t),r(Y.$$.fragment,t),kt=n(t),r(F.$$.fragment,t),It=n(t),B=p(t,"P",{"data-svelte-h":!0}),u(B)!=="svelte-g5np0a"&&(B.textContent=le),xt=n(t),r(D.$$.fragment,t),Ht=n(t),r(O.$$.fragment,t),St=n(t),K=p(t,"P",{}),Gt(K).forEach(l),this.h()},h(){m(w,"name","hf:doc:metadata"),m(w,"content",ye),m(v,"class","warning"),se(b.src,Nt="https://openevals-open-benchmark-index.hf.space")||m(b,"src",Nt),m(b,"frameborder","0"),m(b,"width","850"),m(b,"height","450"),m(d,"start","2"),m(Z,"start","3"),m(g,"start","4"),m($,"start","5"),m(C,"start","6"),m(T,"start","7"),m(j,"start","8"),se(U.src,Pt="https://openevals-evals.static.hf.space")||m(U,"src",Pt),m(U,"frameborder","0"),m(U,"width","850"),m(U,"height","450"),m(L,"start","9"),m(W,"start","10")},m(t,e){ue(document.head,w),s(t,tt,e),s(t,P,e),s(t,et,e),h(J,t,e),s(t,lt,e),h(k,t,e),s(t,st,e),s(t,I,e),s(t,at,e),s(t,v,e),s(t,nt,e),s(t,b,e),s(t,it,e),s(t,x,e),s(t,pt,e),h(H,t,e),s(t,mt,e),s(t,S,e),s(t,Mt,e),h(A,t,e),s(t,rt,e),s(t,d,e),s(t,ut,e),h(G,t,e),s(t,ht,e),s(t,Z,e),s(t,ct,e),h(_,t,e),s(t,ot,e),s(t,g,e),s(t,yt,e),h(Q,t,e),s(t,ft,e),s(t,$,e),s(t,bt,e),h(R,t,e),s(t,Ut,e),s(t,C,e),s(t,wt,e),h(N,t,e),s(t,vt,e),s(t,T,e),s(t,dt,e),h(V,t,e),s(t,Zt,e),s(t,j,e),s(t,gt,e),h(z,t,e),s(t,$t,e),s(t,E,e),s(t,Ct,e),s(t,U,e),s(t,Tt,e),s(t,L,e),s(t,jt,e),s(t,q,e),s(t,Lt,e),h(X,t,e),s(t,Wt,e),s(t,W,e),s(t,Jt,e),h(Y,t,e),s(t,kt,e),h(F,t,e),s(t,It,e),s(t,B,e),s(t,xt,e),h(D,t,e),s(t,Ht,e),h(O,t,e),s(t,St,e),s(t,K,e),At=!0},p:ie,i(t){At||(c(J.$$.fragment,t),c(k.$$.fragment,t),c(H.$$.fragment,t),c(A.$$.fragment,t),c(G.$$.fragment,t),c(_.$$.fragment,t),c(Q.$$.fragment,t),c(R.$$.fragment,t),c(N.$$.fragment,t),c(V.$$.fragment,t),c(z.$$.fragment,t),c(X.$$.fragment,t),c(Y.$$.fragment,t),c(F.$$.fragment,t),c(D.$$.fragment,t),c(O.$$.fragment,t),At=!0)},o(t){o(J.$$.fragment,t),o(k.$$.fragment,t),o(H.$$.fragment,t),o(A.$$.fragment,t),o(G.$$.fragment,t),o(_.$$.fragment,t),o(Q.$$.fragment,t),o(R.$$.fragment,t),o(N.$$.fragment,t),o(V.$$.fragment,t),o(z.$$.fragment,t),o(X.$$.fragment,t),o(Y.$$.fragment,t),o(F.$$.fragment,t),o(D.$$.fragment,t),o(O.$$.fragment,t),At=!1},d(t){t&&(l(tt),l(P),l(et),l(lt),l(st),l(I),l(at),l(v),l(nt),l(b),l(it),l(x),l(pt),l(mt),l(S),l(Mt),l(rt),l(d),l(ut),l(ht),l(Z),l(ct),l(ot),l(g),l(yt),l(ft),l($),l(bt),l(Ut),l(C),l(wt),l(vt),l(T),l(dt),l(Zt),l(j),l(gt),l($t),l(E),l(Ct),l(U),l(Tt),l(L),l(jt),l(q),l(Lt),l(Wt),l(W),l(Jt),l(kt),l(It),l(B),l(xt),l(Ht),l(St),l(K)),l(w),y(J,t),y(k,t),y(H,t),y(A,t),y(G,t),y(_,t),y(Q,t),y(R,t),y(N,t),y(V,t),y(z,t),y(X,t),y(Y,t),y(F,t),y(D,t),y(O,t)}}}const ye='{"title":"Evaluate your model with Inspect-AI","local":"evaluate-your-model-with-inspect-ai","sections":[{"title":"Examples","local":"examples","sections":[],"depth":3}],"depth":1}';function fe(_t){return pe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class de extends me{constructor(w){super(),Me(this,w,fe,oe,ne,{})}}export{de as component};
