import{s as oe,n as re,o as pe}from"../chunks/scheduler.9bc65507.js";import{S as me,i as ce,g as i,s,r as $,A as ue,h as o,f as n,c as a,j as se,u as j,x as v,k as ae,y as de,a as l,v as k,d as R,t as _,w as U}from"../chunks/index.707bf1b6.js";import{C as ie}from"../chunks/CodeBlock.54a9f38d.js";import{H as x}from"../chunks/Heading.342b1fa6.js";function he(Q){let r,B,Z,C,p,G,m,X,c,q='DePlot was proposed in the paper <a href="https://arxiv.org/abs/2212.10505" rel="nofollow">DePlot: One-shot visual language reasoning by plot-to-table translation</a> from Fangyu Liu, Julian Martin Eisenschlos, Francesco Piccinno, Syrine Krichene, Chenxi Pang, Kenton Lee, Mandar Joshi, Wenhu Chen, Nigel Collier, Yasemin Altun.',P,u,K="The abstract of the paper states the following:",V,d,O="<em>Visual language such as charts and plots is ubiquitous in the human world. Comprehending plots and charts requires strong reasoning skills. Prior state-of-the-art (SOTA) models require at least tens of thousands of training examples and their reasoning capabilities are still much limited, especially on complex human-written queries. This paper presents the first one-shot solution to visual language reasoning. We decompose the challenge of visual language reasoning into two steps: (1) plot-to-text translation, and (2) reasoning over the translated text. The key in this method is a modality conversion module, named as DePlot, which translates the image of a plot or chart to a linearized table. The output of DePlot can then be directly used to prompt a pretrained large language model (LLM), exploiting the few-shot reasoning capabilities of LLMs. To obtain DePlot, we standardize the plot-to-table task by establishing unified task formats and metrics, and train DePlot end-to-end on this task. DePlot can then be used off-the-shelf together with LLMs in a plug-and-play fashion. Compared with a SOTA model finetuned on more than &gt;28k data points, DePlot+LLM with just one-shot prompting achieves a 24.0% improvement over finetuned SOTA on human-written queries from the task of chart QA.</em>",Y,h,H,f,ee=`DePlot is a model that is trained using <code>Pix2Struct</code> architecture. You can find more information about <code>Pix2Struct</code> in the <a href="https://huggingface.co/docs/transformers/main/en/model_doc/pix2struct" rel="nofollow">Pix2Struct documentation</a>.
DePlot is a Visual Question Answering subset of <code>Pix2Struct</code> architecture. It renders the input question on the image and predicts the answer.`,I,g,z,M,te="Currently one checkpoint is available for DePlot:",F,y,ne="<li><code>google/deplot</code>: DePlot fine-tuned on ChartQA dataset</li>",D,w,E,b,L,J,le='To fine-tune DePlot, refer to the pix2struct <a href="https://github.com/huggingface/notebooks/blob/main/examples/image_captioning_pix2struct.ipynb" rel="nofollow">fine-tuning notebook</a>. For <code>Pix2Struct</code> models, we have found out that fine-tuning the model with Adafactor and cosine learning rate scheduler leads to faster convergence:',N,T,S,W,A;return p=new x({props:{title:"DePlot",local:"deplot",headingTag:"h1"}}),m=new x({props:{title:"Overview",local:"overview",headingTag:"h2"}}),h=new x({props:{title:"Model description",local:"model-description",headingTag:"h2"}}),g=new x({props:{title:"Usage",local:"usage",headingTag:"h2"}}),w=new ie({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Qcm9jZXNzb3IlMkMlMjBQaXgyU3RydWN0Rm9yQ29uZGl0aW9uYWxHZW5lcmF0aW9uJTBBaW1wb3J0JTIwcmVxdWVzdHMlMEFmcm9tJTIwUElMJTIwaW1wb3J0JTIwSW1hZ2UlMEElMEFtb2RlbCUyMCUzRCUyMFBpeDJTdHJ1Y3RGb3JDb25kaXRpb25hbEdlbmVyYXRpb24uZnJvbV9wcmV0cmFpbmVkKCUyMmdvb2dsZSUyRmRlcGxvdCUyMiklMEFwcm9jZXNzb3IlMjAlM0QlMjBBdXRvUHJvY2Vzc29yLmZyb21fcHJldHJhaW5lZCglMjJnb29nbGUlMkZkZXBsb3QlMjIpJTBBdXJsJTIwJTNEJTIwJTIyaHR0cHMlM0ElMkYlMkZyYXcuZ2l0aHVidXNlcmNvbnRlbnQuY29tJTJGdmlzLW5scCUyRkNoYXJ0UUElMkZtYWluJTJGQ2hhcnRRQSUyNTIwRGF0YXNldCUyRnZhbCUyRnBuZyUyRjUwOTAucG5nJTIyJTBBaW1hZ2UlMjAlM0QlMjBJbWFnZS5vcGVuKHJlcXVlc3RzLmdldCh1cmwlMkMlMjBzdHJlYW0lM0RUcnVlKS5yYXcpJTBBJTBBaW5wdXRzJTIwJTNEJTIwcHJvY2Vzc29yKGltYWdlcyUzRGltYWdlJTJDJTIwdGV4dCUzRCUyMkdlbmVyYXRlJTIwdW5kZXJseWluZyUyMGRhdGElMjB0YWJsZSUyMG9mJTIwdGhlJTIwZmlndXJlJTIwYmVsb3clM0ElMjIlMkMlMjByZXR1cm5fdGVuc29ycyUzRCUyMnB0JTIyKSUwQXByZWRpY3Rpb25zJTIwJTNEJTIwbW9kZWwuZ2VuZXJhdGUoKippbnB1dHMlMkMlMjBtYXhfbmV3X3Rva2VucyUzRDUxMiklMEFwcmludChwcm9jZXNzb3IuZGVjb2RlKHByZWRpY3Rpb25zJTVCMCU1RCUyQyUyMHNraXBfc3BlY2lhbF90b2tlbnMlM0RUcnVlKSk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoProcessor, Pix2StructForConditionalGeneration
<span class="hljs-keyword">import</span> requests
<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image

model = Pix2StructForConditionalGeneration.from_pretrained(<span class="hljs-string">&quot;google/deplot&quot;</span>)
processor = AutoProcessor.from_pretrained(<span class="hljs-string">&quot;google/deplot&quot;</span>)
url = <span class="hljs-string">&quot;https://raw.githubusercontent.com/vis-nlp/ChartQA/main/ChartQA%20Dataset/val/png/5090.png&quot;</span>
image = Image.<span class="hljs-built_in">open</span>(requests.get(url, stream=<span class="hljs-literal">True</span>).raw)

inputs = processor(images=image, text=<span class="hljs-string">&quot;Generate underlying data table of the figure below:&quot;</span>, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>)
predictions = model.generate(**inputs, max_new_tokens=<span class="hljs-number">512</span>)
<span class="hljs-built_in">print</span>(processor.decode(predictions[<span class="hljs-number">0</span>], skip_special_tokens=<span class="hljs-literal">True</span>))`,wrap:!1}}),b=new x({props:{title:"Fine-tuning",local:"fine-tuning",headingTag:"h2"}}),T=new ie({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycy5vcHRpbWl6YXRpb24lMjBpbXBvcnQlMjBBZGFmYWN0b3IlMkMlMjBnZXRfY29zaW5lX3NjaGVkdWxlX3dpdGhfd2FybXVwJTBBJTBBb3B0aW1pemVyJTIwJTNEJTIwQWRhZmFjdG9yKHNlbGYucGFyYW1ldGVycygpJTJDJTIwc2NhbGVfcGFyYW1ldGVyJTNERmFsc2UlMkMlMjByZWxhdGl2ZV9zdGVwJTNERmFsc2UlMkMlMjBsciUzRDAuMDElMkMlMjB3ZWlnaHRfZGVjYXklM0QxZS0wNSklMEFzY2hlZHVsZXIlMjAlM0QlMjBnZXRfY29zaW5lX3NjaGVkdWxlX3dpdGhfd2FybXVwKG9wdGltaXplciUyQyUyMG51bV93YXJtdXBfc3RlcHMlM0QxMDAwJTJDJTIwbnVtX3RyYWluaW5nX3N0ZXBzJTNENDAwMDAp",highlighted:`<span class="hljs-keyword">from</span> transformers.optimization <span class="hljs-keyword">import</span> Adafactor, get_cosine_schedule_with_warmup

optimizer = Adafactor(self.parameters(), scale_parameter=<span class="hljs-literal">False</span>, relative_step=<span class="hljs-literal">False</span>, lr=<span class="hljs-number">0.01</span>, weight_decay=<span class="hljs-number">1e-05</span>)
scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=<span class="hljs-number">1000</span>, num_training_steps=<span class="hljs-number">40000</span>)`,wrap:!1}}),{c(){r=i("meta"),B=s(),Z=i("p"),C=s(),$(p.$$.fragment),G=s(),$(m.$$.fragment),X=s(),c=i("p"),c.innerHTML=q,P=s(),u=i("p"),u.textContent=K,V=s(),d=i("p"),d.innerHTML=O,Y=s(),$(h.$$.fragment),H=s(),f=i("p"),f.innerHTML=ee,I=s(),$(g.$$.fragment),z=s(),M=i("p"),M.textContent=te,F=s(),y=i("ul"),y.innerHTML=ne,D=s(),$(w.$$.fragment),E=s(),$(b.$$.fragment),L=s(),J=i("p"),J.innerHTML=le,N=s(),$(T.$$.fragment),S=s(),W=i("p"),this.h()},l(e){const t=ue("svelte-u9bgzb",document.head);r=o(t,"META",{name:!0,content:!0}),t.forEach(n),B=a(e),Z=o(e,"P",{}),se(Z).forEach(n),C=a(e),j(p.$$.fragment,e),G=a(e),j(m.$$.fragment,e),X=a(e),c=o(e,"P",{"data-svelte-h":!0}),v(c)!=="svelte-19mspuy"&&(c.innerHTML=q),P=a(e),u=o(e,"P",{"data-svelte-h":!0}),v(u)!=="svelte-1pvthah"&&(u.textContent=K),V=a(e),d=o(e,"P",{"data-svelte-h":!0}),v(d)!=="svelte-tgrbdp"&&(d.innerHTML=O),Y=a(e),j(h.$$.fragment,e),H=a(e),f=o(e,"P",{"data-svelte-h":!0}),v(f)!=="svelte-1ihm3oy"&&(f.innerHTML=ee),I=a(e),j(g.$$.fragment,e),z=a(e),M=o(e,"P",{"data-svelte-h":!0}),v(M)!=="svelte-1010knq"&&(M.textContent=te),F=a(e),y=o(e,"UL",{"data-svelte-h":!0}),v(y)!=="svelte-kycb4s"&&(y.innerHTML=ne),D=a(e),j(w.$$.fragment,e),E=a(e),j(b.$$.fragment,e),L=a(e),J=o(e,"P",{"data-svelte-h":!0}),v(J)!=="svelte-11vjzlg"&&(J.innerHTML=le),N=a(e),j(T.$$.fragment,e),S=a(e),W=o(e,"P",{}),se(W).forEach(n),this.h()},h(){ae(r,"name","hf:doc:metadata"),ae(r,"content",fe)},m(e,t){de(document.head,r),l(e,B,t),l(e,Z,t),l(e,C,t),k(p,e,t),l(e,G,t),k(m,e,t),l(e,X,t),l(e,c,t),l(e,P,t),l(e,u,t),l(e,V,t),l(e,d,t),l(e,Y,t),k(h,e,t),l(e,H,t),l(e,f,t),l(e,I,t),k(g,e,t),l(e,z,t),l(e,M,t),l(e,F,t),l(e,y,t),l(e,D,t),k(w,e,t),l(e,E,t),k(b,e,t),l(e,L,t),l(e,J,t),l(e,N,t),k(T,e,t),l(e,S,t),l(e,W,t),A=!0},p:re,i(e){A||(R(p.$$.fragment,e),R(m.$$.fragment,e),R(h.$$.fragment,e),R(g.$$.fragment,e),R(w.$$.fragment,e),R(b.$$.fragment,e),R(T.$$.fragment,e),A=!0)},o(e){_(p.$$.fragment,e),_(m.$$.fragment,e),_(h.$$.fragment,e),_(g.$$.fragment,e),_(w.$$.fragment,e),_(b.$$.fragment,e),_(T.$$.fragment,e),A=!1},d(e){e&&(n(B),n(Z),n(C),n(G),n(X),n(c),n(P),n(u),n(V),n(d),n(Y),n(H),n(f),n(I),n(z),n(M),n(F),n(y),n(D),n(E),n(L),n(J),n(N),n(S),n(W)),n(r),U(p,e),U(m,e),U(h,e),U(g,e),U(w,e),U(b,e),U(T,e)}}}const fe='{"title":"DePlot","local":"deplot","sections":[{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Model description","local":"model-description","sections":[],"depth":2},{"title":"Usage","local":"usage","sections":[],"depth":2},{"title":"Fine-tuning","local":"fine-tuning","sections":[],"depth":2}],"depth":1}';function ge(Q){return pe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Je extends me{constructor(r){super(),ce(this,r,ge,he,oe,{})}}export{Je as component};
