import{s as rs,n as ys,o as cs}from"../chunks/scheduler.9cda857a.js";import{S as ms,i as Ms,e as m,s as e,c as p,h as Js,a as M,d as a,b as n,f as is,g as i,j as Z,k as os,l as hs,m as l,n as o,t as u,o as r,p as y}from"../chunks/index.003612b0.js";import{C as ws}from"../chunks/CopyLLMTxtMenu.44d08a39.js";import{C as us}from"../chunks/CodeBlock.a90632f1.js";import{H as G,E as ds}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.e5cfda8c.js";function gs(ss){let c,S,Y,v,J,B,h,K,w,ts="We briefly touched upon the difference between lazy and eager evaluation. On this page we will show how the lazy API can be used to get huge performance benefits.",W,d,R,g,as="Polars supports two modes of operation: lazy and eager. In the eager API the query is executed immediately while in the lazy API the query is only evaluated once it’s ‘needed’. Deferring the execution to the last minute can have significant performance advantages and is why the lazy API is preferred in most non-interactive cases.",X,T,F,j,ls="We will be using the example from the previous page to show the performance benefits of using the lazy API. The code below will compute the number of uploads from <code>archive.org</code>.",x,U,Q,f,L,I,N,b,E,C,V,q,es="Running both queries leads to following run times on a regular laptop with a household internet connection:",A,$,ns="<li>Eager: <code>1.96</code> seconds</li> <li>Lazy: <code>410</code> milliseconds</li>",H,_,ps="The lazy query is ~5 times faster than the eager one. The reason for this is the query optimizer: if we delay <code>collect</code>-ing our dataset until the end, Polars will be able to reason about which columns and rows are required and apply filters as early as possible when reading the data. For file formats such as Parquet that contain metadata (e.g. min, max in a certain group of rows) the difference can even be bigger as Polars can skip entire row groups based on the filters and the metadata without sending the data over the wire.",P,k,D,z,O;return J=new ws({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),h=new G({props:{title:"Optimizations",local:"optimizations",headingTag:"h1"}}),d=new G({props:{title:"Lazy vs Eager",local:"lazy-vs-eager",headingTag:"h2"}}),T=new G({props:{title:"Example",local:"example",headingTag:"h2"}}),U=new G({props:{title:"Eager",local:"eager",headingTag:"h3"}}),f=new us({props:{code:"aW1wb3J0JTIwcG9sYXJzJTIwYXMlMjBwbCUwQWltcG9ydCUyMGRhdGV0aW1lJTBBJTBBZGYlMjAlM0QlMjBwbC5yZWFkX2NzdiglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGY29tbW9uY3Jhd2wlMkZzdGF0aXN0aWNzJTJGdGxkcy5jc3YlMjIlMkMlMjB0cnlfcGFyc2VfZGF0ZXMlM0RUcnVlKSUwQSUwQWRmJTIwJTNEJTIwZGYuc2VsZWN0KCUyMnN1ZmZpeCUyMiUyQyUyMCUyMmNyYXdsJTIyJTJDJTIwJTIyZGF0ZSUyMiUyQyUyMCUyMnRsZCUyMiUyQyUyMCUyMnBhZ2VzJTIyJTJDJTIwJTIyZG9tYWlucyUyMiklMEFkZiUyMCUzRCUyMGRmLmZpbHRlciglMEElMjAlMjAlMjAlMjAocGwuY29sKCUyMmRhdGUlMjIpJTIwJTNFJTNEJTIwZGF0ZXRpbWUuZGF0ZSgyMDIwJTJDJTIwMSUyQyUyMDEpKSUyMCU3QyUwQSUyMCUyMCUyMCUyMHBsLmNvbCglMjJjcmF3bCUyMikuc3RyLmNvbnRhaW5zKCUyMkNDJTIyKSUwQSklMEFkZiUyMCUzRCUyMGRmLndpdGhfY29sdW1ucyglMEElMjAlMjAlMjAlMjAocGwuY29sKCUyMnBhZ2VzJTIyKSUyMCUyRiUyMHBsLmNvbCglMjJkb21haW5zJTIyKSkuYWxpYXMoJTIycGFnZXNfcGVyX2RvbWFpbiUyMiklMEEpJTBBZGYlMjAlM0QlMjBkZi5ncm91cF9ieSglMjJ0bGQlMjIlMkMlMjAlMjJkYXRlJTIyKS5hZ2coJTBBJTIwJTIwJTIwJTIwcGwuY29sKCUyMnBhZ2VzJTIyKS5zdW0oKSUyQyUwQSUyMCUyMCUyMCUyMHBsLmNvbCglMjJkb21haW5zJTIyKS5zdW0oKSUyQyUwQSklMEFkZiUyMCUzRCUyMGRmLmdyb3VwX2J5KCUyMnRsZCUyMikuYWdnKCUwQSUyMCUyMCUyMCUyMHBsLmNvbCglMjJkYXRlJTIyKS51bmlxdWUoKS5jb3VudCgpLmFsaWFzKCUyMm51bWJlcl9vZl9zY3JhcGVzJTIyKSUyQyUwQSUyMCUyMCUyMCUyMHBsLmNvbCglMjJkb21haW5zJTIyKS5tZWFuKCkuYWxpYXMoJTIyYXZnX251bWJlcl9vZl9kb21haW5zJTIyKSUyQyUwQSUyMCUyMCUyMCUyMHBsLmNvbCglMjJwYWdlcyUyMikuc29ydF9ieSglMjJkYXRlJTIyKS5wY3RfY2hhbmdlKCkubWVhbigpLmFsaWFzKCUyMmF2Z19wYWdlX2dyb3d0aF9yYXRlJTIyKSUyQyUwQSkuc29ydCglMjJhdmdfbnVtYmVyX29mX2RvbWFpbnMlMjIlMkMlMjBkZXNjZW5kaW5nJTNEVHJ1ZSkuaGVhZCgxMCk=",highlighted:`<span class="hljs-keyword">import</span> polars <span class="hljs-keyword">as</span> pl
<span class="hljs-keyword">import</span> datetime

df = pl.read_csv(<span class="hljs-string">&quot;hf://datasets/commoncrawl/statistics/tlds.csv&quot;</span>, try_parse_dates=<span class="hljs-literal">True</span>)

df = df.select(<span class="hljs-string">&quot;suffix&quot;</span>, <span class="hljs-string">&quot;crawl&quot;</span>, <span class="hljs-string">&quot;date&quot;</span>, <span class="hljs-string">&quot;tld&quot;</span>, <span class="hljs-string">&quot;pages&quot;</span>, <span class="hljs-string">&quot;domains&quot;</span>)
df = df.<span class="hljs-built_in">filter</span>(
    (pl.col(<span class="hljs-string">&quot;date&quot;</span>) &gt;= datetime.date(<span class="hljs-number">2020</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>)) |
    pl.col(<span class="hljs-string">&quot;crawl&quot;</span>).<span class="hljs-built_in">str</span>.contains(<span class="hljs-string">&quot;CC&quot;</span>)
)
df = df.with_columns(
    (pl.col(<span class="hljs-string">&quot;pages&quot;</span>) / pl.col(<span class="hljs-string">&quot;domains&quot;</span>)).alias(<span class="hljs-string">&quot;pages_per_domain&quot;</span>)
)
df = df.group_by(<span class="hljs-string">&quot;tld&quot;</span>, <span class="hljs-string">&quot;date&quot;</span>).agg(
    pl.col(<span class="hljs-string">&quot;pages&quot;</span>).<span class="hljs-built_in">sum</span>(),
    pl.col(<span class="hljs-string">&quot;domains&quot;</span>).<span class="hljs-built_in">sum</span>(),
)
df = df.group_by(<span class="hljs-string">&quot;tld&quot;</span>).agg(
    pl.col(<span class="hljs-string">&quot;date&quot;</span>).unique().count().alias(<span class="hljs-string">&quot;number_of_scrapes&quot;</span>),
    pl.col(<span class="hljs-string">&quot;domains&quot;</span>).mean().alias(<span class="hljs-string">&quot;avg_number_of_domains&quot;</span>),
    pl.col(<span class="hljs-string">&quot;pages&quot;</span>).sort_by(<span class="hljs-string">&quot;date&quot;</span>).pct_change().mean().alias(<span class="hljs-string">&quot;avg_page_growth_rate&quot;</span>),
).sort(<span class="hljs-string">&quot;avg_number_of_domains&quot;</span>, descending=<span class="hljs-literal">True</span>).head(<span class="hljs-number">10</span>)`,wrap:!1}}),I=new G({props:{title:"Lazy",local:"lazy",headingTag:"h3"}}),b=new us({props:{code:"aW1wb3J0JTIwcG9sYXJzJTIwYXMlMjBwbCUwQWltcG9ydCUyMGRhdGV0aW1lJTBBJTBBbGYlMjAlM0QlMjAoJTBBJTIwJTIwJTIwJTIwcGwuc2Nhbl9jc3YoJTIyaGYlM0ElMkYlMkZkYXRhc2V0cyUyRmNvbW1vbmNyYXdsJTJGc3RhdGlzdGljcyUyRnRsZHMuY3N2JTIyJTJDJTIwdHJ5X3BhcnNlX2RhdGVzJTNEVHJ1ZSklMEElMjAlMjAlMjAlMjAuZmlsdGVyKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMChwbC5jb2woJTIyZGF0ZSUyMiklMjAlM0UlM0QlMjBkYXRldGltZS5kYXRlKDIwMjAlMkMlMjAxJTJDJTIwMSkpJTIwJTdDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcGwuY29sKCUyMmNyYXdsJTIyKS5zdHIuY29udGFpbnMoJTIyQ0MlMjIpJTBBJTIwJTIwJTIwJTIwKS53aXRoX2NvbHVtbnMoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKHBsLmNvbCglMjJwYWdlcyUyMiklMjAlMkYlMjBwbC5jb2woJTIyZG9tYWlucyUyMikpLmFsaWFzKCUyMnBhZ2VzX3Blcl9kb21haW4lMjIpJTBBJTIwJTIwJTIwJTIwKS5ncm91cF9ieSglMjJ0bGQlMjIlMkMlMjAlMjJkYXRlJTIyKS5hZ2coJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcGwuY29sKCUyMnBhZ2VzJTIyKS5zdW0oKSUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHBsLmNvbCglMjJkb21haW5zJTIyKS5zdW0oKSUyQyUwQSUyMCUyMCUyMCUyMCkuZ3JvdXBfYnkoJTIydGxkJTIyKS5hZ2coJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcGwuY29sKCUyMmRhdGUlMjIpLnVuaXF1ZSgpLmNvdW50KCkuYWxpYXMoJTIybnVtYmVyX29mX3NjcmFwZXMlMjIpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcGwuY29sKCUyMmRvbWFpbnMlMjIpLm1lYW4oKS5hbGlhcyglMjJhdmdfbnVtYmVyX29mX2RvbWFpbnMlMjIpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcGwuY29sKCUyMnBhZ2VzJTIyKS5zb3J0X2J5KCUyMmRhdGUlMjIpLnBjdF9jaGFuZ2UoKS5tZWFuKCkuYWxpYXMoJTIyYXZnX3BhZ2VfZ3Jvd3RoX3JhdGUlMjIpJTJDJTBBJTIwJTIwJTIwJTIwKS5zb3J0KCUyMmF2Z19udW1iZXJfb2ZfZG9tYWlucyUyMiUyQyUyMGRlc2NlbmRpbmclM0RUcnVlKS5oZWFkKDEwKSUwQSklMEFkZiUyMCUzRCUyMGxmLmNvbGxlY3QoKQ==",highlighted:`<span class="hljs-keyword">import</span> polars <span class="hljs-keyword">as</span> pl
<span class="hljs-keyword">import</span> datetime

lf = (
    pl.scan_csv(<span class="hljs-string">&quot;hf://datasets/commoncrawl/statistics/tlds.csv&quot;</span>, try_parse_dates=<span class="hljs-literal">True</span>)
    .<span class="hljs-built_in">filter</span>(
        (pl.col(<span class="hljs-string">&quot;date&quot;</span>) &gt;= datetime.date(<span class="hljs-number">2020</span>, <span class="hljs-number">1</span>, <span class="hljs-number">1</span>)) |
        pl.col(<span class="hljs-string">&quot;crawl&quot;</span>).<span class="hljs-built_in">str</span>.contains(<span class="hljs-string">&quot;CC&quot;</span>)
    ).with_columns(
        (pl.col(<span class="hljs-string">&quot;pages&quot;</span>) / pl.col(<span class="hljs-string">&quot;domains&quot;</span>)).alias(<span class="hljs-string">&quot;pages_per_domain&quot;</span>)
    ).group_by(<span class="hljs-string">&quot;tld&quot;</span>, <span class="hljs-string">&quot;date&quot;</span>).agg(
        pl.col(<span class="hljs-string">&quot;pages&quot;</span>).<span class="hljs-built_in">sum</span>(),
        pl.col(<span class="hljs-string">&quot;domains&quot;</span>).<span class="hljs-built_in">sum</span>(),
    ).group_by(<span class="hljs-string">&quot;tld&quot;</span>).agg(
        pl.col(<span class="hljs-string">&quot;date&quot;</span>).unique().count().alias(<span class="hljs-string">&quot;number_of_scrapes&quot;</span>),
        pl.col(<span class="hljs-string">&quot;domains&quot;</span>).mean().alias(<span class="hljs-string">&quot;avg_number_of_domains&quot;</span>),
        pl.col(<span class="hljs-string">&quot;pages&quot;</span>).sort_by(<span class="hljs-string">&quot;date&quot;</span>).pct_change().mean().alias(<span class="hljs-string">&quot;avg_page_growth_rate&quot;</span>),
    ).sort(<span class="hljs-string">&quot;avg_number_of_domains&quot;</span>, descending=<span class="hljs-literal">True</span>).head(<span class="hljs-number">10</span>)
)
df = lf.collect()`,wrap:!1}}),C=new G({props:{title:"Timings",local:"timings",headingTag:"h3"}}),k=new ds({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/datasets-polars-optimizations.md"}}),{c(){c=m("meta"),S=e(),Y=m("p"),v=e(),p(J.$$.fragment),B=e(),p(h.$$.fragment),K=e(),w=m("p"),w.textContent=ts,W=e(),p(d.$$.fragment),R=e(),g=m("p"),g.textContent=as,X=e(),p(T.$$.fragment),F=e(),j=m("p"),j.innerHTML=ls,x=e(),p(U.$$.fragment),Q=e(),p(f.$$.fragment),L=e(),p(I.$$.fragment),N=e(),p(b.$$.fragment),E=e(),p(C.$$.fragment),V=e(),q=m("p"),q.textContent=es,A=e(),$=m("ul"),$.innerHTML=ns,H=e(),_=m("p"),_.innerHTML=ps,P=e(),p(k.$$.fragment),D=e(),z=m("p"),this.h()},l(s){const t=Js("svelte-u9bgzb",document.head);c=M(t,"META",{name:!0,content:!0}),t.forEach(a),S=n(s),Y=M(s,"P",{}),is(Y).forEach(a),v=n(s),i(J.$$.fragment,s),B=n(s),i(h.$$.fragment,s),K=n(s),w=M(s,"P",{"data-svelte-h":!0}),Z(w)!=="svelte-o4igm"&&(w.textContent=ts),W=n(s),i(d.$$.fragment,s),R=n(s),g=M(s,"P",{"data-svelte-h":!0}),Z(g)!=="svelte-16m8tgd"&&(g.textContent=as),X=n(s),i(T.$$.fragment,s),F=n(s),j=M(s,"P",{"data-svelte-h":!0}),Z(j)!=="svelte-1qs4ke0"&&(j.innerHTML=ls),x=n(s),i(U.$$.fragment,s),Q=n(s),i(f.$$.fragment,s),L=n(s),i(I.$$.fragment,s),N=n(s),i(b.$$.fragment,s),E=n(s),i(C.$$.fragment,s),V=n(s),q=M(s,"P",{"data-svelte-h":!0}),Z(q)!=="svelte-7wcll"&&(q.textContent=es),A=n(s),$=M(s,"UL",{"data-svelte-h":!0}),Z($)!=="svelte-1ptfoh2"&&($.innerHTML=ns),H=n(s),_=M(s,"P",{"data-svelte-h":!0}),Z(_)!=="svelte-13w92en"&&(_.innerHTML=ps),P=n(s),i(k.$$.fragment,s),D=n(s),z=M(s,"P",{}),is(z).forEach(a),this.h()},h(){os(c,"name","hf:doc:metadata"),os(c,"content",Ts)},m(s,t){hs(document.head,c),l(s,S,t),l(s,Y,t),l(s,v,t),o(J,s,t),l(s,B,t),o(h,s,t),l(s,K,t),l(s,w,t),l(s,W,t),o(d,s,t),l(s,R,t),l(s,g,t),l(s,X,t),o(T,s,t),l(s,F,t),l(s,j,t),l(s,x,t),o(U,s,t),l(s,Q,t),o(f,s,t),l(s,L,t),o(I,s,t),l(s,N,t),o(b,s,t),l(s,E,t),o(C,s,t),l(s,V,t),l(s,q,t),l(s,A,t),l(s,$,t),l(s,H,t),l(s,_,t),l(s,P,t),o(k,s,t),l(s,D,t),l(s,z,t),O=!0},p:ys,i(s){O||(u(J.$$.fragment,s),u(h.$$.fragment,s),u(d.$$.fragment,s),u(T.$$.fragment,s),u(U.$$.fragment,s),u(f.$$.fragment,s),u(I.$$.fragment,s),u(b.$$.fragment,s),u(C.$$.fragment,s),u(k.$$.fragment,s),O=!0)},o(s){r(J.$$.fragment,s),r(h.$$.fragment,s),r(d.$$.fragment,s),r(T.$$.fragment,s),r(U.$$.fragment,s),r(f.$$.fragment,s),r(I.$$.fragment,s),r(b.$$.fragment,s),r(C.$$.fragment,s),r(k.$$.fragment,s),O=!1},d(s){s&&(a(S),a(Y),a(v),a(B),a(K),a(w),a(W),a(R),a(g),a(X),a(F),a(j),a(x),a(Q),a(L),a(N),a(E),a(V),a(q),a(A),a($),a(H),a(_),a(P),a(D),a(z)),a(c),y(J,s),y(h,s),y(d,s),y(T,s),y(U,s),y(f,s),y(I,s),y(b,s),y(C,s),y(k,s)}}}const Ts='{"title":"Optimizations","local":"optimizations","sections":[{"title":"Lazy vs Eager","local":"lazy-vs-eager","sections":[],"depth":2},{"title":"Example","local":"example","sections":[{"title":"Eager","local":"eager","sections":[],"depth":3},{"title":"Lazy","local":"lazy","sections":[],"depth":3},{"title":"Timings","local":"timings","sections":[],"depth":3}],"depth":2}],"depth":1}';function js(ss){return cs(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class qs extends ms{constructor(c){super(),Ms(this,c,js,gs,rs,{})}}export{qs as component};
