import{s as K,n as q,o as O}from"../chunks/scheduler.9cda857a.js";import{S as j,i as R,e as u,s as i,c as E,h as U,a as p,d as n,b as s,f as F,g as X,j as T,k as D,l as W,m as a,n as G,t as S,o as z,p as A}from"../chunks/index.f47f3121.js";import{H as I,E as J}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.64d112fc.js";function N(L){let o,b,m,k,l,w,r,x='Xet-enabled repositories utilize <a href="https://huggingface.co/blog/from-files-to-chunks" rel="nofollow">content-defined chunking (CDC)</a> to deduplicate on the level of bytes (~64KB of data, also referred to as a “chunk”). Each chunk is identified by a rolling hash that determines chunk boundaries based on the actual file contents, making it resilient to insertions or deletions anywhere in the file. When a file is uploaded to a Xet-backed repository using a Xet-aware client, its contents are broken down into these variable-sized chunks. Only new chunks not already present in Xet storage are kept after chunking, everything else is discarded.',v,c,C='To avoid the overhead of communicating and managing at the level of chunks, new chunks are grouped together in <a href="https://huggingface.co/blog/from-chunks-to-blocks#scaling-deduplication-with-aggregation" rel="nofollow">64MB blocks</a> and uploaded. Each block is stored once in a content-addressed store (CAS), keyed by its hash.',y,h,P='The Hub’s <a href="https://huggingface.co/docs/hub/storage-limits#recommendations" rel="nofollow">current recommendation</a> is to limit files to 20GB. At a 64KB chunk size, a 20GB file has 312,500 chunks, many of which go unchanged from version to version. Git LFS is designed to notice only that a file has changed and store the entirety of that revision. By deduplicating at the level of chunks, the Xet backend enables storing only the modified content in a file (which might only be a few KB or MB) and securely deduplicates shared blocks across repositories. For the large binary files found in Model and Dataset repositories, this provides significant improvements to file transfer times.',_,d,B='For more details, refer to the <a href="https://huggingface.co/blog/from-files-to-chunks" rel="nofollow">From Files to Chunks</a> and <a href="https://huggingface.co/blog/from-chunks-to-blocks" rel="nofollow">From Chunks to Blocks</a> blog posts, or the <a href="https://www.cidrdb.org/cidr2023/papers/p43-low.pdf" rel="nofollow">Git is for Data</a> paper by Low et al. that served as the launch point for XetHub prior to being acquired by Hugging Face.',$,f,H,g,M;return l=new I({props:{title:"Deduplication",local:"deduplication",headingTag:"h2"}}),f=new J({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/xet/deduplication.md"}}),{c(){o=u("meta"),b=i(),m=u("p"),k=i(),E(l.$$.fragment),w=i(),r=u("p"),r.innerHTML=x,v=i(),c=u("p"),c.innerHTML=C,y=i(),h=u("p"),h.innerHTML=P,_=i(),d=u("p"),d.innerHTML=B,$=i(),E(f.$$.fragment),H=i(),g=u("p"),this.h()},l(e){const t=U("svelte-u9bgzb",document.head);o=p(t,"META",{name:!0,content:!0}),t.forEach(n),b=s(e),m=p(e,"P",{}),F(m).forEach(n),k=s(e),X(l.$$.fragment,e),w=s(e),r=p(e,"P",{"data-svelte-h":!0}),T(r)!=="svelte-1xuwi99"&&(r.innerHTML=x),v=s(e),c=p(e,"P",{"data-svelte-h":!0}),T(c)!=="svelte-gtfkag"&&(c.innerHTML=C),y=s(e),h=p(e,"P",{"data-svelte-h":!0}),T(h)!=="svelte-bib7sx"&&(h.innerHTML=P),_=s(e),d=p(e,"P",{"data-svelte-h":!0}),T(d)!=="svelte-de2heu"&&(d.innerHTML=B),$=s(e),X(f.$$.fragment,e),H=s(e),g=p(e,"P",{}),F(g).forEach(n),this.h()},h(){D(o,"name","hf:doc:metadata"),D(o,"content",Q)},m(e,t){W(document.head,o),a(e,b,t),a(e,m,t),a(e,k,t),G(l,e,t),a(e,w,t),a(e,r,t),a(e,v,t),a(e,c,t),a(e,y,t),a(e,h,t),a(e,_,t),a(e,d,t),a(e,$,t),G(f,e,t),a(e,H,t),a(e,g,t),M=!0},p:q,i(e){M||(S(l.$$.fragment,e),S(f.$$.fragment,e),M=!0)},o(e){z(l.$$.fragment,e),z(f.$$.fragment,e),M=!1},d(e){e&&(n(b),n(m),n(k),n(w),n(r),n(v),n(c),n(y),n(h),n(_),n(d),n($),n(H),n(g)),n(o),A(l,e),A(f,e)}}}const Q='{"title":"Deduplication","local":"deduplication","sections":[],"depth":2}';function V(L){return O(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class te extends j{constructor(o){super(),R(this,o,V,N,K,{})}}export{te as component};
