import{s as O,n as j,o as R}from"../chunks/scheduler.9cda857a.js";import{S as U,i as W,e as p,s as o,c as C,h as I,a as m,d as n,b as s,f as K,g as P,j as T,k as q,l as J,m as i,n as B,t as E,o as F,p as S}from"../chunks/index.003612b0.js";import{C as N}from"../chunks/CopyLLMTxtMenu.44d08a39.js";import{H as Q,E as V}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.e5cfda8c.js";function Y(X){let a,k,g,w,l,y,r,$,c,z='Xet-enabled repositories utilize <a href="https://huggingface.co/blog/from-files-to-chunks" rel="nofollow">content-defined chunking (CDC)</a> to deduplicate on the level of bytes (~64KB of data, also referred to as a “chunk”). Each chunk is identified by a rolling hash that determines chunk boundaries based on the actual file contents, making it resilient to insertions or deletions anywhere in the file. When a file is uploaded to a Xet-backed repository using a Xet-aware client, its contents are broken down into these variable-sized chunks. Only new chunks not already present in Xet storage are kept after chunking, everything else is discarded.',v,h,D='To avoid the overhead of communicating and managing at the level of chunks, new chunks are grouped together in <a href="https://huggingface.co/blog/from-chunks-to-blocks#scaling-deduplication-with-aggregation" rel="nofollow">64MB blocks</a> and uploaded. Each block is stored once in a content-addressed store (CAS), keyed by its hash.',_,f,G='The Hub’s <a href="https://huggingface.co/docs/hub/storage-limits#recommendations" rel="nofollow">current recommendation</a> is to limit files to 20GB. At a 64KB chunk size, a 20GB file has 312,500 chunks, many of which go unchanged from version to version. Git LFS is designed to notice only that a file has changed and store the entirety of that revision. By deduplicating at the level of chunks, the Xet backend enables storing only the modified content in a file (which might only be a few KB or MB) and securely deduplicates shared blocks across repositories. For the large binary files found in Model and Dataset repositories, this provides significant improvements to file transfer times.',x,d,A='For more details, refer to the <a href="https://huggingface.co/blog/from-files-to-chunks" rel="nofollow">From Files to Chunks</a> and <a href="https://huggingface.co/blog/from-chunks-to-blocks" rel="nofollow">From Chunks to Blocks</a> blog posts, or the <a href="https://www.cidrdb.org/cidr2023/papers/p43-low.pdf" rel="nofollow">Git is for Data</a> paper by Low et al. that served as the launch point for XetHub prior to being acquired by Hugging Face.',M,u,H,b,L;return l=new N({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),r=new Q({props:{title:"Deduplication",local:"deduplication",headingTag:"h1"}}),u=new V({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/xet/deduplication.md"}}),{c(){a=p("meta"),k=o(),g=p("p"),w=o(),C(l.$$.fragment),y=o(),C(r.$$.fragment),$=o(),c=p("p"),c.innerHTML=z,v=o(),h=p("p"),h.innerHTML=D,_=o(),f=p("p"),f.innerHTML=G,x=o(),d=p("p"),d.innerHTML=A,M=o(),C(u.$$.fragment),H=o(),b=p("p"),this.h()},l(e){const t=I("svelte-u9bgzb",document.head);a=m(t,"META",{name:!0,content:!0}),t.forEach(n),k=s(e),g=m(e,"P",{}),K(g).forEach(n),w=s(e),P(l.$$.fragment,e),y=s(e),P(r.$$.fragment,e),$=s(e),c=m(e,"P",{"data-svelte-h":!0}),T(c)!=="svelte-1xuwi99"&&(c.innerHTML=z),v=s(e),h=m(e,"P",{"data-svelte-h":!0}),T(h)!=="svelte-gtfkag"&&(h.innerHTML=D),_=s(e),f=m(e,"P",{"data-svelte-h":!0}),T(f)!=="svelte-bib7sx"&&(f.innerHTML=G),x=s(e),d=m(e,"P",{"data-svelte-h":!0}),T(d)!=="svelte-de2heu"&&(d.innerHTML=A),M=s(e),P(u.$$.fragment,e),H=s(e),b=m(e,"P",{}),K(b).forEach(n),this.h()},h(){q(a,"name","hf:doc:metadata"),q(a,"content",Z)},m(e,t){J(document.head,a),i(e,k,t),i(e,g,t),i(e,w,t),B(l,e,t),i(e,y,t),B(r,e,t),i(e,$,t),i(e,c,t),i(e,v,t),i(e,h,t),i(e,_,t),i(e,f,t),i(e,x,t),i(e,d,t),i(e,M,t),B(u,e,t),i(e,H,t),i(e,b,t),L=!0},p:j,i(e){L||(E(l.$$.fragment,e),E(r.$$.fragment,e),E(u.$$.fragment,e),L=!0)},o(e){F(l.$$.fragment,e),F(r.$$.fragment,e),F(u.$$.fragment,e),L=!1},d(e){e&&(n(k),n(g),n(w),n(y),n($),n(c),n(v),n(h),n(_),n(f),n(x),n(d),n(M),n(H),n(b)),n(a),S(l,e),S(r,e),S(u,e)}}}const Z='{"title":"Deduplication","local":"deduplication","sections":[],"depth":1}';function ee(X){return R(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class oe extends U{constructor(a){super(),W(this,a,ee,Y,O,{})}}export{oe as component};
