import{s as Le,n as Ze,o as Xe}from"../chunks/scheduler.9bc65507.js";import{S as Re,i as We,g as i,s,r,A as Qe,h as o,f as l,c as a,j as $e,u as M,x as p,k as Ee,y as He,a as n,v as w,d as y,t as h,w as c}from"../chunks/index.707bf1b6.js";import{C as pt}from"../chunks/CodeBlock.54a9f38d.js";import{H as rt}from"../chunks/Heading.342b1fa6.js";function Ye(ne){let m,Mt,it,wt,d,yt,U,se='The hardware you use to run model training and inference can have a big effect on performance. For a deep dive into GPUs make sure to check out Tim Dettmer’s excellent <a href="https://timdettmers.com/2020/09/07/which-gpu-for-deep-learning/" rel="nofollow">blog post</a>.',ht,u,ae="Let’s have a look at some practical advice for GPU setups.",ct,C,mt,T,ie="When you train bigger models you have essentially three options:",dt,f,oe='<li>bigger GPUs</li> <li>more GPUs</li> <li>more CPU and NVMe (offloaded to by <a href="main_classes/deepspeed#nvme-support">DeepSpeed-Infinity</a>)</li>',Ut,J,pe="Let’s start at the case where you have a single GPU.",ut,I,Ct,v,re="If you bought an expensive high end GPU make sure you give it the correct power and sufficient cooling.",Tt,b,Me="<strong>Power</strong>:",ft,j,we="Some high end consumer GPU cards have 2 and sometimes 3 PCI-E 8-Pin power sockets. Make sure you have as many independent 12V PCI-E 8-Pin cables plugged into the card as there are sockets. Do not use the 2 splits at one end of the same cable (also known as pigtail cable). That is if you have 2 sockets on the GPU, you want 2 PCI-E 8-Pin cables going from your PSU to the card and not one that has 2 PCI-E 8-Pin connectors at the end! You won’t get the full performance out of your card otherwise.",Jt,x,ye="Each PCI-E 8-Pin power cable needs to be plugged into a 12V rail on the PSU side and can supply up to 150W of power.",It,P,he="Some other cards may use a PCI-E 12-Pin connectors, and these can deliver up to 500-600W of power.",vt,k,ce="Low end cards may use 6-Pin connectors, which supply up to 75W of power.",bt,B,me="Additionally you want the high-end PSU that has stable voltage. Some lower quality ones may not give the card the stable voltage it needs to function at its peak.",jt,g,de="And of course the PSU needs to have enough unused Watts to power the card.",xt,_,Ue="<strong>Cooling</strong>:",Pt,N,ue="When a GPU gets overheated it will start throttling down and will not deliver full performance and it can even shutdown if it gets too hot.",kt,V,Ce="It’s hard to tell the exact best temperature to strive for when a GPU is heavily loaded, but probably anything under +80C is good, but lower is better - perhaps 70-75C is an excellent range to be in. The throttling down is likely to start at around 84-90C. But other than throttling performance a prolonged very high temperature is likely to reduce the lifespan of a GPU.",Bt,G,Te="Next let’s have a look at one of the most important aspects when having multiple GPUs: connectivity.",gt,A,_t,S,fe="If you use multiple GPUs the way cards are inter-connected can have a huge impact on the total training time. If the GPUs are on the same physical node, you can run:",Nt,$,Vt,E,Je="and it will tell you how the GPUs are inter-connected. On a machine with dual-GPU and which are connected with NVLink, you will most likely see something like:",Gt,L,At,Z,Ie="on a different machine w/o NVLink we may see:",St,X,$t,R,ve="The report includes this legend:",Et,W,Lt,Q,be="So the first report <code>NV2</code> tells us the GPUs are interconnected with 2 NVLinks, and the second report <code>PHB</code> we have a typical consumer-level PCIe+Bridge setup.",Zt,H,je="Check what type of connectivity you have on your setup. Some of these will make the communication between cards faster (e.g. NVLink), others slower (e.g. PHB).",Xt,Y,xe="Depending on the type of scalability solution used, the connectivity speed could have a major or a minor impact. If the GPUs need to sync rarely, as in DDP, the impact of a slower connection will be less significant. If the GPUs need to send messages to each other often, as in ZeRO-DP, then faster connectivity becomes super important to achieve faster training.",Rt,F,Wt,z,Pe='<a href="https://en.wikipedia.org/wiki/NVLink" rel="nofollow">NVLink</a> is a wire-based serial multi-lane near-range communications link developed by Nvidia.',Qt,D,ke='Each new generation provides a faster bandwidth, e.g. here is a quote from <a href="https://www.nvidia.com/content/dam/en-zz/Solutions/geforce/ampere/pdf/NVIDIA-ampere-GA102-GPU-Architecture-Whitepaper-V1.pdf" rel="nofollow">Nvidia Ampere GA102 GPU Architecture</a>:',Ht,O,Be=`<p>Third-Generation NVLink®
GA102 GPUs utilize NVIDIA’s third-generation NVLink interface, which includes four x4 links,
with each link providing 14.0625 GB/sec bandwidth in each direction between two GPUs. Four
links provide 56.25 GB/sec bandwidth in each direction, and 112.5 GB/sec total bandwidth
between two GPUs. Two RTX 3090 GPUs can be connected together for SLI using NVLink.
(Note that 3-Way and 4-Way SLI configurations are not supported.)</p>`,Yt,q,ge="So the higher <code>X</code> you get in the report of <code>NVX</code> in the output of <code>nvidia-smi topo -m</code> the better. The generation will depend on your GPU architecture.",Ft,K,_e="Let’s compare the execution of a gpt2 language model training over a small sample of wikitext.",zt,tt,Ne="The results are:",Dt,et,Ve='<thead><tr><th>NVlink</th> <th align="right">Time</th></tr></thead> <tbody><tr><td>Y</td> <td align="right">101s</td></tr> <tr><td>N</td> <td align="right">131s</td></tr></tbody>',Ot,lt,Ge="You can see that NVLink completes the training ~23% faster. In the second benchmark we use <code>NCCL_P2P_DISABLE=1</code> to tell the GPUs not to use NVLink.",qt,nt,Ae="Here is the full benchmark code and outputs:",Kt,st,te,at,Se=`Hardware: 2x TITAN RTX 24GB each + NVlink with 2 NVLinks (<code>NV2</code> in <code>nvidia-smi topo -m</code>)
Software: <code>pytorch-1.8-to-be</code> + <code>cuda-11.0</code> / <code>transformers==4.3.0.dev0</code>`,ee,ot,le;return d=new rt({props:{title:"Custom hardware for training",local:"custom-hardware-for-training",headingTag:"h1"}}),C=new rt({props:{title:"GPU",local:"gpu",headingTag:"h2"}}),I=new rt({props:{title:"Power and Cooling",local:"power-and-cooling",headingTag:"h3"}}),A=new rt({props:{title:"Multi-GPU Connectivity",local:"multi-gpu-connectivity",headingTag:"h3"}}),$=new pt({props:{code:"bnZpZGlhLXNtaSUyMHRvcG8lMjAtbQ==",highlighted:'<span class="hljs-symbol">nvidia</span>-<span class="hljs-keyword">smi</span> topo -m',wrap:!1}}),L=new pt({props:{code:"JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwR1BVMCUyMCUyMCUyMCUyMEdQVTElMjAlMjAlMjAlMjBDUFUlMjBBZmZpbml0eSUyMCUyMCUyMCUyME5VTUElMjBBZmZpbml0eSUwQUdQVTAlMjAlMjAlMjAlMjAlMjBYJTIwJTIwJTIwJTIwJTIwJTIwTlYyJTIwJTIwJTIwJTIwJTIwMC0yMyUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyME4lMkZBJTBBR1BVMSUyMCUyMCUyMCUyME5WMiUyMCUyMCUyMCUyMCUyMCUyMFglMjAlMjAlMjAlMjAlMjAlMjAwLTIzJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTiUyRkE=",highlighted:`        <span class="hljs-attribute">GPU0</span>    GPU1    CPU Affinity    NUMA Affinity
<span class="hljs-attribute">GPU0</span>     X      NV2     <span class="hljs-number">0</span>-<span class="hljs-number">23</span>            N/A
<span class="hljs-attribute">GPU1</span>    NV2      X      <span class="hljs-number">0</span>-<span class="hljs-number">23</span>            N/A`,wrap:!1}}),X=new pt({props:{code:"JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwR1BVMCUyMCUyMCUyMCUyMEdQVTElMjAlMjAlMjAlMjBDUFUlMjBBZmZpbml0eSUyMCUyMCUyMCUyME5VTUElMjBBZmZpbml0eSUwQUdQVTAlMjAlMjAlMjAlMjAlMjBYJTIwJTIwJTIwJTIwJTIwJTIwUEhCJTIwJTIwJTIwJTIwJTIwMC0xMSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyME4lMkZBJTBBR1BVMSUyMCUyMCUyMCUyMFBIQiUyMCUyMCUyMCUyMCUyMCUyMFglMjAlMjAlMjAlMjAlMjAlMjAwLTExJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwTiUyRkE=",highlighted:`        <span class="hljs-attribute">GPU0</span>    GPU1    CPU Affinity    NUMA Affinity
<span class="hljs-attribute">GPU0</span>     X      PHB     <span class="hljs-number">0</span>-<span class="hljs-number">11</span>            N/A
<span class="hljs-attribute">GPU1</span>    PHB      X      <span class="hljs-number">0</span>-<span class="hljs-number">11</span>            N/A`,wrap:!1}}),W=new pt({props:{code:"JTIwJTIwWCUyMCUyMCUyMCUyMCUzRCUyMFNlbGYlMEElMjAlMjBTWVMlMjAlMjAlM0QlMjBDb25uZWN0aW9uJTIwdHJhdmVyc2luZyUyMFBDSWUlMjBhcyUyMHdlbGwlMjBhcyUyMHRoZSUyMFNNUCUyMGludGVyY29ubmVjdCUyMGJldHdlZW4lMjBOVU1BJTIwbm9kZXMlMjAoZS5nLiUyQyUyMFFQSSUyRlVQSSklMEElMjAlMjBOT0RFJTIwJTNEJTIwQ29ubmVjdGlvbiUyMHRyYXZlcnNpbmclMjBQQ0llJTIwYXMlMjB3ZWxsJTIwYXMlMjB0aGUlMjBpbnRlcmNvbm5lY3QlMjBiZXR3ZWVuJTIwUENJZSUyMEhvc3QlMjBCcmlkZ2VzJTIwd2l0aGluJTIwYSUyME5VTUElMjBub2RlJTBBJTIwJTIwUEhCJTIwJTIwJTNEJTIwQ29ubmVjdGlvbiUyMHRyYXZlcnNpbmclMjBQQ0llJTIwYXMlMjB3ZWxsJTIwYXMlMjBhJTIwUENJZSUyMEhvc3QlMjBCcmlkZ2UlMjAodHlwaWNhbGx5JTIwdGhlJTIwQ1BVKSUwQSUyMCUyMFBYQiUyMCUyMCUzRCUyMENvbm5lY3Rpb24lMjB0cmF2ZXJzaW5nJTIwbXVsdGlwbGUlMjBQQ0llJTIwYnJpZGdlcyUyMCh3aXRob3V0JTIwdHJhdmVyc2luZyUyMHRoZSUyMFBDSWUlMjBIb3N0JTIwQnJpZGdlKSUwQSUyMCUyMFBJWCUyMCUyMCUzRCUyMENvbm5lY3Rpb24lMjB0cmF2ZXJzaW5nJTIwYXQlMjBtb3N0JTIwYSUyMHNpbmdsZSUyMFBDSWUlMjBicmlkZ2UlMEElMjAlMjBOViUyMyUyMCUyMCUzRCUyMENvbm5lY3Rpb24lMjB0cmF2ZXJzaW5nJTIwYSUyMGJvbmRlZCUyMHNldCUyMG9mJTIwJTIzJTIwTlZMaW5rcw==",highlighted:`  X    = Self
  SYS  = Connection traversing PCIe <span class="hljs-keyword">as</span> well <span class="hljs-keyword">as</span> <span class="hljs-keyword">the</span> SMP interconnect between NUMA nodes (e.g., QPI/UPI)
  NODE = Connection traversing PCIe <span class="hljs-keyword">as</span> well <span class="hljs-keyword">as</span> <span class="hljs-keyword">the</span> interconnect between PCIe Host Bridges <span class="hljs-keyword">within</span> <span class="hljs-keyword">a</span> NUMA node
  PHB  = Connection traversing PCIe <span class="hljs-keyword">as</span> well <span class="hljs-keyword">as</span> <span class="hljs-keyword">a</span> PCIe Host Bridge (typically <span class="hljs-keyword">the</span> CPU)
  PXB  = Connection traversing multiple PCIe bridges (<span class="hljs-keyword">without</span> traversing <span class="hljs-keyword">the</span> PCIe Host Bridge)
  PIX  = Connection traversing <span class="hljs-keyword">at</span> most <span class="hljs-keyword">a</span> single PCIe bridge
  NV<span class="hljs-comment">#  = Connection traversing a bonded set of # NVLinks</span>`,wrap:!1}}),F=new rt({props:{title:"NVlink",local:"nvlink",headingTag:"h4"}}),st=new pt({props:{code:"JTIzJTIwRERQJTIwdyUyRiUyME5WTGluayUwQSUwQXJtJTIwLXIlMjAlMkZ0bXAlMkZ0ZXN0LWNsbSUzQiUyMENVREFfVklTSUJMRV9ERVZJQ0VTJTNEMCUyQzElMjBweXRob24lMjAtbSUyMHRvcmNoLmRpc3RyaWJ1dGVkLmxhdW5jaCUyMCU1QyUwQS0tbnByb2NfcGVyX25vZGUlMjAyJTIwZXhhbXBsZXMlMkZweXRvcmNoJTJGbGFuZ3VhZ2UtbW9kZWxpbmclMkZydW5fY2xtLnB5JTIwLS1tb2RlbF9uYW1lX29yX3BhdGglMjBncHQyJTIwJTVDJTBBLS1kYXRhc2V0X25hbWUlMjB3aWtpdGV4dCUyMC0tZGF0YXNldF9jb25maWdfbmFtZSUyMHdpa2l0ZXh0LTItcmF3LXYxJTIwLS1kb190cmFpbiUyMCU1QyUwQS0tb3V0cHV0X2RpciUyMCUyRnRtcCUyRnRlc3QtY2xtJTIwLS1wZXJfZGV2aWNlX3RyYWluX2JhdGNoX3NpemUlMjA0JTIwLS1tYXhfc3RlcHMlMjAyMDAlMEElMEElN0IndHJhaW5fcnVudGltZSclM0ElMjAxMDEuOTAwMyUyQyUyMCd0cmFpbl9zYW1wbGVzX3Blcl9zZWNvbmQnJTNBJTIwMS45NjMlMkMlMjAnZXBvY2gnJTNBJTIwMC42OSU3RCUwQSUwQSUyMyUyMEREUCUyMHclMkZvJTIwTlZMaW5rJTBBJTBBcm0lMjAtciUyMCUyRnRtcCUyRnRlc3QtY2xtJTNCJTIwQ1VEQV9WSVNJQkxFX0RFVklDRVMlM0QwJTJDMSUyME5DQ0xfUDJQX0RJU0FCTEUlM0QxJTIwcHl0aG9uJTIwLW0lMjB0b3JjaC5kaXN0cmlidXRlZC5sYXVuY2glMjAlNUMlMEEtLW5wcm9jX3Blcl9ub2RlJTIwMiUyMGV4YW1wbGVzJTJGcHl0b3JjaCUyRmxhbmd1YWdlLW1vZGVsaW5nJTJGcnVuX2NsbS5weSUyMC0tbW9kZWxfbmFtZV9vcl9wYXRoJTIwZ3B0MiUyMCU1QyUwQS0tZGF0YXNldF9uYW1lJTIwd2lraXRleHQlMjAtLWRhdGFzZXRfY29uZmlnX25hbWUlMjB3aWtpdGV4dC0yLXJhdy12MSUyMC0tZG9fdHJhaW4lMEEtLW91dHB1dF9kaXIlMjAlMkZ0bXAlMkZ0ZXN0LWNsbSUyMC0tcGVyX2RldmljZV90cmFpbl9iYXRjaF9zaXplJTIwNCUyMC0tbWF4X3N0ZXBzJTIwMjAwJTBBJTBBJTdCJ3RyYWluX3J1bnRpbWUnJTNBJTIwMTMxLjQzNjclMkMlMjAndHJhaW5fc2FtcGxlc19wZXJfc2Vjb25kJyUzQSUyMDEuNTIyJTJDJTIwJ2Vwb2NoJyUzQSUyMDAuNjklN0Q=",highlighted:`<span class="hljs-comment"># DDP w/ NVLink</span>

<span class="hljs-built_in">rm</span> -r /tmp/test-clm; CUDA_VISIBLE_DEVICES=0,1 python -m torch.distributed.launch \\
--nproc_per_node 2 examples/pytorch/language-modeling/run_clm.py --model_name_or_path gpt2 \\
--dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --do_train \\
--output_dir /tmp/test-clm --per_device_train_batch_size 4 --max_steps 200

{<span class="hljs-string">&#x27;train_runtime&#x27;</span>: 101.9003, <span class="hljs-string">&#x27;train_samples_per_second&#x27;</span>: 1.963, <span class="hljs-string">&#x27;epoch&#x27;</span>: 0.69}

<span class="hljs-comment"># DDP w/o NVLink</span>

<span class="hljs-built_in">rm</span> -r /tmp/test-clm; CUDA_VISIBLE_DEVICES=0,1 NCCL_P2P_DISABLE=1 python -m torch.distributed.launch \\
--nproc_per_node 2 examples/pytorch/language-modeling/run_clm.py --model_name_or_path gpt2 \\
--dataset_name wikitext --dataset_config_name wikitext-2-raw-v1 --do_train
--output_dir /tmp/test-clm --per_device_train_batch_size 4 --max_steps 200

{<span class="hljs-string">&#x27;train_runtime&#x27;</span>: 131.4367, <span class="hljs-string">&#x27;train_samples_per_second&#x27;</span>: 1.522, <span class="hljs-string">&#x27;epoch&#x27;</span>: 0.69}`,wrap:!1}}),{c(){m=i("meta"),Mt=s(),it=i("p"),wt=s(),r(d.$$.fragment),yt=s(),U=i("p"),U.innerHTML=se,ht=s(),u=i("p"),u.textContent=ae,ct=s(),r(C.$$.fragment),mt=s(),T=i("p"),T.textContent=ie,dt=s(),f=i("ul"),f.innerHTML=oe,Ut=s(),J=i("p"),J.textContent=pe,ut=s(),r(I.$$.fragment),Ct=s(),v=i("p"),v.textContent=re,Tt=s(),b=i("p"),b.innerHTML=Me,ft=s(),j=i("p"),j.textContent=we,Jt=s(),x=i("p"),x.textContent=ye,It=s(),P=i("p"),P.textContent=he,vt=s(),k=i("p"),k.textContent=ce,bt=s(),B=i("p"),B.textContent=me,jt=s(),g=i("p"),g.textContent=de,xt=s(),_=i("p"),_.innerHTML=Ue,Pt=s(),N=i("p"),N.textContent=ue,kt=s(),V=i("p"),V.textContent=Ce,Bt=s(),G=i("p"),G.textContent=Te,gt=s(),r(A.$$.fragment),_t=s(),S=i("p"),S.textContent=fe,Nt=s(),r($.$$.fragment),Vt=s(),E=i("p"),E.textContent=Je,Gt=s(),r(L.$$.fragment),At=s(),Z=i("p"),Z.textContent=Ie,St=s(),r(X.$$.fragment),$t=s(),R=i("p"),R.textContent=ve,Et=s(),r(W.$$.fragment),Lt=s(),Q=i("p"),Q.innerHTML=be,Zt=s(),H=i("p"),H.textContent=je,Xt=s(),Y=i("p"),Y.textContent=xe,Rt=s(),r(F.$$.fragment),Wt=s(),z=i("p"),z.innerHTML=Pe,Qt=s(),D=i("p"),D.innerHTML=ke,Ht=s(),O=i("blockquote"),O.innerHTML=Be,Yt=s(),q=i("p"),q.innerHTML=ge,Ft=s(),K=i("p"),K.textContent=_e,zt=s(),tt=i("p"),tt.textContent=Ne,Dt=s(),et=i("table"),et.innerHTML=Ve,Ot=s(),lt=i("p"),lt.innerHTML=Ge,qt=s(),nt=i("p"),nt.textContent=Ae,Kt=s(),r(st.$$.fragment),te=s(),at=i("p"),at.innerHTML=Se,ee=s(),ot=i("p"),this.h()},l(t){const e=Qe("svelte-u9bgzb",document.head);m=o(e,"META",{name:!0,content:!0}),e.forEach(l),Mt=a(t),it=o(t,"P",{}),$e(it).forEach(l),wt=a(t),M(d.$$.fragment,t),yt=a(t),U=o(t,"P",{"data-svelte-h":!0}),p(U)!=="svelte-iifysf"&&(U.innerHTML=se),ht=a(t),u=o(t,"P",{"data-svelte-h":!0}),p(u)!=="svelte-1dshoiu"&&(u.textContent=ae),ct=a(t),M(C.$$.fragment,t),mt=a(t),T=o(t,"P",{"data-svelte-h":!0}),p(T)!=="svelte-eleqzp"&&(T.textContent=ie),dt=a(t),f=o(t,"UL",{"data-svelte-h":!0}),p(f)!=="svelte-pz3vuj"&&(f.innerHTML=oe),Ut=a(t),J=o(t,"P",{"data-svelte-h":!0}),p(J)!=="svelte-z1tcso"&&(J.textContent=pe),ut=a(t),M(I.$$.fragment,t),Ct=a(t),v=o(t,"P",{"data-svelte-h":!0}),p(v)!=="svelte-ppjikm"&&(v.textContent=re),Tt=a(t),b=o(t,"P",{"data-svelte-h":!0}),p(b)!=="svelte-11nr6dq"&&(b.innerHTML=Me),ft=a(t),j=o(t,"P",{"data-svelte-h":!0}),p(j)!=="svelte-11nhv2k"&&(j.textContent=we),Jt=a(t),x=o(t,"P",{"data-svelte-h":!0}),p(x)!=="svelte-150xmew"&&(x.textContent=ye),It=a(t),P=o(t,"P",{"data-svelte-h":!0}),p(P)!=="svelte-12gfejp"&&(P.textContent=he),vt=a(t),k=o(t,"P",{"data-svelte-h":!0}),p(k)!=="svelte-uh6lop"&&(k.textContent=ce),bt=a(t),B=o(t,"P",{"data-svelte-h":!0}),p(B)!=="svelte-s9j0io"&&(B.textContent=me),jt=a(t),g=o(t,"P",{"data-svelte-h":!0}),p(g)!=="svelte-8g3p2u"&&(g.textContent=de),xt=a(t),_=o(t,"P",{"data-svelte-h":!0}),p(_)!=="svelte-1gvtzcc"&&(_.innerHTML=Ue),Pt=a(t),N=o(t,"P",{"data-svelte-h":!0}),p(N)!=="svelte-hba1wh"&&(N.textContent=ue),kt=a(t),V=o(t,"P",{"data-svelte-h":!0}),p(V)!=="svelte-1ho6s7e"&&(V.textContent=Ce),Bt=a(t),G=o(t,"P",{"data-svelte-h":!0}),p(G)!=="svelte-1c43r32"&&(G.textContent=Te),gt=a(t),M(A.$$.fragment,t),_t=a(t),S=o(t,"P",{"data-svelte-h":!0}),p(S)!=="svelte-1b7hx6s"&&(S.textContent=fe),Nt=a(t),M($.$$.fragment,t),Vt=a(t),E=o(t,"P",{"data-svelte-h":!0}),p(E)!=="svelte-1m7rqth"&&(E.textContent=Je),Gt=a(t),M(L.$$.fragment,t),At=a(t),Z=o(t,"P",{"data-svelte-h":!0}),p(Z)!=="svelte-hggxkn"&&(Z.textContent=Ie),St=a(t),M(X.$$.fragment,t),$t=a(t),R=o(t,"P",{"data-svelte-h":!0}),p(R)!=="svelte-869n3z"&&(R.textContent=ve),Et=a(t),M(W.$$.fragment,t),Lt=a(t),Q=o(t,"P",{"data-svelte-h":!0}),p(Q)!=="svelte-xobw9p"&&(Q.innerHTML=be),Zt=a(t),H=o(t,"P",{"data-svelte-h":!0}),p(H)!=="svelte-abx4jy"&&(H.textContent=je),Xt=a(t),Y=o(t,"P",{"data-svelte-h":!0}),p(Y)!=="svelte-1hcsw7p"&&(Y.textContent=xe),Rt=a(t),M(F.$$.fragment,t),Wt=a(t),z=o(t,"P",{"data-svelte-h":!0}),p(z)!=="svelte-1gqi5fx"&&(z.innerHTML=Pe),Qt=a(t),D=o(t,"P",{"data-svelte-h":!0}),p(D)!=="svelte-gh3n8e"&&(D.innerHTML=ke),Ht=a(t),O=o(t,"BLOCKQUOTE",{"data-svelte-h":!0}),p(O)!=="svelte-14khom1"&&(O.innerHTML=Be),Yt=a(t),q=o(t,"P",{"data-svelte-h":!0}),p(q)!=="svelte-1naftur"&&(q.innerHTML=ge),Ft=a(t),K=o(t,"P",{"data-svelte-h":!0}),p(K)!=="svelte-smgvyt"&&(K.textContent=_e),zt=a(t),tt=o(t,"P",{"data-svelte-h":!0}),p(tt)!=="svelte-j6eczh"&&(tt.textContent=Ne),Dt=a(t),et=o(t,"TABLE",{"data-svelte-h":!0}),p(et)!=="svelte-1hvmx1i"&&(et.innerHTML=Ve),Ot=a(t),lt=o(t,"P",{"data-svelte-h":!0}),p(lt)!=="svelte-192o6bz"&&(lt.innerHTML=Ge),qt=a(t),nt=o(t,"P",{"data-svelte-h":!0}),p(nt)!=="svelte-16jny0z"&&(nt.textContent=Ae),Kt=a(t),M(st.$$.fragment,t),te=a(t),at=o(t,"P",{"data-svelte-h":!0}),p(at)!=="svelte-pittpg"&&(at.innerHTML=Se),ee=a(t),ot=o(t,"P",{}),$e(ot).forEach(l),this.h()},h(){Ee(m,"name","hf:doc:metadata"),Ee(m,"content",Fe)},m(t,e){He(document.head,m),n(t,Mt,e),n(t,it,e),n(t,wt,e),w(d,t,e),n(t,yt,e),n(t,U,e),n(t,ht,e),n(t,u,e),n(t,ct,e),w(C,t,e),n(t,mt,e),n(t,T,e),n(t,dt,e),n(t,f,e),n(t,Ut,e),n(t,J,e),n(t,ut,e),w(I,t,e),n(t,Ct,e),n(t,v,e),n(t,Tt,e),n(t,b,e),n(t,ft,e),n(t,j,e),n(t,Jt,e),n(t,x,e),n(t,It,e),n(t,P,e),n(t,vt,e),n(t,k,e),n(t,bt,e),n(t,B,e),n(t,jt,e),n(t,g,e),n(t,xt,e),n(t,_,e),n(t,Pt,e),n(t,N,e),n(t,kt,e),n(t,V,e),n(t,Bt,e),n(t,G,e),n(t,gt,e),w(A,t,e),n(t,_t,e),n(t,S,e),n(t,Nt,e),w($,t,e),n(t,Vt,e),n(t,E,e),n(t,Gt,e),w(L,t,e),n(t,At,e),n(t,Z,e),n(t,St,e),w(X,t,e),n(t,$t,e),n(t,R,e),n(t,Et,e),w(W,t,e),n(t,Lt,e),n(t,Q,e),n(t,Zt,e),n(t,H,e),n(t,Xt,e),n(t,Y,e),n(t,Rt,e),w(F,t,e),n(t,Wt,e),n(t,z,e),n(t,Qt,e),n(t,D,e),n(t,Ht,e),n(t,O,e),n(t,Yt,e),n(t,q,e),n(t,Ft,e),n(t,K,e),n(t,zt,e),n(t,tt,e),n(t,Dt,e),n(t,et,e),n(t,Ot,e),n(t,lt,e),n(t,qt,e),n(t,nt,e),n(t,Kt,e),w(st,t,e),n(t,te,e),n(t,at,e),n(t,ee,e),n(t,ot,e),le=!0},p:Ze,i(t){le||(y(d.$$.fragment,t),y(C.$$.fragment,t),y(I.$$.fragment,t),y(A.$$.fragment,t),y($.$$.fragment,t),y(L.$$.fragment,t),y(X.$$.fragment,t),y(W.$$.fragment,t),y(F.$$.fragment,t),y(st.$$.fragment,t),le=!0)},o(t){h(d.$$.fragment,t),h(C.$$.fragment,t),h(I.$$.fragment,t),h(A.$$.fragment,t),h($.$$.fragment,t),h(L.$$.fragment,t),h(X.$$.fragment,t),h(W.$$.fragment,t),h(F.$$.fragment,t),h(st.$$.fragment,t),le=!1},d(t){t&&(l(Mt),l(it),l(wt),l(yt),l(U),l(ht),l(u),l(ct),l(mt),l(T),l(dt),l(f),l(Ut),l(J),l(ut),l(Ct),l(v),l(Tt),l(b),l(ft),l(j),l(Jt),l(x),l(It),l(P),l(vt),l(k),l(bt),l(B),l(jt),l(g),l(xt),l(_),l(Pt),l(N),l(kt),l(V),l(Bt),l(G),l(gt),l(_t),l(S),l(Nt),l(Vt),l(E),l(Gt),l(At),l(Z),l(St),l($t),l(R),l(Et),l(Lt),l(Q),l(Zt),l(H),l(Xt),l(Y),l(Rt),l(Wt),l(z),l(Qt),l(D),l(Ht),l(O),l(Yt),l(q),l(Ft),l(K),l(zt),l(tt),l(Dt),l(et),l(Ot),l(lt),l(qt),l(nt),l(Kt),l(te),l(at),l(ee),l(ot)),l(m),c(d,t),c(C,t),c(I,t),c(A,t),c($,t),c(L,t),c(X,t),c(W,t),c(F,t),c(st,t)}}}const Fe='{"title":"Custom hardware for training","local":"custom-hardware-for-training","sections":[{"title":"GPU","local":"gpu","sections":[{"title":"Power and Cooling","local":"power-and-cooling","sections":[],"depth":3},{"title":"Multi-GPU Connectivity","local":"multi-gpu-connectivity","sections":[{"title":"NVlink","local":"nvlink","sections":[],"depth":4}],"depth":3}],"depth":2}],"depth":1}';function ze(ne){return Xe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class tl extends Re{constructor(m){super(),We(this,m,ze,Ye,Le,{})}}export{tl as component};
