import{s as Vt,o as Qt,n as Ot}from"../chunks/scheduler.b9285784.js";import{S as qt,i as Kt,e as n,s,c as d,h as el,a as o,d as l,b as i,f as It,g as p,j as r,k as Yt,l as tl,m as a,n as c,t as f,o as m,p as h}from"../chunks/index.26bc89a1.js";import{T as ll}from"../chunks/Tip.e4eba3d6.js";import{C as al,H as _e,E as sl}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.7242ff19.js";import{C as w}from"../chunks/CodeBlock.c8ee1711.js";function il(Me){let u,T='For those interested in the similarities and differences between FSDP and DeepSpeed, please check out the <a href="../concept_guides/fsdp_and_deepspeed">concept guide here</a>!';return{c(){u=n("p"),u.innerHTML=T},l(_){u=o(_,"P",{"data-svelte-h":!0}),r(u)!=="svelte-we3qam"&&(u.innerHTML=T)},m(_,ue){a(_,u,ue)},p:Ot,d(_){_&&l(u)}}}function nl(Me){let u,T,_,ue,y,Te,g,ye,b,_t=`To accelerate training huge models on larger batch sizes, we can use a fully sharded data parallel model.
This type of data parallel paradigm enables fitting more data and larger models by sharding the optimizer states, gradients and parameters.
To read more about it and the benefits, check out the <a href="https://pytorch.org/blog/introducing-pytorch-fully-sharded-data-parallel-api/" rel="nofollow">Fully Sharded Data Parallel blog</a>.
We have integrated the latest PyTorch’s Fully Sharded Data Parallel (FSDP) training feature.
All you need to do is enable it through the config.`,ge,$,be,v,wt="On your machine(s) just run:",$e,Z,ve,B,Mt=`and answer the questions asked. This will generate a config file that will be used automatically to properly set the
default options when doing`,Ze,j,Be,J,Tt="For instance, here is how you would run <code>examples/nlp_example.py</code> (from the root of the repo) with FSDP enabled:",je,F,Je,S,Fe,H,yt="Currently, <code>Accelerate</code> supports the following config through the CLI:",Se,R,gt='<code>fsdp_sharding_strategy</code>: [1] FULL_SHARD (shards optimizer states, gradients and parameters), [2] SHARD_GRAD_OP (shards optimizer states and gradients), [3] NO_SHARD (DDP), [4] HYBRID_SHARD (shards optimizer states, gradients and parameters within each node while each node has full copy), [5] HYBRID_SHARD_ZERO2 (shards optimizer states and gradients within each node while each node has full copy). For more information, please refer the official <a href="https://pytorch.org/docs/stable/fsdp.html#torch.distributed.fsdp.ShardingStrategy" rel="nofollow">PyTorch docs</a>.',He,x,bt="<code>fsdp_offload_params</code> : Decides Whether to offload parameters and gradients to CPU",Re,W,$t="<code>fsdp_auto_wrap_policy</code>: [1] TRANSFORMER_BASED_WRAP, [2] SIZE_BASED_WRAP, [3] NO_WRAP",xe,U,vt="<code>fsdp_transformer_layer_cls_to_wrap</code>: Only applicable for Transformers. When using <code>fsdp_auto_wrap_policy=TRANSFORMER_BASED_WRAP</code>, a user may provide a comma-separated string of transformer layer class names (case-sensitive) to wrap, e.g., <code>BertLayer</code>, <code>GPTJBlock</code>, <code>T5Block</code>, <code>BertLayer,BertEmbeddings,BertSelfOutput</code>. This is important because submodules that share weights (e.g., embedding layers) should not end up in different FSDP wrapped units. Using this policy, wrapping happens for each block containing Multi-Head Attention followed by a couple of MLP layers. Remaining layers including the shared embeddings are conveniently wrapped in same outermost FSDP unit. Therefore, use this for transformer-based models. You can use the <code>model._no_split_modules</code> for Transformer models by answering <code>yes</code> to <code>Do you want to use the model&#39;s </code>_no_split_modules<code>to wrap. It will try to use</code>model._no_split_modules` when possible.",We,X,Zt="<code>fsdp_min_num_params</code>: minimum number of parameters when using <code>fsdp_auto_wrap_policy=SIZE_BASED_WRAP</code>.",Ue,C,Bt="<code>fsdp_backward_prefetch_policy</code>: [1] BACKWARD_PRE, [2] BACKWARD_POST, [3] NO_PREFETCH",Xe,G,jt="<code>fsdp_forward_prefetch</code>: if True, then FSDP explicitly prefetches the next upcoming all-gather while executing in the forward pass. Should only be used for static-graph models since the prefetching follows the first iteration’s execution order. i.e., if the sub-modules’ order changes dynamically during the model’s execution do not enable this feature.",Ce,k,Jt="<code>fsdp_state_dict_type</code>: [1] FULL_STATE_DICT, [2] LOCAL_STATE_DICT, [3] SHARDED_STATE_DICT",Ge,A,Ft='<code>fsdp_use_orig_params</code>: If True, allows non-uniform <code>requires_grad</code> during init, which means support for interspersed frozen and trainable parameters. This setting is useful in cases such as parameter-efficient fine-tuning as discussed in <a href="https://dev-discuss.pytorch.org/t/rethinking-pytorch-fully-sharded-data-parallel-fsdp-from-first-principles/1019" rel="nofollow">this post</a>. This option also allows one to have multiple optimizer param groups. This should be <code>True</code> when creating an optimizer before preparing/wrapping the model with FSDP.',ke,E,St="<code>fsdp_cpu_ram_efficient_loading</code>: Only applicable for Transformers models. If True, only the first process loads the pretrained model checkpoint while all other processes have empty weights. This should be set to False if you experience errors when loading the pretrained Transformers model via <code>from_pretrained</code> method. When this setting is True <code>fsdp_sync_module_states</code> also must to be True, otherwise all the processes except the main process would have random weights leading to unexpected behaviour during training. For this to work, make sure the distributed process group is initialized before calling Transformers <code>from_pretrained</code> method. When using Trainer API, the distributed process group is initialized when you create an instance of <code>TrainingArguments</code> class.",Ae,P,Ht="<code>fsdp_sync_module_states</code>: If True, each individually wrapped FSDP unit will broadcast module parameters from rank 0.",Ee,z,Rt=`For additional and more nuanced control, you can specify other FSDP parameters via <code>FullyShardedDataParallelPlugin</code>.
When creating <code>FullyShardedDataParallelPlugin</code> object, pass it the parameters that weren’t part of the accelerate config or if you want to override them.
The FSDP parameters will be picked based on the accelerate config file or launch command arguments and other parameters that you will pass directly through the <code>FullyShardedDataParallelPlugin</code> object will set/override that.`,Pe,D,xt="Below is an example:",ze,N,De,L,Ne,I,Wt=`The new recommended way of checkpointing when using FSDP models is to use <code>SHARDED_STATE_DICT</code> as <code>StateDictType</code> when setting up the accelerate config.
Below is the code snippet to save using <code>save_state</code> utility of accelerate.`,Le,Y,Ie,V,Ut="Inspect the checkpoint folder to see model and optimizer as shards per process:",Ye,Q,Ve,O,Xt="To load them back for resuming the training, use the <code>load_state</code> utility of accelerate",Qe,q,Oe,K,Ct=`When using transformers <code>save_pretrained</code>, pass <code>state_dict=accelerator.get_state_dict(model)</code> to save the model state dict.
Below is an example:`,qe,ee,Ke,te,et,le,Gt="<code>accelerator.get_state_dict</code> will call the underlying <code>model.state_dict</code> implementation using <code>FullStateDictConfig(offload_to_cpu=True, rank0_only=True)</code> context manager to get the state dict only for rank 0 and it will be offloaded to CPU.",tt,ae,kt='You can then pass <code>state</code> into the <code>save_pretrained</code> method.  There are several modes for <code>StateDictType</code> and <code>FullStateDictConfig</code> that you can use to control the behavior of <code>state_dict</code>.  For more information, see the <a href="https://pytorch.org/docs/stable/fsdp.html" rel="nofollow">PyTorch documentation</a>.',lt,se,At=`If you choose to use <code>StateDictType.SHARDED_STATE_DICT</code>, the weights of the model during <code>Accelerator.save_state</code> will be split into <code>n</code> files for each sub-split on the model. To merge them back into
a single dictionary to load back into the model later after training you can use the <code>merge_weights</code> utility:`,at,ie,st,ne,Et="The final output will then either be saved to <code>model.safetensors</code> or <code>pytorch_model.bin</code> (if <code>safe_serialization=False</code> is passed).",it,oe,Pt="This can also be called using the CLI:",nt,re,ot,de,rt,pe,zt="<li><code>FULL_SHARD</code> maps to the DeepSpeed <code>ZeRO Stage-3</code>. Shards optimizer states, gradients and parameters.</li> <li><code>SHARD_GRAD_OP</code> maps to the DeepSpeed <code>ZeRO Stage-2</code>. Shards optimizer states and gradients.</li> <li><code>NO_SHARD</code> maps to <code>ZeRO Stage-0</code>. No sharding wherein each GPU has full copy of model, optimizer states and gradients.</li> <li><code>HYBRID_SHARD</code> maps to <code>ZeRO++ Stage-3</code> wherein <code>zero_hpz_partition_size=&lt;num_gpus_per_node&gt;</code>. Here, this will shard optimizer states, gradients and parameters within each node while each node has full copy.</li>",dt,ce,pt,fe,Dt="<li>In case of multiple models, pass the optimizers to the prepare call in the same order as corresponding models else <code>accelerator.save_state()</code> and <code>accelerator.load_state()</code> will result in wrong/unexpected behaviour.</li> <li>This feature is incompatible with <code>--predict_with_generate</code> in the <code>run_translation.py</code> script of <code>Transformers</code> library.</li>",ct,me,Nt=`For more control, users can leverage the <code>FullyShardedDataParallelPlugin</code>. After creating an instance of this class, users can pass it to the Accelerator class instantiation.
For more information on these options, please refer to the PyTorch <a href="https://github.com/pytorch/pytorch/blob/0df2e863fbd5993a7b9e652910792bd21a516ff3/torch/distributed/fsdp/fully_sharded_data_parallel.py#L236" rel="nofollow">FullyShardedDataParallel</a> code.`,ft,M,mt,he,ht,we,ut;return y=new al({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),g=new _e({props:{title:"Fully Sharded Data Parallel",local:"fully-sharded-data-parallel",headingTag:"h1"}}),$=new _e({props:{title:"How it works out of the box",local:"how-it-works-out-of-the-box",headingTag:"h2"}}),Z=new w({props:{code:"YWNjZWxlcmF0ZSUyMGNvbmZpZw==",highlighted:"accelerate config",wrap:!1}}),j=new w({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMG15X3NjcmlwdC5weSUyMC0tYXJnc190b19teV9zY3JpcHQ=",highlighted:"accelerate launch my_script.py --args_to_my_script",wrap:!1}}),F=new w({props:{code:"Y29tcHV0ZV9lbnZpcm9ubWVudCUzQSUyMExPQ0FMX01BQ0hJTkUlMEFkZWJ1ZyUzQSUyMGZhbHNlJTBBZGlzdHJpYnV0ZWRfdHlwZSUzQSUyMEZTRFAlMEFkb3duY2FzdF9iZjE2JTNBJTIwJ25vJyUwQWZzZHBfY29uZmlnJTNBJTBBJTIwJTIwZnNkcF9hdXRvX3dyYXBfcG9saWN5JTNBJTIwVFJBTlNGT1JNRVJfQkFTRURfV1JBUCUwQSUyMCUyMGZzZHBfYmFja3dhcmRfcHJlZmV0Y2hfcG9saWN5JTNBJTIwQkFDS1dBUkRfUFJFJTBBJTIwJTIwZnNkcF9mb3J3YXJkX3ByZWZldGNoJTNBJTIwZmFsc2UlMEElMjAlMjBmc2RwX2NwdV9yYW1fZWZmaWNpZW50X2xvYWRpbmclM0ElMjB0cnVlJTBBJTIwJTIwZnNkcF9vZmZsb2FkX3BhcmFtcyUzQSUyMGZhbHNlJTBBJTIwJTIwZnNkcF9zaGFyZGluZ19zdHJhdGVneSUzQSUyMEZVTExfU0hBUkQlMEElMjAlMjBmc2RwX3N0YXRlX2RpY3RfdHlwZSUzQSUyMFNIQVJERURfU1RBVEVfRElDVCUwQSUyMCUyMGZzZHBfc3luY19tb2R1bGVfc3RhdGVzJTNBJTIwdHJ1ZSUwQSUyMCUyMGZzZHBfdHJhbnNmb3JtZXJfbGF5ZXJfY2xzX3RvX3dyYXAlM0ElMjBCZXJ0TGF5ZXIlMEElMjAlMjBmc2RwX3VzZV9vcmlnX3BhcmFtcyUzQSUyMHRydWUlMEFtYWNoaW5lX3JhbmslM0ElMjAwJTBBbWFpbl90cmFpbmluZ19mdW5jdGlvbiUzQSUyMG1haW4lMEFtaXhlZF9wcmVjaXNpb24lM0ElMjBiZjE2JTBBbnVtX21hY2hpbmVzJTNBJTIwMSUwQW51bV9wcm9jZXNzZXMlM0ElMjAyJTBBcmR6dl9iYWNrZW5kJTNBJTIwc3RhdGljJTBBc2FtZV9uZXR3b3JrJTNBJTIwdHJ1ZSUwQXRwdV9lbnYlM0ElMjAlNUIlNUQlMEF0cHVfdXNlX2NsdXN0ZXIlM0ElMjBmYWxzZSUwQXRwdV91c2Vfc3VkbyUzQSUyMGZhbHNlJTBBdXNlX2NwdSUzQSUyMGZhbHNl",highlighted:`compute_environment: LOCAL_MACHINE
debug: <span class="hljs-literal">false</span>
distributed_type: FSDP
downcast_bf16: <span class="hljs-string">&#x27;no&#x27;</span>
fsdp_config:
  fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
  fsdp_backward_prefetch_policy: BACKWARD_PRE
  fsdp_forward_prefetch: <span class="hljs-literal">false</span>
  fsdp_cpu_ram_efficient_loading: <span class="hljs-literal">true</span>
  fsdp_offload_params: <span class="hljs-literal">false</span>
  fsdp_sharding_strategy: FULL_SHARD
  fsdp_state_dict_type: SHARDED_STATE_DICT
  fsdp_sync_module_states: <span class="hljs-literal">true</span>
  fsdp_transformer_layer_cls_to_wrap: BertLayer
  fsdp_use_orig_params: <span class="hljs-literal">true</span>
machine_rank: 0
main_training_function: main
mixed_precision: bf16
num_machines: 1
num_processes: 2
rdzv_backend: static
same_network: <span class="hljs-literal">true</span>
tpu_env: []
tpu_use_cluster: <span class="hljs-literal">false</span>
tpu_use_sudo: <span class="hljs-literal">false</span>
use_cpu: <span class="hljs-literal">false</span>`,wrap:!1}}),S=new w({props:{code:"YWNjZWxlcmF0ZSUyMGxhdW5jaCUyMGV4YW1wbGVzJTJGbmxwX2V4YW1wbGUucHk=",highlighted:"accelerate launch examples/nlp_example.py",wrap:!1}}),N=new w({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBGdWxseVNoYXJkZWREYXRhUGFyYWxsZWxQbHVnaW4lMEFmcm9tJTIwdG9yY2guZGlzdHJpYnV0ZWQuZnNkcC5mdWxseV9zaGFyZGVkX2RhdGFfcGFyYWxsZWwlMjBpbXBvcnQlMjBGdWxsT3B0aW1TdGF0ZURpY3RDb25maWclMkMlMjBGdWxsU3RhdGVEaWN0Q29uZmlnJTBBJTBBZnNkcF9wbHVnaW4lMjAlM0QlMjBGdWxseVNoYXJkZWREYXRhUGFyYWxsZWxQbHVnaW4oJTBBJTIwJTIwJTIwJTIwc3RhdGVfZGljdF9jb25maWclM0RGdWxsU3RhdGVEaWN0Q29uZmlnKG9mZmxvYWRfdG9fY3B1JTNERmFsc2UlMkMlMjByYW5rMF9vbmx5JTNERmFsc2UpJTJDJTBBJTIwJTIwJTIwJTIwb3B0aW1fc3RhdGVfZGljdF9jb25maWclM0RGdWxsT3B0aW1TdGF0ZURpY3RDb25maWcob2ZmbG9hZF90b19jcHUlM0RGYWxzZSUyQyUyMHJhbmswX29ubHklM0RGYWxzZSklMkMlMEEpJTBBJTBBYWNjZWxlcmF0b3IlMjAlM0QlMjBBY2NlbGVyYXRvcihmc2RwX3BsdWdpbiUzRGZzZHBfcGx1Z2luKQ==",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> FullyShardedDataParallelPlugin
<span class="hljs-keyword">from</span> torch.distributed.fsdp.fully_sharded_data_parallel <span class="hljs-keyword">import</span> FullOptimStateDictConfig, FullStateDictConfig

fsdp_plugin = FullyShardedDataParallelPlugin(
    state_dict_config=FullStateDictConfig(offload_to_cpu=<span class="hljs-literal">False</span>, rank0_only=<span class="hljs-literal">False</span>),
    optim_state_dict_config=FullOptimStateDictConfig(offload_to_cpu=<span class="hljs-literal">False</span>, rank0_only=<span class="hljs-literal">False</span>),
)

accelerator = Accelerator(fsdp_plugin=fsdp_plugin)`,wrap:!1}}),L=new _e({props:{title:"Saving and loading",local:"saving-and-loading",headingTag:"h2"}}),Y=new w({props:{code:"YWNjZWxlcmF0b3Iuc2F2ZV9zdGF0ZSglMjJja3B0JTIyKQ==",highlighted:'accelerator.save_state(<span class="hljs-string">&quot;ckpt&quot;</span>)',wrap:!1}}),Q=new w({props:{code:"bHMlMjBja3B0JTBBJTIzJTIwb3B0aW1pemVyXzAlMjAlMjBweXRvcmNoX21vZGVsXzAlMjAlMjByYW5kb21fc3RhdGVzXzAucGtsJTIwJTIwcmFuZG9tX3N0YXRlc18xLnBrbCUyMCUyMHNjaGVkdWxlci5iaW4lMEElMEFjZCUyMGNrcHQlMEElMEFscyUyMG9wdGltaXplcl8wJTBBJTIzJTIwX18wXzAuZGlzdGNwJTIwJTIwX18xXzAuZGlzdGNwJTBBJTBBbHMlMjBweXRvcmNoX21vZGVsXzAlMEElMjMlMjBfXzBfMC5kaXN0Y3AlMjAlMjBfXzFfMC5kaXN0Y3A=",highlighted:`<span class="hljs-built_in">ls</span> ckpt
<span class="hljs-comment"># optimizer_0  pytorch_model_0  random_states_0.pkl  random_states_1.pkl  scheduler.bin</span>

<span class="hljs-built_in">cd</span> ckpt

<span class="hljs-built_in">ls</span> optimizer_0
<span class="hljs-comment"># __0_0.distcp  __1_0.distcp</span>

<span class="hljs-built_in">ls</span> pytorch_model_0
<span class="hljs-comment"># __0_0.distcp  __1_0.distcp</span>`,wrap:!1}}),q=new w({props:{code:"YWNjZWxlcmF0b3IubG9hZF9zdGF0ZSglMjJja3B0JTIyKQ==",highlighted:'accelerator.load_state(<span class="hljs-string">&quot;ckpt&quot;</span>)',wrap:!1}}),ee=new w({props:{code:"JTIwJTIwdW53cmFwcGVkX21vZGVsLnNhdmVfcHJldHJhaW5lZCglMEElMjAlMjAlMjAlMjAlMjAlMjBhcmdzLm91dHB1dF9kaXIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjBpc19tYWluX3Byb2Nlc3MlM0RhY2NlbGVyYXRvci5pc19tYWluX3Byb2Nlc3MlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjBzYXZlX2Z1bmN0aW9uJTNEYWNjZWxlcmF0b3Iuc2F2ZSUyQyUwQSUyQiUyMCUyMCUyMCUyMCUyMHN0YXRlX2RpY3QlM0RhY2NlbGVyYXRvci5nZXRfc3RhdGVfZGljdChtb2RlbCklMkMlMEEp",highlighted:`  unwrapped_model.save_pretrained(
      args.output_dir,
      is_main_process=accelerator.is_main_process,
      save_function=accelerator.save,
<span class="hljs-addition">+     state_dict=accelerator.get_state_dict(model),</span>
)`,wrap:!1}}),te=new _e({props:{title:"State Dict",local:"state-dict",headingTag:"h3"}}),ie=new w({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUudXRpbHMlMjBpbXBvcnQlMjBtZXJnZV9mc2RwX3dlaWdodHMlMEElMEElMjMlMjBPdXIlMjB3ZWlnaHRzJTIwYXJlJTIwc2F2ZWQlMjB1c3VhbGx5JTIwaW4lMjBhJTIwJTYwcHl0b3JjaF9tb2RlbF9mc2RwXyU3Qm1vZGVsX251bWJlciU3RCU2MCUyMGZvbGRlciUwQW1lcmdlX2ZzZHBfd2VpZ2h0cyglMjJweXRvcmNoX21vZGVsX2ZzZHBfMCUyMiUyQyUyMCUyMm91dHB1dF9wYXRoJTIyJTJDJTIwc2FmZV9zZXJpYWxpemF0aW9uJTNEVHJ1ZSk=",highlighted:`<span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> merge_fsdp_weights

<span class="hljs-comment"># Our weights are saved usually in a \`pytorch_model_fsdp_{model_number}\` folder</span>
merge_fsdp_weights(<span class="hljs-string">&quot;pytorch_model_fsdp_0&quot;</span>, <span class="hljs-string">&quot;output_path&quot;</span>, safe_serialization=<span class="hljs-literal">True</span>)`,wrap:!1}}),re=new w({props:{code:"YWNjZWxlcmF0ZSUyMG1lcmdlLXdlaWdodHMlMjBweXRvcmNoX21vZGVsX2ZzZHBfMCUyRiUyMG91dHB1dF9wYXRo",highlighted:"accelerate merge-weights pytorch_model_fsdp_0/ output_path",wrap:!1}}),de=new _e({props:{title:"Mapping between FSDP sharding strategies and DeepSpeed ZeRO Stages",local:"mapping-between-fsdp-sharding-strategies-and-deepspeed-zero-stages",headingTag:"h2"}}),ce=new _e({props:{title:"A few caveats to be aware of",local:"a-few-caveats-to-be-aware-of",headingTag:"h2"}}),M=new ll({props:{$$slots:{default:[il]},$$scope:{ctx:Me}}}),he=new sl({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/usage_guides/fsdp.md"}}),{c(){u=n("meta"),T=s(),_=n("p"),ue=s(),d(y.$$.fragment),Te=s(),d(g.$$.fragment),ye=s(),b=n("p"),b.innerHTML=_t,ge=s(),d($.$$.fragment),be=s(),v=n("p"),v.textContent=wt,$e=s(),d(Z.$$.fragment),ve=s(),B=n("p"),B.textContent=Mt,Ze=s(),d(j.$$.fragment),Be=s(),J=n("p"),J.innerHTML=Tt,je=s(),d(F.$$.fragment),Je=s(),d(S.$$.fragment),Fe=s(),H=n("p"),H.innerHTML=yt,Se=s(),R=n("p"),R.innerHTML=gt,He=s(),x=n("p"),x.innerHTML=bt,Re=s(),W=n("p"),W.innerHTML=$t,xe=s(),U=n("p"),U.innerHTML=vt,We=s(),X=n("p"),X.innerHTML=Zt,Ue=s(),C=n("p"),C.innerHTML=Bt,Xe=s(),G=n("p"),G.innerHTML=jt,Ce=s(),k=n("p"),k.innerHTML=Jt,Ge=s(),A=n("p"),A.innerHTML=Ft,ke=s(),E=n("p"),E.innerHTML=St,Ae=s(),P=n("p"),P.innerHTML=Ht,Ee=s(),z=n("p"),z.innerHTML=Rt,Pe=s(),D=n("p"),D.textContent=xt,ze=s(),d(N.$$.fragment),De=s(),d(L.$$.fragment),Ne=s(),I=n("p"),I.innerHTML=Wt,Le=s(),d(Y.$$.fragment),Ie=s(),V=n("p"),V.textContent=Ut,Ye=s(),d(Q.$$.fragment),Ve=s(),O=n("p"),O.innerHTML=Xt,Qe=s(),d(q.$$.fragment),Oe=s(),K=n("p"),K.innerHTML=Ct,qe=s(),d(ee.$$.fragment),Ke=s(),d(te.$$.fragment),et=s(),le=n("p"),le.innerHTML=Gt,tt=s(),ae=n("p"),ae.innerHTML=kt,lt=s(),se=n("p"),se.innerHTML=At,at=s(),d(ie.$$.fragment),st=s(),ne=n("p"),ne.innerHTML=Et,it=s(),oe=n("p"),oe.textContent=Pt,nt=s(),d(re.$$.fragment),ot=s(),d(de.$$.fragment),rt=s(),pe=n("ul"),pe.innerHTML=zt,dt=s(),d(ce.$$.fragment),pt=s(),fe=n("ul"),fe.innerHTML=Dt,ct=s(),me=n("p"),me.innerHTML=Nt,ft=s(),d(M.$$.fragment),mt=s(),d(he.$$.fragment),ht=s(),we=n("p"),this.h()},l(e){const t=el("svelte-u9bgzb",document.head);u=o(t,"META",{name:!0,content:!0}),t.forEach(l),T=i(e),_=o(e,"P",{}),It(_).forEach(l),ue=i(e),p(y.$$.fragment,e),Te=i(e),p(g.$$.fragment,e),ye=i(e),b=o(e,"P",{"data-svelte-h":!0}),r(b)!=="svelte-qnvjvw"&&(b.innerHTML=_t),ge=i(e),p($.$$.fragment,e),be=i(e),v=o(e,"P",{"data-svelte-h":!0}),r(v)!=="svelte-3pof6s"&&(v.textContent=wt),$e=i(e),p(Z.$$.fragment,e),ve=i(e),B=o(e,"P",{"data-svelte-h":!0}),r(B)!=="svelte-ovz84c"&&(B.textContent=Mt),Ze=i(e),p(j.$$.fragment,e),Be=i(e),J=o(e,"P",{"data-svelte-h":!0}),r(J)!=="svelte-109sebi"&&(J.innerHTML=Tt),je=i(e),p(F.$$.fragment,e),Je=i(e),p(S.$$.fragment,e),Fe=i(e),H=o(e,"P",{"data-svelte-h":!0}),r(H)!=="svelte-aporbc"&&(H.innerHTML=yt),Se=i(e),R=o(e,"P",{"data-svelte-h":!0}),r(R)!=="svelte-mtvmo1"&&(R.innerHTML=gt),He=i(e),x=o(e,"P",{"data-svelte-h":!0}),r(x)!=="svelte-173mpds"&&(x.innerHTML=bt),Re=i(e),W=o(e,"P",{"data-svelte-h":!0}),r(W)!=="svelte-56dvve"&&(W.innerHTML=$t),xe=i(e),U=o(e,"P",{"data-svelte-h":!0}),r(U)!=="svelte-1tuw09a"&&(U.innerHTML=vt),We=i(e),X=o(e,"P",{"data-svelte-h":!0}),r(X)!=="svelte-fbrtfo"&&(X.innerHTML=Zt),Ue=i(e),C=o(e,"P",{"data-svelte-h":!0}),r(C)!=="svelte-gnni23"&&(C.innerHTML=Bt),Xe=i(e),G=o(e,"P",{"data-svelte-h":!0}),r(G)!=="svelte-1b8s7mr"&&(G.innerHTML=jt),Ce=i(e),k=o(e,"P",{"data-svelte-h":!0}),r(k)!=="svelte-1bw4fgg"&&(k.innerHTML=Jt),Ge=i(e),A=o(e,"P",{"data-svelte-h":!0}),r(A)!=="svelte-1hsz8jb"&&(A.innerHTML=Ft),ke=i(e),E=o(e,"P",{"data-svelte-h":!0}),r(E)!=="svelte-w8koju"&&(E.innerHTML=St),Ae=i(e),P=o(e,"P",{"data-svelte-h":!0}),r(P)!=="svelte-9z96c5"&&(P.innerHTML=Ht),Ee=i(e),z=o(e,"P",{"data-svelte-h":!0}),r(z)!=="svelte-1mlu7qv"&&(z.innerHTML=Rt),Pe=i(e),D=o(e,"P",{"data-svelte-h":!0}),r(D)!=="svelte-ks9e2y"&&(D.textContent=xt),ze=i(e),p(N.$$.fragment,e),De=i(e),p(L.$$.fragment,e),Ne=i(e),I=o(e,"P",{"data-svelte-h":!0}),r(I)!=="svelte-105rpu1"&&(I.innerHTML=Wt),Le=i(e),p(Y.$$.fragment,e),Ie=i(e),V=o(e,"P",{"data-svelte-h":!0}),r(V)!=="svelte-zz881x"&&(V.textContent=Ut),Ye=i(e),p(Q.$$.fragment,e),Ve=i(e),O=o(e,"P",{"data-svelte-h":!0}),r(O)!=="svelte-1v7i9ht"&&(O.innerHTML=Xt),Qe=i(e),p(q.$$.fragment,e),Oe=i(e),K=o(e,"P",{"data-svelte-h":!0}),r(K)!=="svelte-1y7f54p"&&(K.innerHTML=Ct),qe=i(e),p(ee.$$.fragment,e),Ke=i(e),p(te.$$.fragment,e),et=i(e),le=o(e,"P",{"data-svelte-h":!0}),r(le)!=="svelte-y5myy"&&(le.innerHTML=Gt),tt=i(e),ae=o(e,"P",{"data-svelte-h":!0}),r(ae)!=="svelte-57zs50"&&(ae.innerHTML=kt),lt=i(e),se=o(e,"P",{"data-svelte-h":!0}),r(se)!=="svelte-ik1th2"&&(se.innerHTML=At),at=i(e),p(ie.$$.fragment,e),st=i(e),ne=o(e,"P",{"data-svelte-h":!0}),r(ne)!=="svelte-1qedos9"&&(ne.innerHTML=Et),it=i(e),oe=o(e,"P",{"data-svelte-h":!0}),r(oe)!=="svelte-1duybas"&&(oe.textContent=Pt),nt=i(e),p(re.$$.fragment,e),ot=i(e),p(de.$$.fragment,e),rt=i(e),pe=o(e,"UL",{"data-svelte-h":!0}),r(pe)!=="svelte-1mojn5u"&&(pe.innerHTML=zt),dt=i(e),p(ce.$$.fragment,e),pt=i(e),fe=o(e,"UL",{"data-svelte-h":!0}),r(fe)!=="svelte-1cyfpa6"&&(fe.innerHTML=Dt),ct=i(e),me=o(e,"P",{"data-svelte-h":!0}),r(me)!=="svelte-1dlu8nc"&&(me.innerHTML=Nt),ft=i(e),p(M.$$.fragment,e),mt=i(e),p(he.$$.fragment,e),ht=i(e),we=o(e,"P",{}),It(we).forEach(l),this.h()},h(){Yt(u,"name","hf:doc:metadata"),Yt(u,"content",ol)},m(e,t){tl(document.head,u),a(e,T,t),a(e,_,t),a(e,ue,t),c(y,e,t),a(e,Te,t),c(g,e,t),a(e,ye,t),a(e,b,t),a(e,ge,t),c($,e,t),a(e,be,t),a(e,v,t),a(e,$e,t),c(Z,e,t),a(e,ve,t),a(e,B,t),a(e,Ze,t),c(j,e,t),a(e,Be,t),a(e,J,t),a(e,je,t),c(F,e,t),a(e,Je,t),c(S,e,t),a(e,Fe,t),a(e,H,t),a(e,Se,t),a(e,R,t),a(e,He,t),a(e,x,t),a(e,Re,t),a(e,W,t),a(e,xe,t),a(e,U,t),a(e,We,t),a(e,X,t),a(e,Ue,t),a(e,C,t),a(e,Xe,t),a(e,G,t),a(e,Ce,t),a(e,k,t),a(e,Ge,t),a(e,A,t),a(e,ke,t),a(e,E,t),a(e,Ae,t),a(e,P,t),a(e,Ee,t),a(e,z,t),a(e,Pe,t),a(e,D,t),a(e,ze,t),c(N,e,t),a(e,De,t),c(L,e,t),a(e,Ne,t),a(e,I,t),a(e,Le,t),c(Y,e,t),a(e,Ie,t),a(e,V,t),a(e,Ye,t),c(Q,e,t),a(e,Ve,t),a(e,O,t),a(e,Qe,t),c(q,e,t),a(e,Oe,t),a(e,K,t),a(e,qe,t),c(ee,e,t),a(e,Ke,t),c(te,e,t),a(e,et,t),a(e,le,t),a(e,tt,t),a(e,ae,t),a(e,lt,t),a(e,se,t),a(e,at,t),c(ie,e,t),a(e,st,t),a(e,ne,t),a(e,it,t),a(e,oe,t),a(e,nt,t),c(re,e,t),a(e,ot,t),c(de,e,t),a(e,rt,t),a(e,pe,t),a(e,dt,t),c(ce,e,t),a(e,pt,t),a(e,fe,t),a(e,ct,t),a(e,me,t),a(e,ft,t),c(M,e,t),a(e,mt,t),c(he,e,t),a(e,ht,t),a(e,we,t),ut=!0},p(e,[t]){const Lt={};t&2&&(Lt.$$scope={dirty:t,ctx:e}),M.$set(Lt)},i(e){ut||(f(y.$$.fragment,e),f(g.$$.fragment,e),f($.$$.fragment,e),f(Z.$$.fragment,e),f(j.$$.fragment,e),f(F.$$.fragment,e),f(S.$$.fragment,e),f(N.$$.fragment,e),f(L.$$.fragment,e),f(Y.$$.fragment,e),f(Q.$$.fragment,e),f(q.$$.fragment,e),f(ee.$$.fragment,e),f(te.$$.fragment,e),f(ie.$$.fragment,e),f(re.$$.fragment,e),f(de.$$.fragment,e),f(ce.$$.fragment,e),f(M.$$.fragment,e),f(he.$$.fragment,e),ut=!0)},o(e){m(y.$$.fragment,e),m(g.$$.fragment,e),m($.$$.fragment,e),m(Z.$$.fragment,e),m(j.$$.fragment,e),m(F.$$.fragment,e),m(S.$$.fragment,e),m(N.$$.fragment,e),m(L.$$.fragment,e),m(Y.$$.fragment,e),m(Q.$$.fragment,e),m(q.$$.fragment,e),m(ee.$$.fragment,e),m(te.$$.fragment,e),m(ie.$$.fragment,e),m(re.$$.fragment,e),m(de.$$.fragment,e),m(ce.$$.fragment,e),m(M.$$.fragment,e),m(he.$$.fragment,e),ut=!1},d(e){e&&(l(T),l(_),l(ue),l(Te),l(ye),l(b),l(ge),l(be),l(v),l($e),l(ve),l(B),l(Ze),l(Be),l(J),l(je),l(Je),l(Fe),l(H),l(Se),l(R),l(He),l(x),l(Re),l(W),l(xe),l(U),l(We),l(X),l(Ue),l(C),l(Xe),l(G),l(Ce),l(k),l(Ge),l(A),l(ke),l(E),l(Ae),l(P),l(Ee),l(z),l(Pe),l(D),l(ze),l(De),l(Ne),l(I),l(Le),l(Ie),l(V),l(Ye),l(Ve),l(O),l(Qe),l(Oe),l(K),l(qe),l(Ke),l(et),l(le),l(tt),l(ae),l(lt),l(se),l(at),l(st),l(ne),l(it),l(oe),l(nt),l(ot),l(rt),l(pe),l(dt),l(pt),l(fe),l(ct),l(me),l(ft),l(mt),l(ht),l(we)),l(u),h(y,e),h(g,e),h($,e),h(Z,e),h(j,e),h(F,e),h(S,e),h(N,e),h(L,e),h(Y,e),h(Q,e),h(q,e),h(ee,e),h(te,e),h(ie,e),h(re,e),h(de,e),h(ce,e),h(M,e),h(he,e)}}}const ol='{"title":"Fully Sharded Data Parallel","local":"fully-sharded-data-parallel","sections":[{"title":"How it works out of the box","local":"how-it-works-out-of-the-box","sections":[],"depth":2},{"title":"Saving and loading","local":"saving-and-loading","sections":[{"title":"State Dict","local":"state-dict","sections":[],"depth":3}],"depth":2},{"title":"Mapping between FSDP sharding strategies and DeepSpeed ZeRO Stages","local":"mapping-between-fsdp-sharding-strategies-and-deepspeed-zero-stages","sections":[],"depth":2},{"title":"A few caveats to be aware of","local":"a-few-caveats-to-be-aware-of","sections":[],"depth":2}],"depth":1}';function rl(Me){return Qt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class hl extends qt{constructor(u){super(),Kt(this,u,rl,nl,Vt,{})}}export{hl as component};
