import{s as Ft,o as Et,n as St}from"../chunks/scheduler.b9285784.js";import{S as Lt,i as Dt,e as p,s as a,c as u,h as Pt,a as i,d as s,b as n,f as Ht,g as r,j as c,k as xt,l as Kt,m as l,n as M,t as d,o as J,p as T}from"../chunks/index.26bc89a1.js";import{T as Qt}from"../chunks/Tip.e4eba3d6.js";import{C as Ot,H as ye,E as es}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.7242ff19.js";import{C as w}from"../chunks/CodeBlock.c8ee1711.js";function ts(we){let o,m='This guide assumes that you have read and understood the <a href="./deepspeed.md">DeepSpeed usage guide</a>.';return{c(){o=p("p"),o.innerHTML=m},l(y){o=i(y,"P",{"data-svelte-h":!0}),c(o)!=="svelte-vrzv4k"&&(o.innerHTML=m)},m(y,h){l(y,o,h)},p:St,d(y){y&&s(o)}}}function ss(we){let o,m="DeepSpeed will raise an error if <code>train_micro_batch_size_per_gpu</code> isn’t specified, even if this particular model isn’t being trained.";return{c(){o=p("p"),o.innerHTML=m},l(y){o=i(y,"P",{"data-svelte-h":!0}),c(o)!=="svelte-qz5bxi"&&(o.innerHTML=m)},m(y,h){l(y,o,h)},p:St,d(y){y&&s(o)}}}function ls(we){let o,m,y,h,I,he,U,je,j,fe,b,yt="Running multiple models with Accelerate and DeepSpeed is useful for:",Ie,g,wt='<li>Knowledge distillation</li> <li>Post-training techniques like RLHF (see the <a href="https://github.com/huggingface/trl" rel="nofollow">TRL</a> library for more examples)</li> <li>Training multiple models at once</li>',Ue,C,mt="Currently, Accelerate has a <strong>very experimental API</strong> to help you use multiple models.",be,_,ht="This tutorial will focus on two common use cases:",ge,$,jt="<li>Knowledge distillation, where a smaller student model is trained to mimic a larger, better-performing teacher.  If the student model fits on a single GPU, we can use ZeRO-2 for training and ZeRO-3 to shard the teacher for inference. This is significantly faster than using ZeRO-3 for both models.</li> <li>Training multiple <em>disjoint</em> models at once.</li>",Ce,Z,_e,B,ft="Knowledge distillation is a good example of using multiple models, but only training one of them.",$e,W,It='Normally, you would use a single <a href="/docs/accelerate/main/en/package_reference/deepspeed#accelerate.DeepSpeedPlugin">utils.DeepSpeedPlugin</a> for both models. However, in this case, there are two separate configurations. Accelerate allows you to create and use multiple plugins <strong>if and only if</strong> they are in a <code>dict</code> so that you can reference and enable the proper plugin when needed.',Ze,X,Be,G,Ut="The <code>zero2_config.json</code> should be configured for full training (so specify <code>scheduler</code> and <code>optimizer</code> if you are not utilizing your own), while <code>zero3_config.json</code> should only be configured for the inference model, as shown in the example below.",We,v,Xe,R,bt="An example <code>zero2_config.json</code> configuration is shown below.",Ge,V,ve,f,Re,z,gt='From here, create a single <a href="/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a> and pass in both configurations.',Ve,Y,ze,A,Ct="Now let’s see how to use them.",Ye,k,Ae,q,_t='By default, Accelerate sets the first item in the <code>dict</code> as the default or enabled plugin (<code>&quot;student&quot;</code> plugin). Verify this by using the <a href="/docs/accelerate/main/en/package_reference/deepspeed#accelerate.utils.get_active_deepspeed_plugin">utils.deepspeed.get_active_deepspeed_plugin()</a> function to see which plugin is enabled.',ke,N,qe,H,$t="<code>AcceleratorState</code> also keeps the active DeepSpeed plugin saved in <code>state.deepspeed_plugin</code>.",Ne,x,He,Q,Zt="Since <code>student</code> is the currently active plugin, let’s go ahead and prepare the model, optimizer, and scheduler.",xe,S,Qe,F,Bt="Now it’s time to deal with the teacher model.",Se,E,Fe,L,Wt='First, you need to specify in <a href="/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a> that the <code>zero3_config.json</code> configuration should be used.',Ee,D,Le,P,Xt=`This disables the <code>&quot;student&quot;</code> plugin and enables the <code>&quot;teacher&quot;</code> plugin instead. The
DeepSpeed stateful config inside of Transformers is updated, and it changes which plugin configuration gets called when using
<code>deepspeed.initialize()</code>. This allows you to use the automatic <code>deepspeed.zero.Init</code>  context manager integration Transformers provides.`,De,K,Pe,O,Gt="Otherwise, you should manually initialize the model with <code>deepspeed.zero.Init</code>.",Ke,ee,Oe,te,et,se,vt="From here, your training loop can be whatever you like, as long as <code>teacher_model</code> is never being trained on.",tt,le,st,ae,lt,ne,Rt=`Training multiple models is a more complicated scenario.
In its current state, we assume each model is <strong>completely disjointed</strong> from the other during training.`,at,pe,Vt='This scenario still requires two <a href="/docs/accelerate/main/en/package_reference/deepspeed#accelerate.DeepSpeedPlugin">utils.DeepSpeedPlugin</a>’s to be made. However, you also need a second <a href="/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a>, since different <code>deepspeed</code> engines are being called at different times. A single <a href="/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a> can only carry one instance at a time.',nt,ie,zt='Since the <a href="/docs/accelerate/main/en/package_reference/state#accelerate.state.AcceleratorState">state.AcceleratorState</a> is a stateful object though, it is already aware of both <a href="/docs/accelerate/main/en/package_reference/deepspeed#accelerate.DeepSpeedPlugin">utils.DeepSpeedPlugin</a>’s available. You can just instantiate a second <a href="/docs/accelerate/main/en/package_reference/accelerator#accelerate.Accelerator">Accelerator</a> with no extra arguments.',pt,ce,it,oe,Yt=`You can call either <code>first_accelerator.state.select_deepspeed_plugin()</code> to enable or disable
a particular plugin, and then call <code>prepare</code>.`,ct,ue,ot,re,At="And now you can train:",ut,Me,rt,de,Mt,Je,kt='To see more examples, please check out the <a href="https://github.com/huggingface/accelerate/blob/main/src/accelerate/test_utils/scripts/external_deps/test_ds_multiple_model.py" rel="nofollow">related tests</a> currently in [Accelerate].',dt,Te,Jt,me,Tt;return I=new Ot({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),U=new ye({props:{title:"Using multiple models with DeepSpeed",local:"using-multiple-models-with-deepspeed",headingTag:"h1"}}),j=new Qt({props:{warning:!0,$$slots:{default:[ts]},$$scope:{ctx:we}}}),Z=new ye({props:{title:"Knowledge distillation",local:"knowledge-distillation",headingTag:"h2"}}),X=new w({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUudXRpbHMlMjBpbXBvcnQlMjBEZWVwU3BlZWRQbHVnaW4lMEElMEF6ZXJvMl9wbHVnaW4lMjAlM0QlMjBEZWVwU3BlZWRQbHVnaW4oaGZfZHNfY29uZmlnJTNEJTIyemVybzJfY29uZmlnLmpzb24lMjIpJTBBemVybzNfcGx1Z2luJTIwJTNEJTIwRGVlcFNwZWVkUGx1Z2luKGhmX2RzX2NvbmZpZyUzRCUyMnplcm8zX2NvbmZpZy5qc29uJTIyKSUwQSUwQWRlZXBzcGVlZF9wbHVnaW5zJTIwJTNEJTIwJTdCJTIyc3R1ZGVudCUyMiUzQSUyMHplcm8yX3BsdWdpbiUyQyUyMCUyMnRlYWNoZXIlMjIlM0ElMjB6ZXJvM19wbHVnaW4lN0Q=",highlighted:`<span class="hljs-keyword">from</span> accelerate.utils <span class="hljs-keyword">import</span> DeepSpeedPlugin

zero2_plugin = DeepSpeedPlugin(hf_ds_config=<span class="hljs-string">&quot;zero2_config.json&quot;</span>)
zero3_plugin = DeepSpeedPlugin(hf_ds_config=<span class="hljs-string">&quot;zero3_config.json&quot;</span>)

deepspeed_plugins = {<span class="hljs-string">&quot;student&quot;</span>: zero2_plugin, <span class="hljs-string">&quot;teacher&quot;</span>: zero3_plugin}`,wrap:!1}}),v=new w({props:{code:"JTdCJTBBJTIwJTIwJTIwJTIwJTIyYmYxNiUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmVuYWJsZWQlMjIlM0ElMjAlMjJhdXRvJTIyJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIyemVyb19vcHRpbWl6YXRpb24lMjIlM0ElMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGFnZSUyMiUzQSUyMDMlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJvdmVybGFwX2NvbW0lMjIlM0ElMjB0cnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycmVkdWNlX2J1Y2tldF9zaXplJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnN0YWdlM19wcmVmZXRjaF9idWNrZXRfc2l6ZSUyMiUzQSUyMCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGFnZTNfcGFyYW1fcGVyc2lzdGVuY2VfdGhyZXNob2xkJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnN0YWdlM19tYXhfbGl2ZV9wYXJhbWV0ZXJzJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnN0YWdlM19tYXhfcmV1c2VfZGlzdGFuY2UlMjIlM0ElMjAlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIydHJhaW5fbWljcm9fYmF0Y2hfc2l6ZV9wZXJfZ3B1JTIyJTNBJTIwMSUwQSU3RA==",highlighted:`<span class="hljs-punctuation">{</span>
    <span class="hljs-attr">&quot;bf16&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;enabled&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;zero_optimization&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;stage&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">3</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;overlap_comm&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;reduce_bucket_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;stage3_prefetch_bucket_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;stage3_param_persistence_threshold&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;stage3_max_live_parameters&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;stage3_max_reuse_distance&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;train_micro_batch_size_per_gpu&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">1</span>
<span class="hljs-punctuation">}</span>`,wrap:!1}}),V=new w({props:{code:"JTdCJTBBJTIwJTIwJTIwJTIwJTIyYmYxNiUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmVuYWJsZWQlMjIlM0ElMjAlMjJhdXRvJTIyJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIyb3B0aW1pemVyJTIyJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIydHlwZSUyMiUzQSUyMCUyMkFkYW1XJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycGFyYW1zJTIyJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIybHIlMjIlM0ElMjAlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyd2VpZ2h0X2RlY2F5JTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnRvcmNoX2FkYW0lMjIlM0ElMjB0cnVlJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyYWRhbV93X21vZGUlMjIlM0ElMjB0cnVlJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIyc2NoZWR1bGVyJTIyJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIydHlwZSUyMiUzQSUyMCUyMldhcm11cExSJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIycGFyYW1zJTIyJTNBJTIwJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyd2FybXVwX21pbl9sciUyMiUzQSUyMCUyMmF1dG8lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJ3YXJtdXBfbWF4X2xyJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMndhcm11cF9udW1fc3RlcHMlMjIlM0ElMjAlMjJhdXRvJTIyJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIyemVyb19vcHRpbWl6YXRpb24lMjIlM0ElMjAlN0IlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJzdGFnZSUyMiUzQSUyMDIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJvZmZsb2FkX29wdGltaXplciUyMiUzQSUyMCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMmRldmljZSUyMiUzQSUyMCUyMmNwdSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnBpbl9tZW1vcnklMjIlM0ElMjB0cnVlJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIyZ3JhZGllbnRfYWNjdW11bGF0aW9uX3N0ZXBzJTIyJTNBJTIwMSUyQyUwQSUyMCUyMCUyMCUyMCUyMmdyYWRpZW50X2NsaXBwaW5nJTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMnRyYWluX2JhdGNoX3NpemUlMjIlM0ElMjAlMjJhdXRvJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIydHJhaW5fbWljcm9fYmF0Y2hfc2l6ZV9wZXJfZ3B1JTIyJTNBJTIwJTIyYXV0byUyMiUyQyUwQSU3RA==",highlighted:`<span class="hljs-punctuation">{</span>
    <span class="hljs-attr">&quot;bf16&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;enabled&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;optimizer&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;type&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;AdamW&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;params&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
            <span class="hljs-attr">&quot;lr&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;weight_decay&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;torch_adam&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;adam_w_mode&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span>
        <span class="hljs-punctuation">}</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;scheduler&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;type&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;WarmupLR&quot;</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;params&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
            <span class="hljs-attr">&quot;warmup_min_lr&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;warmup_max_lr&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;warmup_num_steps&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span>
        <span class="hljs-punctuation">}</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;zero_optimization&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
        <span class="hljs-attr">&quot;stage&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">2</span><span class="hljs-punctuation">,</span>
        <span class="hljs-attr">&quot;offload_optimizer&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-punctuation">{</span>
            <span class="hljs-attr">&quot;device&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;cpu&quot;</span><span class="hljs-punctuation">,</span>
            <span class="hljs-attr">&quot;pin_memory&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-literal"><span class="hljs-keyword">true</span></span>
        <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-punctuation">}</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;gradient_accumulation_steps&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-number">1</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;gradient_clipping&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;train_batch_size&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
    <span class="hljs-attr">&quot;train_micro_batch_size_per_gpu&quot;</span><span class="hljs-punctuation">:</span> <span class="hljs-string">&quot;auto&quot;</span><span class="hljs-punctuation">,</span>
<span class="hljs-punctuation">}</span>`,wrap:!1}}),f=new Qt({props:{$$slots:{default:[ss]},$$scope:{ctx:we}}}),Y=new w({props:{code:"ZnJvbSUyMGFjY2VsZXJhdGUlMjBpbXBvcnQlMjBBY2NlbGVyYXRvciUwQSUwQWFjY2VsZXJhdG9yJTIwJTNEJTIwQWNjZWxlcmF0b3IoZGVlcHNwZWVkX3BsdWdpbnMlM0RkZWVwc3BlZWRfcGx1Z2lucyk=",highlighted:`<span class="hljs-keyword">from</span> accelerate <span class="hljs-keyword">import</span> Accelerator

accelerator = Accelerator(deepspeed_plugins=deepspeed_plugins)`,wrap:!1}}),k=new ye({props:{title:"Student model",local:"student-model",headingTag:"h3"}}),N=new w({props:{code:"YWN0aXZlX3BsdWdpbiUyMCUzRCUyMGdldF9hY3RpdmVfZGVlcHNwZWVkX3BsdWdpbihhY2NlbGVyYXRvci5zdGF0ZSklMEFhc3NlcnQlMjBhY3RpdmVfcGx1Z2luJTIwaXMlMjBkZWVwc3BlZWRfcGx1Z2lucyU1QiUyMnN0dWRlbnQlMjIlNUQ=",highlighted:`active_plugin = get_active_deepspeed_plugin(accelerator.state)
<span class="hljs-keyword">assert</span> active_plugin <span class="hljs-keyword">is</span> deepspeed_plugins[<span class="hljs-string">&quot;student&quot;</span>]`,wrap:!1}}),x=new w({props:{code:"YXNzZXJ0JTIwYWN0aXZlX3BsdWdpbiUyMGlzJTIwYWNjZWxlcmF0b3IuZGVlcHNwZWVkX3BsdWdpbg==",highlighted:'<span class="hljs-keyword">assert</span> active_plugin <span class="hljs-keyword">is</span> accelerator.deepspeed_plugin',wrap:!1}}),S=new w({props:{code:"c3R1ZGVudF9tb2RlbCUyQyUyMG9wdGltaXplciUyQyUyMHNjaGVkdWxlciUyMCUzRCUyMC4uLiUwQXN0dWRlbnRfbW9kZWwlMkMlMjBvcHRpbWl6ZXIlMkMlMjBzY2hlZHVsZXIlMkMlMjB0cmFpbl9kYXRhbG9hZGVyJTIwJTNEJTIwYWNjZWxlcmF0b3IucHJlcGFyZShzdHVkZW50X21vZGVsJTJDJTIwb3B0aW1pemVyJTJDJTIwc2NoZWR1bGVyJTJDJTIwdHJhaW5fZGF0YWxvYWRlcik=",highlighted:`student_model, optimizer, scheduler = ...
student_model, optimizer, scheduler, train_dataloader = accelerator.prepare(student_model, optimizer, scheduler, train_dataloader)`,wrap:!1}}),E=new ye({props:{title:"Teacher model",local:"teacher-model",headingTag:"h3"}}),D=new w({props:{code:"YWNjZWxlcmF0b3Iuc3RhdGUuc2VsZWN0X2RlZXBzcGVlZF9wbHVnaW4oJTIydGVhY2hlciUyMik=",highlighted:'accelerator.state.select_deepspeed_plugin(<span class="hljs-string">&quot;teacher&quot;</span>)',wrap:!1}}),K=new w({props:{code:"dGVhY2hlcl9tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoLi4uKSUwQXRlYWNoZXJfbW9kZWwlMjAlM0QlMjBhY2NlbGVyYXRvci5wcmVwYXJlKHRlYWNoZXJfbW9kZWwp",highlighted:`teacher_model = AutoModel.from_pretrained(...)
teacher_model = accelerator.prepare(teacher_model)`,wrap:!1}}),ee=new w({props:{code:"d2l0aCUyMGRlZXBzcGVlZC56ZXJvLkluaXQoYWNjZWxlcmF0b3IuZGVlcHNwZWVkX3BsdWdpbi5jb25maWcpJTNBJTBBJTIwJTIwJTIwJTIwbW9kZWwlMjAlM0QlMjBNeU1vZGVsKC4uLik=",highlighted:`<span class="hljs-keyword">with</span> deepspeed.zero.Init(accelerator.deepspeed_plugin.config):
    model = MyModel(...)`,wrap:!1}}),te=new ye({props:{title:"Training",local:"training",headingTag:"h3"}}),le=new w({props:{code:"dGVhY2hlcl9tb2RlbC5ldmFsKCklMEFzdHVkZW50X21vZGVsLnRyYWluKCklMEFmb3IlMjBiYXRjaCUyMGluJTIwdHJhaW5fZGF0YWxvYWRlciUzQSUwQSUyMCUyMCUyMCUyMHdpdGglMjB0b3JjaC5ub19ncmFkKCklM0ElMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBvdXRwdXRfdGVhY2hlciUyMCUzRCUyMHRlYWNoZXJfbW9kZWwoKipiYXRjaCklMEElMjAlMjAlMjAlMjBvdXRwdXRfc3R1ZGVudCUyMCUzRCUyMHN0dWRlbnRfbW9kZWwoKipiYXRjaCklMEElMjAlMjAlMjAlMjAlMjMlMjBDb21iaW5lJTIwdGhlJTIwbG9zc2VzJTIwb3IlMjBtb2RpZnklMjBpdCUyMGluJTIwc29tZSUyMHdheSUwQSUyMCUyMCUyMCUyMGxvc3MlMjAlM0QlMjBvdXRwdXRfdGVhY2hlci5sb3NzJTIwJTJCJTIwb3V0cHV0X3N0dWRlbnQubG9zcyUwQSUyMCUyMCUyMCUyMGFjY2VsZXJhdG9yLmJhY2t3YXJkKGxvc3MpJTBBJTIwJTIwJTIwJTIwb3B0aW1pemVyLnN0ZXAoKSUwQSUyMCUyMCUyMCUyMHNjaGVkdWxlci5zdGVwKCklMEElMjAlMjAlMjAlMjBvcHRpbWl6ZXIuemVyb19ncmFkKCk=",highlighted:`teacher_model.<span class="hljs-built_in">eval</span>()
student_model.train()
<span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> train_dataloader:
    <span class="hljs-keyword">with</span> torch.no_grad():
        output_teacher = teacher_model(**batch)
    output_student = student_model(**batch)
    <span class="hljs-comment"># Combine the losses or modify it in some way</span>
    loss = output_teacher.loss + output_student.loss
    accelerator.backward(loss)
    optimizer.step()
    scheduler.step()
    optimizer.zero_grad()`,wrap:!1}}),ae=new ye({props:{title:"Train multiple disjoint models",local:"train-multiple-disjoint-models",headingTag:"h2"}}),ce=new w({props:{code:"Zmlyc3RfYWNjZWxlcmF0b3IlMjAlM0QlMjBBY2NlbGVyYXRvcihkZWVwc3BlZWRfcGx1Z2lucyUzRGRlZXBzcGVlZF9wbHVnaW5zKSUwQXNlY29uZF9hY2NlbGVyYXRvciUyMCUzRCUyMEFjY2VsZXJhdG9yKCk=",highlighted:`first_accelerator = Accelerator(deepspeed_plugins=deepspeed_plugins)
second_accelerator = Accelerator()`,wrap:!1}}),ue=new w({props:{code:"JTIzJTIwY2FuJTIwYmUlMjAlNjBhY2NlbGVyYXRvcl8wJTYwJTJDJTIwJTYwYWNjZWxlcmF0b3JfMSU2MCUyQyUyMG9yJTIwYnklMjBjYWxsaW5nJTIwJTYwQWNjZWxlcmF0b3JTdGF0ZSgpLnNlbGVjdF9kZWVwc3BlZWRfcGx1Z2luKC4uLiklNjAlMEFmaXJzdF9hY2NlbGVyYXRvci5zdGF0ZS5zZWxlY3RfZGVlcHNwZWVkX3BsdWdpbiglMjJmaXJzdF9tb2RlbCUyMiklMEFmaXJzdF9tb2RlbCUyMCUzRCUyMEF1dG9Nb2RlbC5mcm9tX3ByZXRyYWluZWQoLi4uKSUwQSUyMyUyMEZvciUyMHRoaXMlMjBleGFtcGxlJTJDJTIwJTYwZ2V0X3RyYWluaW5nX2l0ZW1zJTYwJTIwaXMlMjBhJTIwbm9uZXhpc3RlbnQlMjBmdW5jdGlvbiUyMHRoYXQlMjBnZXRzJTIwdGhlJTIwc2V0dXAlMjB3ZSUyMG5lZWQlMjBmb3IlMjB0cmFpbmluZyUwQWZpcnN0X29wdGltaXplciUyQyUyMGZpcnN0X3NjaGVkdWxlciUyQyUyMHRyYWluX2RsJTJDJTIwZXZhbF9kbCUyMCUzRCUyMGdldF90cmFpbmluZ19pdGVtcyhtb2RlbDEpJTBBZmlyc3RfbW9kZWwlMkMlMjBmaXJzdF9vcHRpbWl6ZXIlMkMlMjBmaXJzdF9zY2hlZHVsZXIlMkMlMjB0cmFpbl9kbCUyQyUyMGV2YWxfZGwlMjAlM0QlMjBhY2NlbGVyYXRvci5wcmVwYXJlKCUwQSUyMCUyMCUyMCUyMGZpcnN0X21vZGVsJTJDJTIwZmlyc3Rfb3B0aW1pemVyJTJDJTIwZmlyc3Rfc2NoZWR1bGVyJTJDJTIwdHJhaW5fZGwlMkMlMjBldmFsX2RsJTBBKSUwQSUwQXNlY29uZF9hY2NlbGVyYXRvci5zdGF0ZS5zZWxlY3RfZGVlcHNwZWVkX3BsdWdpbiglMjJzZWNvbmRfbW9kZWwlMjIpJTBBc2Vjb25kX21vZGVsJTIwJTNEJTIwQXV0b01vZGVsLmZyb21fcHJldHJhaW5lZCguLi4pJTBBJTIzJTIwRm9yJTIwdGhpcyUyMGV4YW1wbGUlMkMlMjAlNjBnZXRfdHJhaW5pbmdfaXRlbXMlNjAlMjBpcyUyMGElMjBub25leGlzdGVudCUyMGZ1bmN0aW9uJTIwdGhhdCUyMGdldHMlMjB0aGUlMjBzZXR1cCUyMHdlJTIwbmVlZCUyMGZvciUyMHRyYWluaW5nJTBBc2Vjb25kX29wdGltaXplciUyQyUyMHNlY29uZF9zY2hlZHVsZXIlMkMlMjBfJTJDJTIwXyUyMCUzRCUyMGdldF90cmFpbmluZ19pdGVtcyhtb2RlbDIpJTBBc2Vjb25kX21vZGVsJTJDJTIwc2Vjb25kX29wdGltaXplciUyQyUyMHNlY29uZF9zY2hlZHVsZXIlMjAlM0QlMjBhY2NlbGVyYXRvci5wcmVwYXJlKCUwQSUyMCUyMCUyMCUyMHNlY29uZF9tb2RlbCUyQyUyMHNlY29uZF9vcHRpbWl6ZXIlMkMlMjBzZWNvbmRfc2NoZWR1bGVyJTBBKQ==",highlighted:`<span class="hljs-comment"># can be \`accelerator_0\`, \`accelerator_1\`, or by calling \`AcceleratorState().select_deepspeed_plugin(...)\`</span>
first_accelerator.state.select_deepspeed_plugin(<span class="hljs-string">&quot;first_model&quot;</span>)
first_model = AutoModel.from_pretrained(...)
<span class="hljs-comment"># For this example, \`get_training_items\` is a nonexistent function that gets the setup we need for training</span>
first_optimizer, first_scheduler, train_dl, eval_dl = get_training_items(model1)
first_model, first_optimizer, first_scheduler, train_dl, eval_dl = accelerator.prepare(
    first_model, first_optimizer, first_scheduler, train_dl, eval_dl
)

second_accelerator.state.select_deepspeed_plugin(<span class="hljs-string">&quot;second_model&quot;</span>)
second_model = AutoModel.from_pretrained(...)
<span class="hljs-comment"># For this example, \`get_training_items\` is a nonexistent function that gets the setup we need for training</span>
second_optimizer, second_scheduler, _, _ = get_training_items(model2)
second_model, second_optimizer, second_scheduler = accelerator.prepare(
    second_model, second_optimizer, second_scheduler
)`,wrap:!1}}),Me=new w({props:{code:"Zm9yJTIwYmF0Y2glMjBpbiUyMGRsJTNBJTBBJTIwJTIwJTIwJTIwb3V0cHV0czElMjAlM0QlMjBmaXJzdF9tb2RlbCgqKmJhdGNoKSUwQSUyMCUyMCUyMCUyMGZpcnN0X2FjY2VsZXJhdG9yLmJhY2t3YXJkKG91dHB1dHMxLmxvc3MpJTBBJTIwJTIwJTIwJTIwZmlyc3Rfb3B0aW1pemVyLnN0ZXAoKSUwQSUyMCUyMCUyMCUyMGZpcnN0X3NjaGVkdWxlci5zdGVwKCklMEElMjAlMjAlMjAlMjBmaXJzdF9vcHRpbWl6ZXIuemVyb19ncmFkKCklMEElMjAlMjAlMjAlMjAlMEElMjAlMjAlMjAlMjBvdXRwdXRzMiUyMCUzRCUyMG1vZGVsMigqKmJhdGNoKSUwQSUyMCUyMCUyMCUyMHNlY29uZF9hY2NlbGVyYXRvci5iYWNrd2FyZChvdXRwdXRzMi5sb3NzKSUwQSUyMCUyMCUyMCUyMHNlY29uZF9vcHRpbWl6ZXIuc3RlcCgpJTBBJTIwJTIwJTIwJTIwc2Vjb25kX3NjaGVkdWxlci5zdGVwKCklMEElMjAlMjAlMjAlMjBzZWNvbmRfb3B0aW1pemVyLnplcm9fZ3JhZCgp",highlighted:`<span class="hljs-keyword">for</span> batch <span class="hljs-keyword">in</span> dl:
    outputs1 = first_model(**batch)
    first_accelerator.backward(outputs1.loss)
    first_optimizer.step()
    first_scheduler.step()
    first_optimizer.zero_grad()
    
    outputs2 = model2(**batch)
    second_accelerator.backward(outputs2.loss)
    second_optimizer.step()
    second_scheduler.step()
    second_optimizer.zero_grad()`,wrap:!1}}),de=new ye({props:{title:"Resources",local:"resources",headingTag:"h2"}}),Te=new es({props:{source:"https://github.com/huggingface/accelerate/blob/main/docs/source/usage_guides/deepspeed_multiple_model.md"}}),{c(){o=p("meta"),m=a(),y=p("p"),h=a(),u(I.$$.fragment),he=a(),u(U.$$.fragment),je=a(),u(j.$$.fragment),fe=a(),b=p("p"),b.textContent=yt,Ie=a(),g=p("ul"),g.innerHTML=wt,Ue=a(),C=p("p"),C.innerHTML=mt,be=a(),_=p("p"),_.textContent=ht,ge=a(),$=p("ol"),$.innerHTML=jt,Ce=a(),u(Z.$$.fragment),_e=a(),B=p("p"),B.textContent=ft,$e=a(),W=p("p"),W.innerHTML=It,Ze=a(),u(X.$$.fragment),Be=a(),G=p("p"),G.innerHTML=Ut,We=a(),u(v.$$.fragment),Xe=a(),R=p("p"),R.innerHTML=bt,Ge=a(),u(V.$$.fragment),ve=a(),u(f.$$.fragment),Re=a(),z=p("p"),z.innerHTML=gt,Ve=a(),u(Y.$$.fragment),ze=a(),A=p("p"),A.textContent=Ct,Ye=a(),u(k.$$.fragment),Ae=a(),q=p("p"),q.innerHTML=_t,ke=a(),u(N.$$.fragment),qe=a(),H=p("p"),H.innerHTML=$t,Ne=a(),u(x.$$.fragment),He=a(),Q=p("p"),Q.innerHTML=Zt,xe=a(),u(S.$$.fragment),Qe=a(),F=p("p"),F.textContent=Bt,Se=a(),u(E.$$.fragment),Fe=a(),L=p("p"),L.innerHTML=Wt,Ee=a(),u(D.$$.fragment),Le=a(),P=p("p"),P.innerHTML=Xt,De=a(),u(K.$$.fragment),Pe=a(),O=p("p"),O.innerHTML=Gt,Ke=a(),u(ee.$$.fragment),Oe=a(),u(te.$$.fragment),et=a(),se=p("p"),se.innerHTML=vt,tt=a(),u(le.$$.fragment),st=a(),u(ae.$$.fragment),lt=a(),ne=p("p"),ne.innerHTML=Rt,at=a(),pe=p("p"),pe.innerHTML=Vt,nt=a(),ie=p("p"),ie.innerHTML=zt,pt=a(),u(ce.$$.fragment),it=a(),oe=p("p"),oe.innerHTML=Yt,ct=a(),u(ue.$$.fragment),ot=a(),re=p("p"),re.textContent=At,ut=a(),u(Me.$$.fragment),rt=a(),u(de.$$.fragment),Mt=a(),Je=p("p"),Je.innerHTML=kt,dt=a(),u(Te.$$.fragment),Jt=a(),me=p("p"),this.h()},l(e){const t=Pt("svelte-u9bgzb",document.head);o=i(t,"META",{name:!0,content:!0}),t.forEach(s),m=n(e),y=i(e,"P",{}),Ht(y).forEach(s),h=n(e),r(I.$$.fragment,e),he=n(e),r(U.$$.fragment,e),je=n(e),r(j.$$.fragment,e),fe=n(e),b=i(e,"P",{"data-svelte-h":!0}),c(b)!=="svelte-1b1gxyz"&&(b.textContent=yt),Ie=n(e),g=i(e,"UL",{"data-svelte-h":!0}),c(g)!=="svelte-7mvbd"&&(g.innerHTML=wt),Ue=n(e),C=i(e,"P",{"data-svelte-h":!0}),c(C)!=="svelte-e8xs1z"&&(C.innerHTML=mt),be=n(e),_=i(e,"P",{"data-svelte-h":!0}),c(_)!=="svelte-owdl16"&&(_.textContent=ht),ge=n(e),$=i(e,"OL",{"data-svelte-h":!0}),c($)!=="svelte-to0mxc"&&($.innerHTML=jt),Ce=n(e),r(Z.$$.fragment,e),_e=n(e),B=i(e,"P",{"data-svelte-h":!0}),c(B)!=="svelte-zc35yl"&&(B.textContent=ft),$e=n(e),W=i(e,"P",{"data-svelte-h":!0}),c(W)!=="svelte-15gxojo"&&(W.innerHTML=It),Ze=n(e),r(X.$$.fragment,e),Be=n(e),G=i(e,"P",{"data-svelte-h":!0}),c(G)!=="svelte-4l2obz"&&(G.innerHTML=Ut),We=n(e),r(v.$$.fragment,e),Xe=n(e),R=i(e,"P",{"data-svelte-h":!0}),c(R)!=="svelte-2zuo2r"&&(R.innerHTML=bt),Ge=n(e),r(V.$$.fragment,e),ve=n(e),r(f.$$.fragment,e),Re=n(e),z=i(e,"P",{"data-svelte-h":!0}),c(z)!=="svelte-t7yg3i"&&(z.innerHTML=gt),Ve=n(e),r(Y.$$.fragment,e),ze=n(e),A=i(e,"P",{"data-svelte-h":!0}),c(A)!=="svelte-edf8im"&&(A.textContent=Ct),Ye=n(e),r(k.$$.fragment,e),Ae=n(e),q=i(e,"P",{"data-svelte-h":!0}),c(q)!=="svelte-bgcpa4"&&(q.innerHTML=_t),ke=n(e),r(N.$$.fragment,e),qe=n(e),H=i(e,"P",{"data-svelte-h":!0}),c(H)!=="svelte-m0abi0"&&(H.innerHTML=$t),Ne=n(e),r(x.$$.fragment,e),He=n(e),Q=i(e,"P",{"data-svelte-h":!0}),c(Q)!=="svelte-107qzzz"&&(Q.innerHTML=Zt),xe=n(e),r(S.$$.fragment,e),Qe=n(e),F=i(e,"P",{"data-svelte-h":!0}),c(F)!=="svelte-202ivx"&&(F.textContent=Bt),Se=n(e),r(E.$$.fragment,e),Fe=n(e),L=i(e,"P",{"data-svelte-h":!0}),c(L)!=="svelte-11n7i8s"&&(L.innerHTML=Wt),Ee=n(e),r(D.$$.fragment,e),Le=n(e),P=i(e,"P",{"data-svelte-h":!0}),c(P)!=="svelte-171sb04"&&(P.innerHTML=Xt),De=n(e),r(K.$$.fragment,e),Pe=n(e),O=i(e,"P",{"data-svelte-h":!0}),c(O)!=="svelte-14r7t5t"&&(O.innerHTML=Gt),Ke=n(e),r(ee.$$.fragment,e),Oe=n(e),r(te.$$.fragment,e),et=n(e),se=i(e,"P",{"data-svelte-h":!0}),c(se)!=="svelte-1843po4"&&(se.innerHTML=vt),tt=n(e),r(le.$$.fragment,e),st=n(e),r(ae.$$.fragment,e),lt=n(e),ne=i(e,"P",{"data-svelte-h":!0}),c(ne)!=="svelte-16bzhhs"&&(ne.innerHTML=Rt),at=n(e),pe=i(e,"P",{"data-svelte-h":!0}),c(pe)!=="svelte-nnmvub"&&(pe.innerHTML=Vt),nt=n(e),ie=i(e,"P",{"data-svelte-h":!0}),c(ie)!=="svelte-177w6m0"&&(ie.innerHTML=zt),pt=n(e),r(ce.$$.fragment,e),it=n(e),oe=i(e,"P",{"data-svelte-h":!0}),c(oe)!=="svelte-5wgsij"&&(oe.innerHTML=Yt),ct=n(e),r(ue.$$.fragment,e),ot=n(e),re=i(e,"P",{"data-svelte-h":!0}),c(re)!=="svelte-1oe955w"&&(re.textContent=At),ut=n(e),r(Me.$$.fragment,e),rt=n(e),r(de.$$.fragment,e),Mt=n(e),Je=i(e,"P",{"data-svelte-h":!0}),c(Je)!=="svelte-1hqyy97"&&(Je.innerHTML=kt),dt=n(e),r(Te.$$.fragment,e),Jt=n(e),me=i(e,"P",{}),Ht(me).forEach(s),this.h()},h(){xt(o,"name","hf:doc:metadata"),xt(o,"content",as)},m(e,t){Kt(document.head,o),l(e,m,t),l(e,y,t),l(e,h,t),M(I,e,t),l(e,he,t),M(U,e,t),l(e,je,t),M(j,e,t),l(e,fe,t),l(e,b,t),l(e,Ie,t),l(e,g,t),l(e,Ue,t),l(e,C,t),l(e,be,t),l(e,_,t),l(e,ge,t),l(e,$,t),l(e,Ce,t),M(Z,e,t),l(e,_e,t),l(e,B,t),l(e,$e,t),l(e,W,t),l(e,Ze,t),M(X,e,t),l(e,Be,t),l(e,G,t),l(e,We,t),M(v,e,t),l(e,Xe,t),l(e,R,t),l(e,Ge,t),M(V,e,t),l(e,ve,t),M(f,e,t),l(e,Re,t),l(e,z,t),l(e,Ve,t),M(Y,e,t),l(e,ze,t),l(e,A,t),l(e,Ye,t),M(k,e,t),l(e,Ae,t),l(e,q,t),l(e,ke,t),M(N,e,t),l(e,qe,t),l(e,H,t),l(e,Ne,t),M(x,e,t),l(e,He,t),l(e,Q,t),l(e,xe,t),M(S,e,t),l(e,Qe,t),l(e,F,t),l(e,Se,t),M(E,e,t),l(e,Fe,t),l(e,L,t),l(e,Ee,t),M(D,e,t),l(e,Le,t),l(e,P,t),l(e,De,t),M(K,e,t),l(e,Pe,t),l(e,O,t),l(e,Ke,t),M(ee,e,t),l(e,Oe,t),M(te,e,t),l(e,et,t),l(e,se,t),l(e,tt,t),M(le,e,t),l(e,st,t),M(ae,e,t),l(e,lt,t),l(e,ne,t),l(e,at,t),l(e,pe,t),l(e,nt,t),l(e,ie,t),l(e,pt,t),M(ce,e,t),l(e,it,t),l(e,oe,t),l(e,ct,t),M(ue,e,t),l(e,ot,t),l(e,re,t),l(e,ut,t),M(Me,e,t),l(e,rt,t),M(de,e,t),l(e,Mt,t),l(e,Je,t),l(e,dt,t),M(Te,e,t),l(e,Jt,t),l(e,me,t),Tt=!0},p(e,[t]){const qt={};t&2&&(qt.$$scope={dirty:t,ctx:e}),j.$set(qt);const Nt={};t&2&&(Nt.$$scope={dirty:t,ctx:e}),f.$set(Nt)},i(e){Tt||(d(I.$$.fragment,e),d(U.$$.fragment,e),d(j.$$.fragment,e),d(Z.$$.fragment,e),d(X.$$.fragment,e),d(v.$$.fragment,e),d(V.$$.fragment,e),d(f.$$.fragment,e),d(Y.$$.fragment,e),d(k.$$.fragment,e),d(N.$$.fragment,e),d(x.$$.fragment,e),d(S.$$.fragment,e),d(E.$$.fragment,e),d(D.$$.fragment,e),d(K.$$.fragment,e),d(ee.$$.fragment,e),d(te.$$.fragment,e),d(le.$$.fragment,e),d(ae.$$.fragment,e),d(ce.$$.fragment,e),d(ue.$$.fragment,e),d(Me.$$.fragment,e),d(de.$$.fragment,e),d(Te.$$.fragment,e),Tt=!0)},o(e){J(I.$$.fragment,e),J(U.$$.fragment,e),J(j.$$.fragment,e),J(Z.$$.fragment,e),J(X.$$.fragment,e),J(v.$$.fragment,e),J(V.$$.fragment,e),J(f.$$.fragment,e),J(Y.$$.fragment,e),J(k.$$.fragment,e),J(N.$$.fragment,e),J(x.$$.fragment,e),J(S.$$.fragment,e),J(E.$$.fragment,e),J(D.$$.fragment,e),J(K.$$.fragment,e),J(ee.$$.fragment,e),J(te.$$.fragment,e),J(le.$$.fragment,e),J(ae.$$.fragment,e),J(ce.$$.fragment,e),J(ue.$$.fragment,e),J(Me.$$.fragment,e),J(de.$$.fragment,e),J(Te.$$.fragment,e),Tt=!1},d(e){e&&(s(m),s(y),s(h),s(he),s(je),s(fe),s(b),s(Ie),s(g),s(Ue),s(C),s(be),s(_),s(ge),s($),s(Ce),s(_e),s(B),s($e),s(W),s(Ze),s(Be),s(G),s(We),s(Xe),s(R),s(Ge),s(ve),s(Re),s(z),s(Ve),s(ze),s(A),s(Ye),s(Ae),s(q),s(ke),s(qe),s(H),s(Ne),s(He),s(Q),s(xe),s(Qe),s(F),s(Se),s(Fe),s(L),s(Ee),s(Le),s(P),s(De),s(Pe),s(O),s(Ke),s(Oe),s(et),s(se),s(tt),s(st),s(lt),s(ne),s(at),s(pe),s(nt),s(ie),s(pt),s(it),s(oe),s(ct),s(ot),s(re),s(ut),s(rt),s(Mt),s(Je),s(dt),s(Jt),s(me)),s(o),T(I,e),T(U,e),T(j,e),T(Z,e),T(X,e),T(v,e),T(V,e),T(f,e),T(Y,e),T(k,e),T(N,e),T(x,e),T(S,e),T(E,e),T(D,e),T(K,e),T(ee,e),T(te,e),T(le,e),T(ae,e),T(ce,e),T(ue,e),T(Me,e),T(de,e),T(Te,e)}}}const as='{"title":"Using multiple models with DeepSpeed","local":"using-multiple-models-with-deepspeed","sections":[{"title":"Knowledge distillation","local":"knowledge-distillation","sections":[{"title":"Student model","local":"student-model","sections":[],"depth":3},{"title":"Teacher model","local":"teacher-model","sections":[],"depth":3},{"title":"Training","local":"training","sections":[],"depth":3}],"depth":2},{"title":"Train multiple disjoint models","local":"train-multiple-disjoint-models","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}';function ns(we){return Et(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class rs extends Lt{constructor(o){super(),Dt(this,o,ns,ls,Ft,{})}}export{rs as component};
