import{s as N,n as L,o as q}from"../chunks/scheduler.d9d004e7.js";import{S as P,i as D,e as h,s as n,c as b,h as K,a as f,d as t,b as s,f as H,g as T,j as B,k as x,l as O,m as l,n as U,t as Z,o as g,p as W}from"../chunks/index.4a2b774c.js";import{C as aa,H as ea,E as ta}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.02c268bd.js";import{C as A}from"../chunks/CodeBlock.67ad8ed0.js";function la(v){let i,j,w,G,r,k,o,X,m,$=`Pretraining a model from Transformers, like BERT, is as easy as fine-tuning it.
The model should be instantiated from a configuration with <code>.from_config</code> and not from a pretrained checkpoint with <code>.from_pretrained</code>.
Here is how it should look with GPT2 for instance:`,I,p,R,c,z="with XXX the task to perform, such as <code>ImageClassification</code> for example.",_,M,S="The following is a working example where BERT is pretrained for masked language modeling:",C,d,Q,u,E='You can see another example of pretraining in <a href="https://huggingface.co/blog/pretraining-bert" rel="nofollow">this blog post</a>.',V,y,F,J,Y;return r=new aa({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),o=new ea({props:{title:"Pretraining Transformers with Optimum for Intel Gaudi",local:"pretraining-transformers-with-optimum-for-intel-gaudi",headingTag:"h1"}}),p=new A({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMEF1dG9Db25maWclMkMlMjBBdXRvTW9kZWxGb3JYWFglMEElMEFjb25maWclMjAlM0QlMjBBdXRvQ29uZmlnLmZyb21fcHJldHJhaW5lZCglMjJncHQyJTIyKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yWFhYLmZyb21fY29uZmlnKGNvbmZpZyk=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoConfig, AutoModelForXXX

config = AutoConfig.from_pretrained(<span class="hljs-string">&quot;gpt2&quot;</span>)
model = AutoModelForXXX.from_config(config)`,wrap:!1}}),d=new A({props:{code:"ZnJvbSUyMGRhdGFzZXRzJTIwaW1wb3J0JTIwbG9hZF9kYXRhc2V0JTBBZnJvbSUyMG9wdGltdW0uaGFiYW5hJTIwaW1wb3J0JTIwR2F1ZGlUcmFpbmVyJTJDJTIwR2F1ZGlUcmFpbmluZ0FyZ3VtZW50cyUwQWZyb20lMjB0cmFuc2Zvcm1lcnMlMjBpbXBvcnQlMjBBdXRvQ29uZmlnJTJDJTIwQXV0b01vZGVsRm9yTWFza2VkTE0lMkMlMjBBdXRvVG9rZW5pemVyJTJDJTIwRGF0YUNvbGxhdG9yRm9yTGFuZ3VhZ2VNb2RlbGluZyUwQSUwQSUyMyUyMExvYWQlMjB0aGUlMjB0cmFpbmluZyUyMHNldCUyMCh0aGlzJTIwb25lJTIwaGFzJTIwYWxyZWFkeSUyMGJlZW4lMjBwcmVwcm9jZXNzZWQpJTBBdHJhaW5pbmdfc2V0JTIwJTNEJTIwbG9hZF9kYXRhc2V0KCUyMnBoaWxzY2htaWQlMkZwcm9jZXNzZWRfYmVydF9kYXRhc2V0JTIyJTJDJTIwc3BsaXQlM0QlMjJ0cmFpbiUyMiklMEElMjMlMjBMb2FkJTIwdGhlJTIwdG9rZW5pemVyJTBBdG9rZW5pemVyJTIwJTNEJTIwQXV0b1Rva2VuaXplci5mcm9tX3ByZXRyYWluZWQoJTIycGhpbHNjaG1pZCUyRmJlcnQtYmFzZS11bmNhc2VkLTIwMjItaGFiYW5hJTIyKSUwQSUwQSUyMyUyMEluc3RhbnRpYXRlJTIwYW4lMjB1bnRyYWluZWQlMjBtb2RlbCUwQWNvbmZpZyUyMCUzRCUyMEF1dG9Db25maWcuZnJvbV9wcmV0cmFpbmVkKCUyMmJlcnQtYmFzZS11bmNhc2VkJTIyKSUwQW1vZGVsJTIwJTNEJTIwQXV0b01vZGVsRm9yTWFza2VkTE0uZnJvbV9jb25maWcoY29uZmlnKSUwQSUwQW1vZGVsLnJlc2l6ZV90b2tlbl9lbWJlZGRpbmdzKGxlbih0b2tlbml6ZXIpKSUwQSUwQSUyMyUyMFRoZSUyMGRhdGElMjBjb2xsYXRvciUyMHdpbGwlMjB0YWtlJTIwY2FyZSUyMG9mJTIwcmFuZG9tbHklMjBtYXNraW5nJTIwdGhlJTIwdG9rZW5zJTBBZGF0YV9jb2xsYXRvciUyMCUzRCUyMERhdGFDb2xsYXRvckZvckxhbmd1YWdlTW9kZWxpbmcodG9rZW5pemVyJTNEdG9rZW5pemVyKSUwQSUwQXRyYWluaW5nX2FyZ3MlMjAlM0QlMjBHYXVkaVRyYWluaW5nQXJndW1lbnRzKCUwQSUyMCUyMCUyMCUyMG91dHB1dF9kaXIlM0QlMjIlMkZ0bXAlMkZiZXJ0LWJhc2UtdW5jYXNlZC1tbG0lMjIlMkMlMEElMjAlMjAlMjAlMjBudW1fdHJhaW5fZXBvY2hzJTNEMSUyQyUwQSUyMCUyMCUyMCUyMHBlcl9kZXZpY2VfdHJhaW5fYmF0Y2hfc2l6ZSUzRDglMkMlMEElMjAlMjAlMjAlMjB1c2VfaGFiYW5hJTNEVHJ1ZSUyQyUwQSUyMCUyMCUyMCUyMHVzZV9sYXp5X21vZGUlM0RUcnVlJTJDJTBBJTIwJTIwJTIwJTIwZ2F1ZGlfY29uZmlnX25hbWUlM0QlMjJIYWJhbmElMkZiZXJ0LWJhc2UtdW5jYXNlZCUyMiUyQyUwQSklMEElMEElMjMlMjBJbml0aWFsaXplJTIwb3VyJTIwVHJhaW5lciUwQXRyYWluZXIlMjAlM0QlMjBHYXVkaVRyYWluZXIoJTBBJTIwJTIwJTIwJTIwbW9kZWwlM0Rtb2RlbCUyQyUwQSUyMCUyMCUyMCUyMGFyZ3MlM0R0cmFpbmluZ19hcmdzJTJDJTBBJTIwJTIwJTIwJTIwdHJhaW5fZGF0YXNldCUzRHRyYWluaW5nX3NldCUyQyUwQSUyMCUyMCUyMCUyMHRva2VuaXplciUzRHRva2VuaXplciUyQyUwQSUyMCUyMCUyMCUyMGRhdGFfY29sbGF0b3IlM0RkYXRhX2NvbGxhdG9yJTJDJTBBKSUwQSUwQXRyYWluZXIudHJhaW4oKQ==",highlighted:`<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset
<span class="hljs-keyword">from</span> optimum.habana <span class="hljs-keyword">import</span> GaudiTrainer, GaudiTrainingArguments
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> AutoConfig, AutoModelForMaskedLM, AutoTokenizer, DataCollatorForLanguageModeling

<span class="hljs-comment"># Load the training set (this one has already been preprocessed)</span>
training_set = load_dataset(<span class="hljs-string">&quot;philschmid/processed_bert_dataset&quot;</span>, split=<span class="hljs-string">&quot;train&quot;</span>)
<span class="hljs-comment"># Load the tokenizer</span>
tokenizer = AutoTokenizer.from_pretrained(<span class="hljs-string">&quot;philschmid/bert-base-uncased-2022-habana&quot;</span>)

<span class="hljs-comment"># Instantiate an untrained model</span>
config = AutoConfig.from_pretrained(<span class="hljs-string">&quot;bert-base-uncased&quot;</span>)
model = AutoModelForMaskedLM.from_config(config)

model.resize_token_embeddings(<span class="hljs-built_in">len</span>(tokenizer))

<span class="hljs-comment"># The data collator will take care of randomly masking the tokens</span>
data_collator = DataCollatorForLanguageModeling(tokenizer=tokenizer)

training_args = GaudiTrainingArguments(
    output_dir=<span class="hljs-string">&quot;/tmp/bert-base-uncased-mlm&quot;</span>,
    num_train_epochs=<span class="hljs-number">1</span>,
    per_device_train_batch_size=<span class="hljs-number">8</span>,
    use_habana=<span class="hljs-literal">True</span>,
    use_lazy_mode=<span class="hljs-literal">True</span>,
    gaudi_config_name=<span class="hljs-string">&quot;Habana/bert-base-uncased&quot;</span>,
)

<span class="hljs-comment"># Initialize our Trainer</span>
trainer = GaudiTrainer(
    model=model,
    args=training_args,
    train_dataset=training_set,
    tokenizer=tokenizer,
    data_collator=data_collator,
)

trainer.train()`,wrap:!1}}),y=new ta({props:{source:"https://github.com/huggingface/optimum-habana/blob/main/docs/source/usage_guides/pretraining.mdx"}}),{c(){i=h("meta"),j=n(),w=h("p"),G=n(),b(r.$$.fragment),k=n(),b(o.$$.fragment),X=n(),m=h("p"),m.innerHTML=$,I=n(),b(p.$$.fragment),R=n(),c=h("p"),c.innerHTML=z,_=n(),M=h("p"),M.textContent=S,C=n(),b(d.$$.fragment),Q=n(),u=h("p"),u.innerHTML=E,V=n(),b(y.$$.fragment),F=n(),J=h("p"),this.h()},l(a){const e=K("svelte-u9bgzb",document.head);i=f(e,"META",{name:!0,content:!0}),e.forEach(t),j=s(a),w=f(a,"P",{}),H(w).forEach(t),G=s(a),T(r.$$.fragment,a),k=s(a),T(o.$$.fragment,a),X=s(a),m=f(a,"P",{"data-svelte-h":!0}),B(m)!=="svelte-gvcwga"&&(m.innerHTML=$),I=s(a),T(p.$$.fragment,a),R=s(a),c=f(a,"P",{"data-svelte-h":!0}),B(c)!=="svelte-17lz52k"&&(c.innerHTML=z),_=s(a),M=f(a,"P",{"data-svelte-h":!0}),B(M)!=="svelte-1t539x1"&&(M.textContent=S),C=s(a),T(d.$$.fragment,a),Q=s(a),u=f(a,"P",{"data-svelte-h":!0}),B(u)!=="svelte-57cz4v"&&(u.innerHTML=E),V=s(a),T(y.$$.fragment,a),F=s(a),J=f(a,"P",{}),H(J).forEach(t),this.h()},h(){x(i,"name","hf:doc:metadata"),x(i,"content",na)},m(a,e){O(document.head,i),l(a,j,e),l(a,w,e),l(a,G,e),U(r,a,e),l(a,k,e),U(o,a,e),l(a,X,e),l(a,m,e),l(a,I,e),U(p,a,e),l(a,R,e),l(a,c,e),l(a,_,e),l(a,M,e),l(a,C,e),U(d,a,e),l(a,Q,e),l(a,u,e),l(a,V,e),U(y,a,e),l(a,F,e),l(a,J,e),Y=!0},p:L,i(a){Y||(Z(r.$$.fragment,a),Z(o.$$.fragment,a),Z(p.$$.fragment,a),Z(d.$$.fragment,a),Z(y.$$.fragment,a),Y=!0)},o(a){g(r.$$.fragment,a),g(o.$$.fragment,a),g(p.$$.fragment,a),g(d.$$.fragment,a),g(y.$$.fragment,a),Y=!1},d(a){a&&(t(j),t(w),t(G),t(k),t(X),t(m),t(I),t(R),t(c),t(_),t(M),t(C),t(Q),t(u),t(V),t(F),t(J)),t(i),W(r,a),W(o,a),W(p,a),W(d,a),W(y,a)}}}const na='{"title":"Pretraining Transformers with Optimum for Intel Gaudi","local":"pretraining-transformers-with-optimum-for-intel-gaudi","sections":[],"depth":1}';function sa(v){return q(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class pa extends P{constructor(i){super(),D(this,i,sa,la,N,{})}}export{pa as component};
