import{s as Ms,a as Ts,n as $s,o as ks}from"../chunks/scheduler.9cda857a.js";import{S as vs,i as Js,e as i,s as n,c as u,h as Is,a as p,d as s,b as l,f as ws,g as r,j as o,k as Je,l as Cs,m as a,n as c,t as h,o as m,p as g}from"../chunks/index.003612b0.js";import{C as Ss}from"../chunks/CopyLLMTxtMenu.44d08a39.js";import{C as f}from"../chunks/CodeBlock.a90632f1.js";import{H as d,E as qs}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.e5cfda8c.js";function _s(Lt){let j,Ie,ke,Ce,w,Se,M,qe,T,Wt="Spark enables real-time, large-scale data processing in a distributed environment.",_e,$,Ft="You can use <code>pyspark_huggingface</code> to access Hugging Face datasets repositories in PySpark via the “huggingface” Data Source.",xe,k,Pt='Try out <a href="https://huggingface.co/spaces/Dataset-Tools/Spark-Notebooks" rel="nofollow">Spark Notebooks</a> on Hugging Face Spaces to get Notebooks with PySpark and <code>pyspark_huggingface</code> pre-installed.',Ze,$e,Gt,Ue,v,He,J,Le,I,zt="To be able to read and write to Hugging Face Datasets, you need to install the <code>pyspark_huggingface</code> library:",We,C,Fe,S,Bt="This will also install required dependencies like <code>huggingface_hub</code> for authentication, and <code>pyarrow</code> for reading and writing datasets.",Pe,q,Ge,_,Nt="You need to authenticate to Hugging Face to read private/gated dataset repositories or to write to your dataset repositories.",ze,x,Yt="You can use the CLI for example:",Be,Z,Ne,U,At=`It’s also possible to provide your Hugging Face token with the <code>HF_TOKEN</code> environment variable or passing the <code>token</code> option to the reader.
For more details about authentication, check out <a href="https://huggingface.co/docs/huggingface_hub/quick-start#authentication" rel="nofollow">this guide</a>.`,Ye,H,Ae,L,Qt=`PySpark 4 came with a new Data Source API which allows to use datasets from custom sources.
If <code>pyspark_huggingface</code> is installed, PySpark auto-imports it and enables the “huggingface” Data Source.`,Qe,W,Xt=`The library also backports the Data Source API for the “huggingface” Data Source for PySpark 3.5, 3.4 and 3.3.
However in this case <code>pyspark_huggingface</code> should be imported explicitly to activate the backport and enable the “huggingface” Data Dource:`,Xe,F,Ee,P,De,G,Et=`The “huggingface” Data Source allows to read datasets from Hugging Face, using <code>pyarrow</code> under the hood to stream Arrow data.
This is compatible with all the dataset in <a href="https://huggingface.co/docs/hub/datasets-adding#file-formats" rel="nofollow">supported format</a> on Hugging Face, like Parquet datasets.`,Re,z,Dt='For example here is how to load the <a href="https://huggingface.co/stanfordnlp/imdb" rel="nofollow">stanfordnlp/imdb</a> dataset:',Ve,B,Ke,N,Rt=`Here is another example with the <a href="https://huggingface.co/datasets/BAAI/Infinity-Instruct" rel="nofollow">BAAI/Infinity-Instruct</a> dataset.
It is a gated repository, users have to accept the terms of use before accessing it.
It also has multiple subsets, namely, “3M” and “7M”. So we need to specify which one to load.`,Oe,b,Vt='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-spark-infinity-instruct-7M-min.png"/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-spark-infinity-instruct-7M-dark-min.png"/>',et,Y,Kt="We use the <code>.format()</code> function to use the “huggingface” Data Source, and <code>.load()</code> to load the dataset (more precisely the config or subset named “7M” containing 7M samples). Then we compute the number of dialogue per language and filter the dataset.",tt,A,Ot="After logging-in to access the gated repository, we can run:",st,Q,at,X,es="This loads the dataset in a streaming fashion, and the output DataFrame has one partition per data file in the dataset to enable efficient distributed processing.",nt,E,ts=`To compute the number of dialogues per language we run this code that uses the <code>columns</code> option and a <code>groupBy()</code> operation.
The <code>columns</code> option is useful to only load the data we need, since PySpark doesn’t enable predicate push-down with the Data Source API.
There is also a <code>filters</code> option to only load data with values within a certain range.`,lt,D,it,R,ss="To filter the dataset and only keep dialogues in Chinese:",pt,V,ot,K,as=`It is also possible to apply filters or remove columns on the loaded DataFrame, but it is more efficient to do it while loading, especially on Parquet datasets.
Indeed, Parquet contains metadata at the file and row group level, which allows to skip entire parts of the dataset that don’t contain samples that satisfy the criteria. Columns in Parquet can also be loaded independently, whch allows to skip the excluded columns and avoid loading unnecessary data.`,ut,O,rt,ee,ns="Here is the list of available options you can pass to <code>read..option()</code>:",ct,te,ls="<li><code>config</code> (string): select a dataset subset/config</li> <li><code>split</code> (string): select a dataset split (default is “train”)</li> <li><code>token</code> (string): your Hugging Face token</li>",ht,se,is="For Parquet datasets:",mt,ae,ps='<li><code>columns</code> (string): select a subset of columns to load, e.g. <code>&#39;[&quot;id&quot;]&#39;</code></li> <li><code>filters</code> (string): to skip files and row groups that don’t match a criteria, e.g. <code>&#39;[&quot;source&quot;, &quot;=&quot;, &quot;code_exercises&quot;]&#39;</code>. Filters are passed to <a href="https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetDataset.html" rel="nofollow">pyarrow.parquet.ParquetDataset</a>.</li>',gt,ne,os='Any other option is passed as an argument to [datasets.load_dataset] (<a href="https://huggingface.co/docs/datasets/en/package_reference/loading_methods#datasets.load_dataset" rel="nofollow">https://huggingface.co/docs/datasets/en/package_reference/loading_methods#datasets.load_dataset</a>)',dt,le,ft,ie,us="Once you have your PySpark Dataframe ready, you can run SQL queries using <code>spark.sql</code>:",jt,pe,bt,oe,rs="Again, specifying the <code>columns</code> option is not necessary, but is useful to avoid loading unnecessary data and make the query faster.",yt,ue,wt,re,cs=`You can write a PySpark Dataframe to Hugging Face with the “huggingface” Data Source.
It uploads Parquet files in parallel in a distributed manner, and only commits the files once they’re all uploaded.
It works like this:`,Mt,ce,Tt,he,hs='Here is how we can use this function to write the filtered version of the <a href="https://huggingface.co/datasets/BAAI/Infinity-Instruct" rel="nofollow">BAAI/Infinity-Instruct</a> dataset back to Hugging Face.',$t,me,ms=`First you need to <a href="https://huggingface.co/new-dataset" rel="nofollow">create a dataset repository</a>, e.g. <code>username/Infinity-Instruct-Chinese-Only</code> (you can set it to private if you want).
Then, make sure you are authenticated and you can use the “huggingface” Data Source, set the <code>mode</code> to “overwrite” (or “append” if you want to extend an existing dataset), and push to Hugging Face with <code>.save()</code>:`,kt,ge,vt,y,gs='<img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-spark-infinity-instruct-chinese-only-min.png"/> <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-spark-infinity-instruct-chinese-only-dark-min.png"/>',Jt,de,It,fe,ds="Two modes are available when pushing a dataset to Hugging Face:",Ct,je,fs="<li>“overwrite”: overwrite the dataset if it already exists</li> <li>“append”: append the dataset to an existing dataset</li>",St,be,qt,ye,js="Here is the list of available options you can pass to <code>write.option()</code>:",_t,we,bs="<li><code>token</code> (string): your Hugging Face token</li>",xt,Me,ys="Contributions are welcome to add more options here, in particular <code>subset</code> and <code>split</code>.",Zt,Te,Ut,ve,Ht;return w=new Ss({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),M=new d({props:{title:"Spark",local:"spark",headingTag:"h1"}}),v=new d({props:{title:"Set up",local:"set-up",headingTag:"h2"}}),J=new d({props:{title:"Installation",local:"installation",headingTag:"h3"}}),C=new f({props:{code:"cGlwJTIwaW5zdGFsbCUyMHB5c3BhcmtfaHVnZ2luZ2ZhY2U=",highlighted:'pip <span class="hljs-keyword">install</span> pyspark_huggingface',wrap:!1}}),q=new d({props:{title:"Authentication",local:"authentication",headingTag:"h3"}}),Z=new f({props:{code:"aGYlMjBhdXRoJTIwbG9naW4=",highlighted:'<span class="hljs-attribute">hf auth login</span>',wrap:!1}}),H=new d({props:{title:"Enable the “huggingface” Data Source",local:"enable-the-huggingface-data-source",headingTag:"h3"}}),F=new f({props:{code:"aW1wb3J0JTIwcHlzcGFya19odWdnaW5nZmFjZQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> pyspark_huggingface
huggingface datasource enabled <span class="hljs-keyword">for</span> pyspark <span class="hljs-number">3.</span>x.x (backport <span class="hljs-keyword">from</span> pyspark <span class="hljs-number">4</span>)`,wrap:!1}}),P=new d({props:{title:"Read",local:"read",headingTag:"h2"}}),B=new f({props:{code:"aW1wb3J0JTIwcHlzcGFya19odWdnaW5nZmFjZSUwQWZyb20lMjBweXNwYXJrLnNxbCUyMGltcG9ydCUyMFNwYXJrU2Vzc2lvbiUwQXNwYXJrJTIwJTNEJTIwU3BhcmtTZXNzaW9uLmJ1aWxkZXIuYXBwTmFtZSglMjJkZW1vJTIyKS5nZXRPckNyZWF0ZSgpJTBBZGYlMjAlM0QlMjBzcGFyay5yZWFkLmZvcm1hdCglMjJodWdnaW5nZmFjZSUyMikubG9hZCglMjJzdGFuZm9yZG5scCUyRmltZGIlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> pyspark_huggingface
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> pyspark.sql <span class="hljs-keyword">import</span> SparkSession
<span class="hljs-meta">&gt;&gt;&gt; </span>spark = SparkSession.builder.appName(<span class="hljs-string">&quot;demo&quot;</span>).getOrCreate()
<span class="hljs-meta">&gt;&gt;&gt; </span>df = spark.read.<span class="hljs-built_in">format</span>(<span class="hljs-string">&quot;huggingface&quot;</span>).load(<span class="hljs-string">&quot;stanfordnlp/imdb&quot;</span>)`,wrap:!1}}),Q=new f({props:{code:"aW1wb3J0JTIwcHlzcGFya19odWdnaW5nZmFjZSUwQWZyb20lMjBweXNwYXJrLnNxbCUyMGltcG9ydCUyMFNwYXJrU2Vzc2lvbiUwQXNwYXJrJTIwJTNEJTIwU3BhcmtTZXNzaW9uLmJ1aWxkZXIuYXBwTmFtZSglMjJkZW1vJTIyKS5nZXRPckNyZWF0ZSgpJTBBZGYlMjAlM0QlMjBzcGFyay5yZWFkLmZvcm1hdCglMjJodWdnaW5nZmFjZSUyMikub3B0aW9uKCUyMmNvbmZpZyUyMiUyQyUyMCUyMjdNJTIyKS5sb2FkKCUyMkJBQUklMkZJbmZpbml0eS1JbnN0cnVjdCUyMiklMEFkZi5zaG93KCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> pyspark_huggingface
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> pyspark.sql <span class="hljs-keyword">import</span> SparkSession
<span class="hljs-meta">&gt;&gt;&gt; </span>spark = SparkSession.builder.appName(<span class="hljs-string">&quot;demo&quot;</span>).getOrCreate()
<span class="hljs-meta">&gt;&gt;&gt; </span>df = spark.read.<span class="hljs-built_in">format</span>(<span class="hljs-string">&quot;huggingface&quot;</span>).option(<span class="hljs-string">&quot;config&quot;</span>, <span class="hljs-string">&quot;7M&quot;</span>).load(<span class="hljs-string">&quot;BAAI/Infinity-Instruct&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>df.show()
+---+----------------------------+-----+----------+--------------------+        
| <span class="hljs-built_in">id</span>|               conversations|label|langdetect|              source|
+---+----------------------------+-----+----------+--------------------+
|  <span class="hljs-number">0</span>|        [{human, <span class="hljs-keyword">def</span> <span class="hljs-title function_">exti</span>...|     |        en|      code_exercises|
|  <span class="hljs-number">1</span>|        [{human, See the ...|     |        en|                flan|
|  <span class="hljs-number">2</span>|        [{human, This <span class="hljs-keyword">is</span> ...|     |        en|                flan|
|  <span class="hljs-number">3</span>|        [{human, If you d...|     |        en|                flan|
|  <span class="hljs-number">4</span>|        [{human, In a Uni...|     |        en|                flan|
|  <span class="hljs-number">5</span>|        [{human, Read the...|     |        en|                flan|
|  <span class="hljs-number">6</span>|        [{human, You are ...|     |        en|          code_bagel|
|  <span class="hljs-number">7</span>|        [{human, I want y...|     |        en|          Subjective|
|  <span class="hljs-number">8</span>|        [{human, Given th...|     |        en|                flan|
|  <span class="hljs-number">9</span>|[{human, 因果联系原则是法...|     |     zh-cn|          Subjective|
| <span class="hljs-number">10</span>|        [{human, Provide ...|     |        en|self-oss-instruct...|
| <span class="hljs-number">11</span>|        [{human, The univ...|     |        en|                flan|
| <span class="hljs-number">12</span>|        [{human, Q: I am ...|     |        en|                flan|
| <span class="hljs-number">13</span>|        [{human, What <span class="hljs-keyword">is</span> ...|     |        en|      OpenHermes-<span class="hljs-number">2.5</span>|
| <span class="hljs-number">14</span>|        [{human, In react...|     |        en|                flan|
| <span class="hljs-number">15</span>|        [{human, Write Py...|     |        en|      code_exercises|
| <span class="hljs-number">16</span>|        [{human, Find the...|     |        en|            MetaMath|
| <span class="hljs-number">17</span>|        [{human, Three of...|     |        en|            MetaMath|
| <span class="hljs-number">18</span>|        [{human, Chandra ...|     |        en|            MetaMath|
| <span class="hljs-number">19</span>|[{human, 用经济学知识分析...|     |     zh-cn|          Subjective|
+---+----------------------------+-----+----------+--------------------+`,wrap:!1}}),D=new f({props:{code:"ZGZfbGFuZ2RldGVjdF9vbmx5JTIwJTNEJTIwKCUwQSUyMCUyMCUyMCUyMHNwYXJrLnJlYWQuZm9ybWF0KCUyMmh1Z2dpbmdmYWNlJTIyKSUwQSUyMCUyMCUyMCUyMC5vcHRpb24oJTIyY29uZmlnJTIyJTJDJTIwJTIyN00lMjIpJTBBJTIwJTIwJTIwJTIwLm9wdGlvbiglMjJjb2x1bW5zJTIyJTJDJTIwJyU1QiUyMmxhbmdkZXRlY3QlMjIlNUQnKSUwQSUyMCUyMCUyMCUyMC5sb2FkKCUyMkJBQUklMkZJbmZpbml0eS1JbnN0cnVjdCUyMiklMEEpJTBBZGZfbGFuZ2RldGVjdF9vbmx5Lmdyb3VwQnkoJTIybGFuZ2RldGVjdCUyMikuY291bnQoKS5zaG93KCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>df_langdetect_only = (
<span class="hljs-meta">... </span>    spark.read.<span class="hljs-built_in">format</span>(<span class="hljs-string">&quot;huggingface&quot;</span>)
<span class="hljs-meta">... </span>    .option(<span class="hljs-string">&quot;config&quot;</span>, <span class="hljs-string">&quot;7M&quot;</span>)
<span class="hljs-meta">... </span>    .option(<span class="hljs-string">&quot;columns&quot;</span>, <span class="hljs-string">&#x27;[&quot;langdetect&quot;]&#x27;</span>)
<span class="hljs-meta">... </span>    .load(<span class="hljs-string">&quot;BAAI/Infinity-Instruct&quot;</span>)
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>df_langdetect_only.groupBy(<span class="hljs-string">&quot;langdetect&quot;</span>).count().show()
+----------+-------+                                                            
|langdetect|  count|
+----------+-------+
|        en|<span class="hljs-number">6697793</span>|
|     zh-cn| <span class="hljs-number">751313</span>|
+----------+-------+`,wrap:!1}}),V=new f({props:{code:"ZGZfY2hpbmVzZV9vbmx5JTIwJTNEJTIwKCUwQSUyMCUyMCUyMCUyMHNwYXJrLnJlYWQuZm9ybWF0KCUyMmh1Z2dpbmdmYWNlJTIyKSUwQSUyMCUyMCUyMCUyMC5vcHRpb24oJTIyY29uZmlnJTIyJTJDJTIwJTIyN00lMjIpJTBBJTIwJTIwJTIwJTIwLm9wdGlvbiglMjJmaWx0ZXJzJTIyJTJDJTIwJyU1QiglMjJsYW5nZGV0ZWN0JTIyJTJDJTIwJTIyJTNEJTIyJTJDJTIwJTIyemgtY24lMjIpJTVEJyklMEElMjAlMjAlMjAlMjAubG9hZCglMjJCQUFJJTJGSW5maW5pdHktSW5zdHJ1Y3QlMjIpJTBBKSUwQWRmX2NoaW5lc2Vfb25seS5zaG93KCk=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span>df_chinese_only = (
<span class="hljs-meta">... </span>    spark.read.<span class="hljs-built_in">format</span>(<span class="hljs-string">&quot;huggingface&quot;</span>)
<span class="hljs-meta">... </span>    .option(<span class="hljs-string">&quot;config&quot;</span>, <span class="hljs-string">&quot;7M&quot;</span>)
<span class="hljs-meta">... </span>    .option(<span class="hljs-string">&quot;filters&quot;</span>, <span class="hljs-string">&#x27;[(&quot;langdetect&quot;, &quot;=&quot;, &quot;zh-cn&quot;)]&#x27;</span>)
<span class="hljs-meta">... </span>    .load(<span class="hljs-string">&quot;BAAI/Infinity-Instruct&quot;</span>)
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>df_chinese_only.show()
+---+----------------------------+-----+----------+----------+                  
| <span class="hljs-built_in">id</span>|               conversations|label|langdetect|    source|
+---+----------------------------+-----+----------+----------+
|  <span class="hljs-number">9</span>|[{human, 因果联系原则是法...|     |     zh-cn|Subjective|
| <span class="hljs-number">19</span>|[{human, 用经济学知识分析...|     |     zh-cn|Subjective|
| <span class="hljs-number">38</span>| [{human, 某个考试共有A、...|     |     zh-cn|Subjective|
| <span class="hljs-number">39</span>|[{human, 撰写一篇关于斐波...|     |     zh-cn|Subjective|
| <span class="hljs-number">57</span>|[{human, 总结世界历史上的...|     |     zh-cn|Subjective|
| <span class="hljs-number">61</span>|[{human, 生成一则广告词。...|     |     zh-cn|Subjective|
| <span class="hljs-number">66</span>|[{human, 描述一个有效的团...|     |     zh-cn|Subjective|
| <span class="hljs-number">94</span>|[{human, 如果比利和蒂芙尼...|     |     zh-cn|Subjective|
|<span class="hljs-number">102</span>|[{human, 生成一句英文名言...|     |     zh-cn|Subjective|
|<span class="hljs-number">106</span>|[{human, 写一封感谢信，感...|     |     zh-cn|Subjective|
|<span class="hljs-number">118</span>| [{human, 生成一个故事。}...|     |     zh-cn|Subjective|
|<span class="hljs-number">174</span>|[{human, 高胆固醇水平的后...|     |     zh-cn|Subjective|
|<span class="hljs-number">180</span>|[{human, 基于以下角色信息...|     |     zh-cn|Subjective|
|<span class="hljs-number">192</span>|[{human, 请写一篇文章，概...|     |     zh-cn|Subjective|
|<span class="hljs-number">221</span>|[{human, 以诗歌形式表达对...|     |     zh-cn|Subjective|
|<span class="hljs-number">228</span>|[{human, 根据给定的指令，...|     |     zh-cn|Subjective|
|<span class="hljs-number">236</span>|[{human, 打开一个新的生成...|     |     zh-cn|Subjective|
|<span class="hljs-number">260</span>|[{human, 生成一个有关未来...|     |     zh-cn|Subjective|
|<span class="hljs-number">268</span>|[{human, 如果有一定数量的...|     |     zh-cn|Subjective|
|<span class="hljs-number">273</span>| [{human, 题目：小明有<span class="hljs-number">5</span>个...|     |     zh-cn|Subjective|
+---+----------------------------+-----+----------+----------+`,wrap:!1}}),O=new d({props:{title:"Options",local:"options",headingTag:"h3"}}),le=new d({props:{title:"Run SQL queries",local:"run-sql-queries",headingTag:"h3"}}),pe=new f({props:{code:"aW1wb3J0JTIwcHlzcGFya19odWdnaW5nZmFjZSUwQWZyb20lMjBweXNwYXJrLnNxbCUyMGltcG9ydCUyMFNwYXJrU2Vzc2lvbiUwQXNwYXJrJTIwJTNEJTIwU3BhcmtTZXNzaW9uLmJ1aWxkZXIuYXBwTmFtZSglMjJkZW1vJTIyKS5nZXRPckNyZWF0ZSgpJTBBZGYlMjAlM0QlMjAoJTBBJTIwJTIwJTIwJTIwc3BhcmsucmVhZC5mb3JtYXQoJTIyaHVnZ2luZ2ZhY2UlMjIpJTBBJTIwJTIwJTIwJTIwLm9wdGlvbiglMjJjb25maWclMjIlMkMlMjAlMjI3TSUyMiklMEElMjAlMjAlMjAlMjAub3B0aW9uKCUyMmNvbHVtbnMlMjIlMkMlMjAnJTVCJTIyc291cmNlJTIyJTVEJyklMEElMjAlMjAlMjAlMjAubG9hZCglMjJCQUFJJTJGSW5maW5pdHktSW5zdHJ1Y3QlMjIpJTBBKSUwQXNwYXJrLnNxbCglMjJTRUxFQ1QlMjBzb3VyY2UlMkMlMjBjb3VudCgqKSUyMEFTJTIwdG90YWwlMjBGUk9NJTIwJTdCZGYlN0QlMjBHUk9VUCUyMEJZJTIwc291cmNlJTIwT1JERVIlMjBCWSUyMHRvdGFsJTIwREVTQyUyMiUyQyUyMGRmJTNEZGYpLnNob3coKQ==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> pyspark_huggingface
<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">from</span> pyspark.sql <span class="hljs-keyword">import</span> SparkSession
<span class="hljs-meta">&gt;&gt;&gt; </span>spark = SparkSession.builder.appName(<span class="hljs-string">&quot;demo&quot;</span>).getOrCreate()
<span class="hljs-meta">&gt;&gt;&gt; </span>df = (
<span class="hljs-meta">... </span>    spark.read.<span class="hljs-built_in">format</span>(<span class="hljs-string">&quot;huggingface&quot;</span>)
<span class="hljs-meta">... </span>    .option(<span class="hljs-string">&quot;config&quot;</span>, <span class="hljs-string">&quot;7M&quot;</span>)
<span class="hljs-meta">... </span>    .option(<span class="hljs-string">&quot;columns&quot;</span>, <span class="hljs-string">&#x27;[&quot;source&quot;]&#x27;</span>)
<span class="hljs-meta">... </span>    .load(<span class="hljs-string">&quot;BAAI/Infinity-Instruct&quot;</span>)
<span class="hljs-meta">... </span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>spark.sql(<span class="hljs-string">&quot;SELECT source, count(*) AS total FROM {df} GROUP BY source ORDER BY total DESC&quot;</span>, df=df).show()
+--------------------+-------+
|              source|  total|
+--------------------+-------+
|                flan|<span class="hljs-number">2435840</span>|
|          Subjective|<span class="hljs-number">1342427</span>|
|      OpenHermes-<span class="hljs-number">2.5</span>| <span class="hljs-number">855478</span>|
|            MetaMath| <span class="hljs-number">690138</span>|
|      code_exercises| <span class="hljs-number">590958</span>|
|Orca-math-word-pr...| <span class="hljs-number">398168</span>|
|          code_bagel| <span class="hljs-number">386649</span>|
|        MathInstruct| <span class="hljs-number">329254</span>|
|python-code-datas...|  <span class="hljs-number">88632</span>|
|instructional_cod...|  <span class="hljs-number">82920</span>|
|        CodeFeedback|  <span class="hljs-number">79513</span>|
|self-oss-instruct...|  <span class="hljs-number">50467</span>|
|Evol-Instruct-Cod...|  <span class="hljs-number">43354</span>|
|CodeExercise-Pyth...|  <span class="hljs-number">27159</span>|
|code_instructions...|  <span class="hljs-number">23130</span>|
|  Code-Instruct-700k|  <span class="hljs-number">10860</span>|
|Glaive-code-assis...|   <span class="hljs-number">9281</span>|
|python_code_instr...|   <span class="hljs-number">2581</span>|
|Python-Code-23k-S...|   <span class="hljs-number">2297</span>|
+--------------------+-------+`,wrap:!1}}),ue=new d({props:{title:"Write",local:"write",headingTag:"h2"}}),ce=new f({props:{code:"aW1wb3J0JTIwcHlzcGFya19odWdnaW5nZmFjZSUwQWRmLndyaXRlLmZvcm1hdCglMjJodWdnaW5nZmFjZSUyMikuc2F2ZSglMjJ1c2VybmFtZSUyRmRhdGFzZXRfbmFtZSUyMik=",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> pyspark_huggingface
<span class="hljs-meta">&gt;&gt;&gt; </span>df.write.<span class="hljs-built_in">format</span>(<span class="hljs-string">&quot;huggingface&quot;</span>).save(<span class="hljs-string">&quot;username/dataset_name&quot;</span>)`,wrap:!1}}),ge=new f({props:{code:"ZGZfY2hpbmVzZV9vbmx5LndyaXRlLmZvcm1hdCglMjJodWdnaW5nZmFjZSUyMikubW9kZSglMjJvdmVyd3JpdGUlMjIpLnNhdmUoJTIydXNlcm5hbWUlMkZJbmZpbml0eS1JbnN0cnVjdC1DaGluZXNlLU9ubHklMjIp",highlighted:'<span class="hljs-meta">&gt;&gt;&gt; </span>df_chinese_only.write.<span class="hljs-built_in">format</span>(<span class="hljs-string">&quot;huggingface&quot;</span>).mode(<span class="hljs-string">&quot;overwrite&quot;</span>).save(<span class="hljs-string">&quot;username/Infinity-Instruct-Chinese-Only&quot;</span>)',wrap:!1}}),de=new d({props:{title:"Mode",local:"mode",headingTag:"h3"}}),be=new d({props:{title:"Options",local:"options",headingTag:"h3"}}),Te=new qs({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/datasets-spark.md"}}),{c(){j=i("meta"),Ie=n(),ke=i("p"),Ce=n(),u(w.$$.fragment),Se=n(),u(M.$$.fragment),qe=n(),T=i("p"),T.textContent=Wt,_e=n(),$=i("p"),$.innerHTML=Ft,xe=n(),k=i("p"),k.innerHTML=Pt,Ze=n(),$e=i("img"),Ue=n(),u(v.$$.fragment),He=n(),u(J.$$.fragment),Le=n(),I=i("p"),I.innerHTML=zt,We=n(),u(C.$$.fragment),Fe=n(),S=i("p"),S.innerHTML=Bt,Pe=n(),u(q.$$.fragment),Ge=n(),_=i("p"),_.textContent=Nt,ze=n(),x=i("p"),x.textContent=Yt,Be=n(),u(Z.$$.fragment),Ne=n(),U=i("p"),U.innerHTML=At,Ye=n(),u(H.$$.fragment),Ae=n(),L=i("p"),L.innerHTML=Qt,Qe=n(),W=i("p"),W.innerHTML=Xt,Xe=n(),u(F.$$.fragment),Ee=n(),u(P.$$.fragment),De=n(),G=i("p"),G.innerHTML=Et,Re=n(),z=i("p"),z.innerHTML=Dt,Ve=n(),u(B.$$.fragment),Ke=n(),N=i("p"),N.innerHTML=Rt,Oe=n(),b=i("div"),b.innerHTML=Vt,et=n(),Y=i("p"),Y.innerHTML=Kt,tt=n(),A=i("p"),A.textContent=Ot,st=n(),u(Q.$$.fragment),at=n(),X=i("p"),X.textContent=es,nt=n(),E=i("p"),E.innerHTML=ts,lt=n(),u(D.$$.fragment),it=n(),R=i("p"),R.textContent=ss,pt=n(),u(V.$$.fragment),ot=n(),K=i("p"),K.textContent=as,ut=n(),u(O.$$.fragment),rt=n(),ee=i("p"),ee.innerHTML=ns,ct=n(),te=i("ul"),te.innerHTML=ls,ht=n(),se=i("p"),se.textContent=is,mt=n(),ae=i("ul"),ae.innerHTML=ps,gt=n(),ne=i("p"),ne.innerHTML=os,dt=n(),u(le.$$.fragment),ft=n(),ie=i("p"),ie.innerHTML=us,jt=n(),u(pe.$$.fragment),bt=n(),oe=i("p"),oe.innerHTML=rs,yt=n(),u(ue.$$.fragment),wt=n(),re=i("p"),re.textContent=cs,Mt=n(),u(ce.$$.fragment),Tt=n(),he=i("p"),he.innerHTML=hs,$t=n(),me=i("p"),me.innerHTML=ms,kt=n(),u(ge.$$.fragment),vt=n(),y=i("div"),y.innerHTML=gs,Jt=n(),u(de.$$.fragment),It=n(),fe=i("p"),fe.textContent=ds,Ct=n(),je=i("ul"),je.innerHTML=fs,St=n(),u(be.$$.fragment),qt=n(),ye=i("p"),ye.innerHTML=js,_t=n(),we=i("ul"),we.innerHTML=bs,xt=n(),Me=i("p"),Me.innerHTML=ys,Zt=n(),u(Te.$$.fragment),Ut=n(),ve=i("p"),this.h()},l(e){const t=Is("svelte-u9bgzb",document.head);j=p(t,"META",{name:!0,content:!0}),t.forEach(s),Ie=l(e),ke=p(e,"P",{}),ws(ke).forEach(s),Ce=l(e),r(w.$$.fragment,e),Se=l(e),r(M.$$.fragment,e),qe=l(e),T=p(e,"P",{"data-svelte-h":!0}),o(T)!=="svelte-s0r53o"&&(T.textContent=Wt),_e=l(e),$=p(e,"P",{"data-svelte-h":!0}),o($)!=="svelte-ireapg"&&($.innerHTML=Ft),xe=l(e),k=p(e,"P",{"data-svelte-h":!0}),o(k)!=="svelte-17lvzmk"&&(k.innerHTML=Pt),Ze=l(e),$e=p(e,"IMG",{src:!0}),Ue=l(e),r(v.$$.fragment,e),He=l(e),r(J.$$.fragment,e),Le=l(e),I=p(e,"P",{"data-svelte-h":!0}),o(I)!=="svelte-1i6u7b2"&&(I.innerHTML=zt),We=l(e),r(C.$$.fragment,e),Fe=l(e),S=p(e,"P",{"data-svelte-h":!0}),o(S)!=="svelte-1dyqnq0"&&(S.innerHTML=Bt),Pe=l(e),r(q.$$.fragment,e),Ge=l(e),_=p(e,"P",{"data-svelte-h":!0}),o(_)!=="svelte-aqw7kt"&&(_.textContent=Nt),ze=l(e),x=p(e,"P",{"data-svelte-h":!0}),o(x)!=="svelte-pdivv8"&&(x.textContent=Yt),Be=l(e),r(Z.$$.fragment,e),Ne=l(e),U=p(e,"P",{"data-svelte-h":!0}),o(U)!=="svelte-dcl8ev"&&(U.innerHTML=At),Ye=l(e),r(H.$$.fragment,e),Ae=l(e),L=p(e,"P",{"data-svelte-h":!0}),o(L)!=="svelte-8zj7ot"&&(L.innerHTML=Qt),Qe=l(e),W=p(e,"P",{"data-svelte-h":!0}),o(W)!=="svelte-5pp20"&&(W.innerHTML=Xt),Xe=l(e),r(F.$$.fragment,e),Ee=l(e),r(P.$$.fragment,e),De=l(e),G=p(e,"P",{"data-svelte-h":!0}),o(G)!=="svelte-1k66fhy"&&(G.innerHTML=Et),Re=l(e),z=p(e,"P",{"data-svelte-h":!0}),o(z)!=="svelte-yczgy2"&&(z.innerHTML=Dt),Ve=l(e),r(B.$$.fragment,e),Ke=l(e),N=p(e,"P",{"data-svelte-h":!0}),o(N)!=="svelte-1q9wgqv"&&(N.innerHTML=Rt),Oe=l(e),b=p(e,"DIV",{class:!0,"data-svelte-h":!0}),o(b)!=="svelte-1ct11n9"&&(b.innerHTML=Vt),et=l(e),Y=p(e,"P",{"data-svelte-h":!0}),o(Y)!=="svelte-phbdhi"&&(Y.innerHTML=Kt),tt=l(e),A=p(e,"P",{"data-svelte-h":!0}),o(A)!=="svelte-19d7o0j"&&(A.textContent=Ot),st=l(e),r(Q.$$.fragment,e),at=l(e),X=p(e,"P",{"data-svelte-h":!0}),o(X)!=="svelte-rdc7zs"&&(X.textContent=es),nt=l(e),E=p(e,"P",{"data-svelte-h":!0}),o(E)!=="svelte-1du45l"&&(E.innerHTML=ts),lt=l(e),r(D.$$.fragment,e),it=l(e),R=p(e,"P",{"data-svelte-h":!0}),o(R)!=="svelte-e2gcj5"&&(R.textContent=ss),pt=l(e),r(V.$$.fragment,e),ot=l(e),K=p(e,"P",{"data-svelte-h":!0}),o(K)!=="svelte-omqngt"&&(K.textContent=as),ut=l(e),r(O.$$.fragment,e),rt=l(e),ee=p(e,"P",{"data-svelte-h":!0}),o(ee)!=="svelte-1s464mf"&&(ee.innerHTML=ns),ct=l(e),te=p(e,"UL",{"data-svelte-h":!0}),o(te)!=="svelte-1tfwhkx"&&(te.innerHTML=ls),ht=l(e),se=p(e,"P",{"data-svelte-h":!0}),o(se)!=="svelte-luhsa2"&&(se.textContent=is),mt=l(e),ae=p(e,"UL",{"data-svelte-h":!0}),o(ae)!=="svelte-85k25g"&&(ae.innerHTML=ps),gt=l(e),ne=p(e,"P",{"data-svelte-h":!0}),o(ne)!=="svelte-1reyf4q"&&(ne.innerHTML=os),dt=l(e),r(le.$$.fragment,e),ft=l(e),ie=p(e,"P",{"data-svelte-h":!0}),o(ie)!=="svelte-13i529n"&&(ie.innerHTML=us),jt=l(e),r(pe.$$.fragment,e),bt=l(e),oe=p(e,"P",{"data-svelte-h":!0}),o(oe)!=="svelte-fpkb7l"&&(oe.innerHTML=rs),yt=l(e),r(ue.$$.fragment,e),wt=l(e),re=p(e,"P",{"data-svelte-h":!0}),o(re)!=="svelte-140g2zh"&&(re.textContent=cs),Mt=l(e),r(ce.$$.fragment,e),Tt=l(e),he=p(e,"P",{"data-svelte-h":!0}),o(he)!=="svelte-1d35p3o"&&(he.innerHTML=hs),$t=l(e),me=p(e,"P",{"data-svelte-h":!0}),o(me)!=="svelte-vthxjd"&&(me.innerHTML=ms),kt=l(e),r(ge.$$.fragment,e),vt=l(e),y=p(e,"DIV",{class:!0,"data-svelte-h":!0}),o(y)!=="svelte-tmflad"&&(y.innerHTML=gs),Jt=l(e),r(de.$$.fragment,e),It=l(e),fe=p(e,"P",{"data-svelte-h":!0}),o(fe)!=="svelte-zuvewl"&&(fe.textContent=ds),Ct=l(e),je=p(e,"UL",{"data-svelte-h":!0}),o(je)!=="svelte-e9isj"&&(je.innerHTML=fs),St=l(e),r(be.$$.fragment,e),qt=l(e),ye=p(e,"P",{"data-svelte-h":!0}),o(ye)!=="svelte-1acq8iy"&&(ye.innerHTML=js),_t=l(e),we=p(e,"UL",{"data-svelte-h":!0}),o(we)!=="svelte-sthc9l"&&(we.innerHTML=bs),xt=l(e),Me=p(e,"P",{"data-svelte-h":!0}),o(Me)!=="svelte-cukjdh"&&(Me.innerHTML=ys),Zt=l(e),r(Te.$$.fragment,e),Ut=l(e),ve=p(e,"P",{}),ws(ve).forEach(s),this.h()},h(){Je(j,"name","hf:doc:metadata"),Je(j,"content",xs),Ts($e.src,Gt="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/spark-notebooks-min.png")||Je($e,"src",Gt),Je(b,"class","flex justify-center"),Je(y,"class","flex justify-center")},m(e,t){Cs(document.head,j),a(e,Ie,t),a(e,ke,t),a(e,Ce,t),c(w,e,t),a(e,Se,t),c(M,e,t),a(e,qe,t),a(e,T,t),a(e,_e,t),a(e,$,t),a(e,xe,t),a(e,k,t),a(e,Ze,t),a(e,$e,t),a(e,Ue,t),c(v,e,t),a(e,He,t),c(J,e,t),a(e,Le,t),a(e,I,t),a(e,We,t),c(C,e,t),a(e,Fe,t),a(e,S,t),a(e,Pe,t),c(q,e,t),a(e,Ge,t),a(e,_,t),a(e,ze,t),a(e,x,t),a(e,Be,t),c(Z,e,t),a(e,Ne,t),a(e,U,t),a(e,Ye,t),c(H,e,t),a(e,Ae,t),a(e,L,t),a(e,Qe,t),a(e,W,t),a(e,Xe,t),c(F,e,t),a(e,Ee,t),c(P,e,t),a(e,De,t),a(e,G,t),a(e,Re,t),a(e,z,t),a(e,Ve,t),c(B,e,t),a(e,Ke,t),a(e,N,t),a(e,Oe,t),a(e,b,t),a(e,et,t),a(e,Y,t),a(e,tt,t),a(e,A,t),a(e,st,t),c(Q,e,t),a(e,at,t),a(e,X,t),a(e,nt,t),a(e,E,t),a(e,lt,t),c(D,e,t),a(e,it,t),a(e,R,t),a(e,pt,t),c(V,e,t),a(e,ot,t),a(e,K,t),a(e,ut,t),c(O,e,t),a(e,rt,t),a(e,ee,t),a(e,ct,t),a(e,te,t),a(e,ht,t),a(e,se,t),a(e,mt,t),a(e,ae,t),a(e,gt,t),a(e,ne,t),a(e,dt,t),c(le,e,t),a(e,ft,t),a(e,ie,t),a(e,jt,t),c(pe,e,t),a(e,bt,t),a(e,oe,t),a(e,yt,t),c(ue,e,t),a(e,wt,t),a(e,re,t),a(e,Mt,t),c(ce,e,t),a(e,Tt,t),a(e,he,t),a(e,$t,t),a(e,me,t),a(e,kt,t),c(ge,e,t),a(e,vt,t),a(e,y,t),a(e,Jt,t),c(de,e,t),a(e,It,t),a(e,fe,t),a(e,Ct,t),a(e,je,t),a(e,St,t),c(be,e,t),a(e,qt,t),a(e,ye,t),a(e,_t,t),a(e,we,t),a(e,xt,t),a(e,Me,t),a(e,Zt,t),c(Te,e,t),a(e,Ut,t),a(e,ve,t),Ht=!0},p:$s,i(e){Ht||(h(w.$$.fragment,e),h(M.$$.fragment,e),h(v.$$.fragment,e),h(J.$$.fragment,e),h(C.$$.fragment,e),h(q.$$.fragment,e),h(Z.$$.fragment,e),h(H.$$.fragment,e),h(F.$$.fragment,e),h(P.$$.fragment,e),h(B.$$.fragment,e),h(Q.$$.fragment,e),h(D.$$.fragment,e),h(V.$$.fragment,e),h(O.$$.fragment,e),h(le.$$.fragment,e),h(pe.$$.fragment,e),h(ue.$$.fragment,e),h(ce.$$.fragment,e),h(ge.$$.fragment,e),h(de.$$.fragment,e),h(be.$$.fragment,e),h(Te.$$.fragment,e),Ht=!0)},o(e){m(w.$$.fragment,e),m(M.$$.fragment,e),m(v.$$.fragment,e),m(J.$$.fragment,e),m(C.$$.fragment,e),m(q.$$.fragment,e),m(Z.$$.fragment,e),m(H.$$.fragment,e),m(F.$$.fragment,e),m(P.$$.fragment,e),m(B.$$.fragment,e),m(Q.$$.fragment,e),m(D.$$.fragment,e),m(V.$$.fragment,e),m(O.$$.fragment,e),m(le.$$.fragment,e),m(pe.$$.fragment,e),m(ue.$$.fragment,e),m(ce.$$.fragment,e),m(ge.$$.fragment,e),m(de.$$.fragment,e),m(be.$$.fragment,e),m(Te.$$.fragment,e),Ht=!1},d(e){e&&(s(Ie),s(ke),s(Ce),s(Se),s(qe),s(T),s(_e),s($),s(xe),s(k),s(Ze),s($e),s(Ue),s(He),s(Le),s(I),s(We),s(Fe),s(S),s(Pe),s(Ge),s(_),s(ze),s(x),s(Be),s(Ne),s(U),s(Ye),s(Ae),s(L),s(Qe),s(W),s(Xe),s(Ee),s(De),s(G),s(Re),s(z),s(Ve),s(Ke),s(N),s(Oe),s(b),s(et),s(Y),s(tt),s(A),s(st),s(at),s(X),s(nt),s(E),s(lt),s(it),s(R),s(pt),s(ot),s(K),s(ut),s(rt),s(ee),s(ct),s(te),s(ht),s(se),s(mt),s(ae),s(gt),s(ne),s(dt),s(ft),s(ie),s(jt),s(bt),s(oe),s(yt),s(wt),s(re),s(Mt),s(Tt),s(he),s($t),s(me),s(kt),s(vt),s(y),s(Jt),s(It),s(fe),s(Ct),s(je),s(St),s(qt),s(ye),s(_t),s(we),s(xt),s(Me),s(Zt),s(Ut),s(ve)),s(j),g(w,e),g(M,e),g(v,e),g(J,e),g(C,e),g(q,e),g(Z,e),g(H,e),g(F,e),g(P,e),g(B,e),g(Q,e),g(D,e),g(V,e),g(O,e),g(le,e),g(pe,e),g(ue,e),g(ce,e),g(ge,e),g(de,e),g(be,e),g(Te,e)}}}const xs='{"title":"Spark","local":"spark","sections":[{"title":"Set up","local":"set-up","sections":[{"title":"Installation","local":"installation","sections":[],"depth":3},{"title":"Authentication","local":"authentication","sections":[],"depth":3},{"title":"Enable the “huggingface” Data Source","local":"enable-the-huggingface-data-source","sections":[],"depth":3}],"depth":2},{"title":"Read","local":"read","sections":[{"title":"Options","local":"options","sections":[],"depth":3},{"title":"Run SQL queries","local":"run-sql-queries","sections":[],"depth":3}],"depth":2},{"title":"Write","local":"write","sections":[{"title":"Mode","local":"mode","sections":[],"depth":3},{"title":"Options","local":"options","sections":[],"depth":3}],"depth":2}],"depth":1}';function Zs(Lt){return ks(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Ps extends vs{constructor(j){super(),Js(this,j,Zs,_s,Ms,{})}}export{Ps as component};
