import{s as Re,n as Fe,o as xe}from"../chunks/scheduler.9cda857a.js";import{S as Qe,i as Ye,e as h,s as l,c as p,h as We,a as y,d as a,b as n,f as Xe,g as i,j as c,k as ye,l as Ne,m as t,n as o,t as r,o as d,p as m}from"../chunks/index.003612b0.js";import{C as Se}from"../chunks/CopyLLMTxtMenu.44d08a39.js";import{C as u}from"../chunks/CodeBlock.a90632f1.js";import{H as f,E as Ve}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.e5cfda8c.js";function He(ce){let M,us,ys,fs,U,Ms,g,ws,J,ue=`<a href="https://github.com/pandas-dev/pandas" rel="nofollow">Pandas</a> is a widely used Python data analysis toolkit.
Since it uses <a href="https://filesystem-spec.readthedocs.io" rel="nofollow">fsspec</a> to read and write remote data, you can use the Hugging Face paths (<a href="/docs/huggingface_hub/guides/hf_file_system#integrations"><code>hf://</code></a>) to read and write data on the Hub.`,Us,T,gs,j,fe="You can load data from local files or from remote storage like Hugging Face Datasets. Pandas supports many formats including CSV, JSON and Parquet:",Js,$,Ts,b,Me='To load a file from Hugging Face, the path needs to start with <code>hf://</code>. For example, the path to the <a href="https://huggingface.co/datasets/stanfordnlp/imdb" rel="nofollow">stanfordnlp/imdb</a> dataset repository is <code>hf://datasets/stanfordnlp/imdb</code>. The dataset on Hugging Face contains multiple Parquet files. The Parquet file format is designed to make reading and writing data frames efficient, and to make sharing data across data analysis languages easy. Here is how to load the file <code>plain_text/train-00000-of-00001.parquet</code>:',js,I,$s,C,we='For more information on the Hugging Face paths and how they are implemented, please refer to the <a href="/docs/huggingface_hub/guides/hf_file_system">the client library’s documentation on the HfFileSystem</a>.',bs,Z,Is,G,Ue="You can save a pandas DataFrame using <code>to_csv/to_json/to_parquet</code> to a local file or to Hugging Face directly.",Cs,k,ge='To save the DataFrame on Hugging Face, you first need to <a href="/docs/huggingface_hub/quick-start#login">Login with your Hugging Face account</a>, for example using:',Zs,q,Gs,B,Je='Then you can <a href="/docs/huggingface_hub/quick-start#create-a-repository">Create a dataset repository</a>, for example using:',ks,v,qs,_,Te='Finally, you can use <a href="/docs/huggingface_hub/guides/hf_file_system#integrations">Hugging Face paths</a> in Pandas:',Bs,X,vs,R,_s,F,je="You can load a folder with a metadata file containing a field for the names or paths to the images, structured like this:",Xs,x,Rs,Q,$e="You can iterate on the images paths like this:",Fs,Y,xs,W,be='Since the dataset is in a <a href="https://huggingface.co/docs/hub/en/datasets-image#additional-columns" rel="nofollow">supported structure</a> (a <code>metadata.csv</code> or <code>.jsonl</code> file with a <code>file_name</code> field), you can save this dataset to Hugging Face and the Dataset Viewer shows both the metadata and images on Hugging Face.',Qs,N,Ys,S,Ws,V,Ie='Using <a href="https://github.com/lhoestq/pandas-image-methods" rel="nofollow">pandas-image-methods</a> you enable <code>PIL.Image</code> methods on an image column. It also enables saving the dataset as one single Parquet file containing both the images and the metadata:',Ns,H,Ss,E,Ce="All the <code>PIL.Image</code> methods are available, e.g.",Vs,L,Hs,z,Es,A,Ze="You can load a folder with a metadata file containing a field for the names or paths to the audios, structured like this:",Ls,P,zs,K,Ge="You can iterate on the audios paths like this:",As,D,Ps,O,ke='Since the dataset is in a <a href="https://huggingface.co/docs/hub/en/datasets-audio#additional-columns" rel="nofollow">supported structure</a> (a <code>metadata.csv</code> or <code>.jsonl</code> file with a <code>file_name</code> field), you can save it to Hugging Face, and the Hub Dataset Viewer shows both the metadata and audio.',Ks,ss,Ds,es,Os,as,qe='Using <a href="https://github.com/lhoestq/pandas-audio-methods" rel="nofollow">pandas-audio-methods</a> you enable <code>soundfile</code> methods on an audio column. It also enables saving the dataset as one single Parquet file containing both the audios and the metadata:',se,ts,ee,ls,Be="This makes it easy to use with <code>librosa</code> e.g. for resampling:",ae,ns,te,ps,le,is,ve=`You can use <code>transformers</code> pipelines on pandas DataFrames to classify, generate text, images, etc.
This section shows a few examples with <code>tqdm</code> for progress bars.`,ne,w,_e="<p>Pipelines don’t accept a <code>tqdm</code> object as input but you can use a python generator instead, in the form <code>x for x in tqdm(...)</code></p>",pe,os,ie,rs,oe,ds,re,ms,de,hs,me,cs,he;return U=new Se({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),g=new f({props:{title:"Pandas",local:"pandas",headingTag:"h1"}}),T=new f({props:{title:"Load a DataFrame",local:"load-a-dataframe",headingTag:"h2"}}),$=new u({props:{code:"aW1wb3J0JTIwcGFuZGFzJTIwYXMlMjBwZCUwQWRmJTIwJTNEJTIwcGQucmVhZF9jc3YoJTIycGF0aCUyRnRvJTJGZGF0YS5jc3YlMjIp",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd
<span class="hljs-meta">&gt;&gt;&gt; </span>df = pd.read_csv(<span class="hljs-string">&quot;path/to/data.csv&quot;</span>)`,wrap:!1}}),I=new u({props:{code:"aW1wb3J0JTIwcGFuZGFzJTIwYXMlMjBwZCUwQWRmJTIwJTNEJTIwcGQucmVhZF9wYXJxdWV0KCUyMmhmJTNBJTJGJTJGZGF0YXNldHMlMkZzdGFuZm9yZG5scCUyRmltZGIlMkZwbGFpbl90ZXh0JTJGdHJhaW4tMDAwMDAtb2YtMDAwMDEucGFycXVldCUyMiklMEFkZiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMC4uLiUyMCUyMCUyMCUyMC4uLg==",highlighted:`<span class="hljs-meta">&gt;&gt;&gt; </span><span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd
<span class="hljs-meta">&gt;&gt;&gt; </span>df = pd.read_parquet(<span class="hljs-string">&quot;hf://datasets/stanfordnlp/imdb/plain_text/train-00000-of-00001.parquet&quot;</span>)
<span class="hljs-meta">&gt;&gt;&gt; </span>df
                                                    text  label
<span class="hljs-number">0</span>      I rented I AM CURIOUS-YELLOW <span class="hljs-keyword">from</span> my video sto...      <span class="hljs-number">0</span>
<span class="hljs-number">1</span>      <span class="hljs-string">&quot;I Am Curious: Yellow&quot;</span> <span class="hljs-keyword">is</span> a risible <span class="hljs-keyword">and</span> preten...      <span class="hljs-number">0</span>
<span class="hljs-number">2</span>      If only to avoid making this <span class="hljs-built_in">type</span> of film <span class="hljs-keyword">in</span> t...      <span class="hljs-number">0</span>
<span class="hljs-number">3</span>      This film was probably inspired by Godard<span class="hljs-string">&#x27;s Ma...      0
4      Oh, brother...after hearing about this ridicul...      0
...                                                  ...    ...
24995  A hit at the time but now better categorised a...      1
24996  I love this movie like no other. Another time ...      1
24997  This film and it&#x27;</span>s sequel Barry Mckenzie holds...      <span class="hljs-number">1</span>
<span class="hljs-number">24998</span>  <span class="hljs-string">&#x27;The Adventures Of Barry McKenzie&#x27;</span> started lif...      <span class="hljs-number">1</span>
<span class="hljs-number">24999</span>  The story centers around Barry McKenzie who mu...      <span class="hljs-number">1</span>`,wrap:!1}}),Z=new f({props:{title:"Save a DataFrame",local:"save-a-dataframe",headingTag:"h2"}}),q=new u({props:{code:"aGYlMjBhdXRoJTIwbG9naW4=",highlighted:'<span class="hljs-attribute">hf auth login</span>',wrap:!1}}),v=new u({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMEhmQXBpJTBBJTBBSGZBcGkoKS5jcmVhdGVfcmVwbyhyZXBvX2lkJTNEJTIydXNlcm5hbWUlMkZteV9kYXRhc2V0JTIyJTJDJTIwcmVwb190eXBlJTNEJTIyZGF0YXNldCUyMik=",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfApi

HfApi().create_repo(repo_id=<span class="hljs-string">&quot;username/my_dataset&quot;</span>, repo_type=<span class="hljs-string">&quot;dataset&quot;</span>)`,wrap:!1}}),X=new u({props:{code:"aW1wb3J0JTIwcGFuZGFzJTIwYXMlMjBwZCUwQSUwQWRmLnRvX3BhcnF1ZXQoJTIyaGYlM0ElMkYlMkZkYXRhc2V0cyUyRnVzZXJuYW1lJTJGbXlfZGF0YXNldCUyRmltZGIucGFycXVldCUyMiklMEElMEElMjMlMjBvciUyMHdyaXRlJTIwaW4lMjBzZXBhcmF0ZSUyMGZpbGVzJTIwaWYlMjB0aGUlMjBkYXRhc2V0JTIwaGFzJTIwdHJhaW4lMkZ2YWxpZGF0aW9uJTJGdGVzdCUyMHNwbGl0cyUwQWRmX3RyYWluLnRvX3BhcnF1ZXQoJTIyaGYlM0ElMkYlMkZkYXRhc2V0cyUyRnVzZXJuYW1lJTJGbXlfZGF0YXNldCUyRnRyYWluLnBhcnF1ZXQlMjIpJTBBZGZfdmFsaWQudG9fcGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZteV9kYXRhc2V0JTJGdmFsaWRhdGlvbi5wYXJxdWV0JTIyKSUwQWRmX3Rlc3QlMjAudG9fcGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZteV9kYXRhc2V0JTJGdGVzdC5wYXJxdWV0JTIyKQ==",highlighted:`<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd

df.to_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/imdb.parquet&quot;</span>)

<span class="hljs-comment"># or write in separate files if the dataset has train/validation/test splits</span>
df_train.to_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/train.parquet&quot;</span>)
df_valid.to_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/validation.parquet&quot;</span>)
df_test .to_parquet(<span class="hljs-string">&quot;hf://datasets/username/my_dataset/test.parquet&quot;</span>)`,wrap:!1}}),R=new f({props:{title:"Use Images",local:"use-images",headingTag:"h2"}}),x=new u({props:{code:"JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwLi4u",highlighted:`Example <span class="hljs-number">1</span>:            Example <span class="hljs-number">2</span>:
folder/               folder/
├── metadata<span class="hljs-selector-class">.csv</span>      ├── metadata<span class="hljs-selector-class">.csv</span>
├── img000<span class="hljs-selector-class">.png</span>        └── images
├── img001<span class="hljs-selector-class">.png</span>            ├── img000<span class="hljs-selector-class">.png</span>
...                       ...
└── imgNNN<span class="hljs-selector-class">.png</span>            └── imgNNN.png`,wrap:!1}}),Y=new u({props:{code:"aW1wb3J0JTIwcGFuZGFzJTIwYXMlMjBwZCUwQSUwQWZvbGRlcl9wYXRoJTIwJTNEJTIwJTIycGF0aCUyRnRvJTJGZm9sZGVyJTJGJTIyJTBBZGYlMjAlM0QlMjBwZC5yZWFkX2Nzdihmb2xkZXJfcGF0aCUyMCUyQiUyMCUyMm1ldGFkYXRhLmNzdiUyMiklMEFmb3IlMjBpbWFnZV9wYXRoJTIwaW4lMjAoZm9sZGVyX3BhdGglMjAlMkIlMjBkZiU1QiUyMmZpbGVfbmFtZSUyMiU1RCklM0ElMEElMjAlMjAlMjAlMjAuLi4=",highlighted:`<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd

folder_path = <span class="hljs-string">&quot;path/to/folder/&quot;</span>
df = pd.read_csv(folder_path + <span class="hljs-string">&quot;metadata.csv&quot;</span>)
<span class="hljs-keyword">for</span> image_path <span class="hljs-keyword">in</span> (folder_path + df[<span class="hljs-string">&quot;file_name&quot;</span>]):
    ...`,wrap:!1}}),N=new u({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMEhmQXBpJTBBYXBpJTIwJTNEJTIwSGZBcGkoKSUwQSUwQWFwaS51cGxvYWRfZm9sZGVyKCUwQSUyMCUyMCUyMCUyMGZvbGRlcl9wYXRoJTNEZm9sZGVyX3BhdGglMkMlMEElMjAlMjAlMjAlMjByZXBvX2lkJTNEJTIydXNlcm5hbWUlMkZteV9pbWFnZV9kYXRhc2V0JTIyJTJDJTBBJTIwJTIwJTIwJTIwcmVwb190eXBlJTNEJTIyZGF0YXNldCUyMiUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfApi
api = HfApi()

api.upload_folder(
    folder_path=folder_path,
    repo_id=<span class="hljs-string">&quot;username/my_image_dataset&quot;</span>,
    repo_type=<span class="hljs-string">&quot;dataset&quot;</span>,
)`,wrap:!1}}),S=new f({props:{title:"Image methods and Parquet",local:"image-methods-and-parquet",headingTag:"h3"}}),H=new u({props:{code:"aW1wb3J0JTIwcGFuZGFzJTIwYXMlMjBwZCUwQWZyb20lMjBwYW5kYXNfaW1hZ2VfbWV0aG9kcyUyMGltcG9ydCUyMFBJTE1ldGhvZHMlMEElMEFwZC5hcGkuZXh0ZW5zaW9ucy5yZWdpc3Rlcl9zZXJpZXNfYWNjZXNzb3IoJTIycGlsJTIyKShQSUxNZXRob2RzKSUwQSUwQWRmJTVCJTIyaW1hZ2UlMjIlNUQlMjAlM0QlMjAoZm9sZGVyX3BhdGglMjAlMkIlMjBkZiU1QiUyMmZpbGVfbmFtZSUyMiU1RCkucGlsLm9wZW4oKSUwQWRmLnRvX3BhcnF1ZXQoJTIyZGF0YS5wYXJxdWV0JTIyKQ==",highlighted:`<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd
<span class="hljs-keyword">from</span> pandas_image_methods <span class="hljs-keyword">import</span> PILMethods

pd.api.extensions.register_series_accessor(<span class="hljs-string">&quot;pil&quot;</span>)(PILMethods)

df[<span class="hljs-string">&quot;image&quot;</span>] = (folder_path + df[<span class="hljs-string">&quot;file_name&quot;</span>]).pil.<span class="hljs-built_in">open</span>()
df.to_parquet(<span class="hljs-string">&quot;data.parquet&quot;</span>)`,wrap:!1}}),L=new u({props:{code:"ZGYlNUIlMjJpbWFnZSUyMiU1RCUyMCUzRCUyMGRmJTVCJTIyaW1hZ2UlMjIlNUQucGlsLnJvdGF0ZSg5MCk=",highlighted:'df[<span class="hljs-string">&quot;image&quot;</span>] = df[<span class="hljs-string">&quot;image&quot;</span>].pil.rotate(<span class="hljs-number">90</span>)',wrap:!1}}),z=new f({props:{title:"Use Audios",local:"use-audios",headingTag:"h2"}}),P=new u({props:{code:"JTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwLi4u",highlighted:`Example <span class="hljs-number">1</span>:            Example <span class="hljs-number">2</span>:
folder/               folder/
├── metadata<span class="hljs-selector-class">.csv</span>      ├── metadata<span class="hljs-selector-class">.csv</span>
├── rec000<span class="hljs-selector-class">.wav</span>        └── audios
├── rec001<span class="hljs-selector-class">.wav</span>            ├── rec000<span class="hljs-selector-class">.wav</span>
...                       ...
└── recNNN<span class="hljs-selector-class">.wav</span>            └── recNNN.wav`,wrap:!1}}),D=new u({props:{code:"aW1wb3J0JTIwcGFuZGFzJTIwYXMlMjBwZCUwQSUwQWZvbGRlcl9wYXRoJTIwJTNEJTIwJTIycGF0aCUyRnRvJTJGZm9sZGVyJTJGJTIyJTBBZGYlMjAlM0QlMjBwZC5yZWFkX2Nzdihmb2xkZXJfcGF0aCUyMCUyQiUyMCUyMm1ldGFkYXRhLmNzdiUyMiklMEFmb3IlMjBhdWRpb19wYXRoJTIwaW4lMjAoZm9sZGVyX3BhdGglMjAlMkIlMjBkZiU1QiUyMmZpbGVfbmFtZSUyMiU1RCklM0ElMEElMjAlMjAlMjAlMjAuLi4=",highlighted:`<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd

folder_path = <span class="hljs-string">&quot;path/to/folder/&quot;</span>
df = pd.read_csv(folder_path + <span class="hljs-string">&quot;metadata.csv&quot;</span>)
<span class="hljs-keyword">for</span> audio_path <span class="hljs-keyword">in</span> (folder_path + df[<span class="hljs-string">&quot;file_name&quot;</span>]):
    ...`,wrap:!1}}),ss=new u({props:{code:"ZnJvbSUyMGh1Z2dpbmdmYWNlX2h1YiUyMGltcG9ydCUyMEhmQXBpJTBBYXBpJTIwJTNEJTIwSGZBcGkoKSUwQSUwQWFwaS51cGxvYWRfZm9sZGVyKCUwQSUyMCUyMCUyMCUyMGZvbGRlcl9wYXRoJTNEZm9sZGVyX3BhdGglMkMlMEElMjAlMjAlMjAlMjByZXBvX2lkJTNEJTIydXNlcm5hbWUlMkZteV9hdWRpb19kYXRhc2V0JTIyJTJDJTBBJTIwJTIwJTIwJTIwcmVwb190eXBlJTNEJTIyZGF0YXNldCUyMiUyQyUwQSk=",highlighted:`<span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> HfApi
api = HfApi()

api.upload_folder(
    folder_path=folder_path,
    repo_id=<span class="hljs-string">&quot;username/my_audio_dataset&quot;</span>,
    repo_type=<span class="hljs-string">&quot;dataset&quot;</span>,
)`,wrap:!1}}),es=new f({props:{title:"Audio methods and Parquet",local:"audio-methods-and-parquet",headingTag:"h3"}}),ts=new u({props:{code:"aW1wb3J0JTIwcGFuZGFzJTIwYXMlMjBwZCUwQWZyb20lMjBwYW5kYXNfYXVkaW9fbWV0aG9kcyUyMGltcG9ydCUyMFNGTWV0aG9kcyUwQSUwQXBkLmFwaS5leHRlbnNpb25zLnJlZ2lzdGVyX3Nlcmllc19hY2Nlc3NvciglMjJzZiUyMikoU0ZNZXRob2RzKSUwQSUwQWRmJTVCJTIyYXVkaW8lMjIlNUQlMjAlM0QlMjAoZm9sZGVyX3BhdGglMjAlMkIlMjBkZiU1QiUyMmZpbGVfbmFtZSUyMiU1RCkuc2Yub3BlbigpJTBBZGYudG9fcGFycXVldCglMjJkYXRhLnBhcnF1ZXQlMjIp",highlighted:`<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd
<span class="hljs-keyword">from</span> pandas_audio_methods <span class="hljs-keyword">import</span> SFMethods

pd.api.extensions.register_series_accessor(<span class="hljs-string">&quot;sf&quot;</span>)(SFMethods)

df[<span class="hljs-string">&quot;audio&quot;</span>] = (folder_path + df[<span class="hljs-string">&quot;file_name&quot;</span>]).sf.<span class="hljs-built_in">open</span>()
df.to_parquet(<span class="hljs-string">&quot;data.parquet&quot;</span>)`,wrap:!1}}),ns=new u({props:{code:"ZGYlNUIlMjJhdWRpbyUyMiU1RCUyMCUzRCUyMCU1QmxpYnJvc2EubG9hZChhdWRpbyUyQyUyMHNyJTNEMTZfMDAwKSUyMGZvciUyMGF1ZGlvJTIwaW4lMjBkZiU1QiUyMmF1ZGlvJTIyJTVEJTVEJTBBZGYlNUIlMjJhdWRpbyUyMiU1RCUyMCUzRCUyMGRmJTVCJTIyYXVkaW8lMjIlNUQuc2Yud3JpdGUoKQ==",highlighted:`df[<span class="hljs-string">&quot;audio&quot;</span>] = [librosa.load(audio, sr=<span class="hljs-number">16_000</span>) <span class="hljs-keyword">for</span> audio <span class="hljs-keyword">in</span> df[<span class="hljs-string">&quot;audio&quot;</span>]]
df[<span class="hljs-string">&quot;audio&quot;</span>] = df[<span class="hljs-string">&quot;audio&quot;</span>].sf.write()`,wrap:!1}}),ps=new f({props:{title:"Use Transformers",local:"use-transformers",headingTag:"h2"}}),os=new f({props:{title:"Text Classification",local:"text-classification",headingTag:"h3"}}),rs=new u({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBZnJvbSUyMHRxZG0lMjBpbXBvcnQlMjB0cWRtJTBBJTBBcGlwZSUyMCUzRCUyMHBpcGVsaW5lKCUyMnRleHQtY2xhc3NpZmljYXRpb24lMjIlMkMlMjBtb2RlbCUzRCUyMmNsYXBBSSUyRm1vZGVybkJFUlQtYmFzZS1tdWx0aWxpbmd1YWwtc2VudGltZW50JTIyKSUwQSUwQSUyMyUyMENvbXB1dGUlMjBsYWJlbHMlMEFkZiU1QiUyMmxhYmVsJTIyJTVEJTIwJTNEJTIwJTVCeSU1QiUyMmxhYmVsJTIyJTVEJTIwZm9yJTIweSUyMGluJTIwcGlwZSh4JTIwZm9yJTIweCUyMGluJTIwdHFkbShkZiU1QiUyMnRleHQlMjIlNUQpKSU1RCUwQSUyMyUyMENvbXB1dGUlMjBsYWJlbHMlMjBhbmQlMjBzY29yZXMlMEFkZiU1QiU1QiUyMmxhYmVsJTIyJTJDJTIwJTIyc2NvcmUlMjIlNUQlNUQlMjAlM0QlMjAlNUIoeSU1QiUyMmxhYmVsJTIyJTVEJTJDJTIweSU1QiUyMnNjb3JlJTIyJTVEKSUyMGZvciUyMHklMjBpbiUyMHBpcGUoeCUyMGZvciUyMHglMjBpbiUyMHRxZG0oZGYlNUIlMjJ0ZXh0JTIyJTVEKSklNUQ=",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline
<span class="hljs-keyword">from</span> tqdm <span class="hljs-keyword">import</span> tqdm

pipe = pipeline(<span class="hljs-string">&quot;text-classification&quot;</span>, model=<span class="hljs-string">&quot;clapAI/modernBERT-base-multilingual-sentiment&quot;</span>)

<span class="hljs-comment"># Compute labels</span>
df[<span class="hljs-string">&quot;label&quot;</span>] = [y[<span class="hljs-string">&quot;label&quot;</span>] <span class="hljs-keyword">for</span> y <span class="hljs-keyword">in</span> pipe(x <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> tqdm(df[<span class="hljs-string">&quot;text&quot;</span>]))]
<span class="hljs-comment"># Compute labels and scores</span>
df[[<span class="hljs-string">&quot;label&quot;</span>, <span class="hljs-string">&quot;score&quot;</span>]] = [(y[<span class="hljs-string">&quot;label&quot;</span>], y[<span class="hljs-string">&quot;score&quot;</span>]) <span class="hljs-keyword">for</span> y <span class="hljs-keyword">in</span> pipe(x <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> tqdm(df[<span class="hljs-string">&quot;text&quot;</span>]))]`,wrap:!1}}),ds=new f({props:{title:"Text Generation",local:"text-generation",headingTag:"h3"}}),ms=new u({props:{code:"ZnJvbSUyMHRyYW5zZm9ybWVycyUyMGltcG9ydCUyMHBpcGVsaW5lJTBBZnJvbSUyMHRxZG0lMjBpbXBvcnQlMjB0cWRtJTBBJTBBcGlwZSUyMCUzRCUyMHBpcGVsaW5lKCUyMnRleHQtZ2VuZXJhdGlvbiUyMiUyQyUyMG1vZGVsJTNEJTIyUXdlbiUyRlF3ZW4yLjUtMS41Qi1JbnN0cnVjdCUyMiklMEElMEElMjMlMjBHZW5lcmF0ZSUyMGNoYXQlMjByZXNwb25zZSUwQXByb21wdCUyMCUzRCUyMCUyMldoYXQlMjBpcyUyMHRoZSUyMG1haW4lMjB0b3BpYyUyMG9mJTIwdGhpcyUyMHNlbnRlbmNlJTIwJTNGJTIwUkVQTFklMjBJTiUyMExFU1MlMjBUSEFOJTIwMyUyMFdPUkRTLiUyMFNlbnRlbmNlJTNBJTIwJyU3QiU3RCclMjIlMEFkZiU1QiUyMm91dHB1dCUyMiU1RCUyMCUzRCUyMCU1QnklNUIlMjJnZW5lcmF0ZWRfdGV4dCUyMiU1RCU1QjElNUQlNUIlMjJjb250ZW50JTIyJTVEJTIwZm9yJTIweSUyMGluJTIwcGlwZSglNUIlN0IlMjJyb2xlJTIyJTNBJTIwJTIydXNlciUyMiUyQyUyMCUyMmNvbnRlbnQlMjIlM0ElMjBwcm9tcHQuZm9ybWF0KHgpJTdEJTVEJTIwZm9yJTIweCUyMGluJTIwdHFkbShkZiU1QiUyMnRleHQlMjIlNUQpKSU1RA==",highlighted:`<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> pipeline
<span class="hljs-keyword">from</span> tqdm <span class="hljs-keyword">import</span> tqdm

pipe = pipeline(<span class="hljs-string">&quot;text-generation&quot;</span>, model=<span class="hljs-string">&quot;Qwen/Qwen2.5-1.5B-Instruct&quot;</span>)

<span class="hljs-comment"># Generate chat response</span>
prompt = <span class="hljs-string">&quot;What is the main topic of this sentence ? REPLY IN LESS THAN 3 WORDS. Sentence: &#x27;{}&#x27;&quot;</span>
df[<span class="hljs-string">&quot;output&quot;</span>] = [y[<span class="hljs-string">&quot;generated_text&quot;</span>][<span class="hljs-number">1</span>][<span class="hljs-string">&quot;content&quot;</span>] <span class="hljs-keyword">for</span> y <span class="hljs-keyword">in</span> pipe([{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: prompt.<span class="hljs-built_in">format</span>(x)}] <span class="hljs-keyword">for</span> x <span class="hljs-keyword">in</span> tqdm(df[<span class="hljs-string">&quot;text&quot;</span>]))]`,wrap:!1}}),hs=new Ve({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/datasets-pandas.md"}}),{c(){M=h("meta"),us=l(),ys=h("p"),fs=l(),p(U.$$.fragment),Ms=l(),p(g.$$.fragment),ws=l(),J=h("p"),J.innerHTML=ue,Us=l(),p(T.$$.fragment),gs=l(),j=h("p"),j.textContent=fe,Js=l(),p($.$$.fragment),Ts=l(),b=h("p"),b.innerHTML=Me,js=l(),p(I.$$.fragment),$s=l(),C=h("p"),C.innerHTML=we,bs=l(),p(Z.$$.fragment),Is=l(),G=h("p"),G.innerHTML=Ue,Cs=l(),k=h("p"),k.innerHTML=ge,Zs=l(),p(q.$$.fragment),Gs=l(),B=h("p"),B.innerHTML=Je,ks=l(),p(v.$$.fragment),qs=l(),_=h("p"),_.innerHTML=Te,Bs=l(),p(X.$$.fragment),vs=l(),p(R.$$.fragment),_s=l(),F=h("p"),F.textContent=je,Xs=l(),p(x.$$.fragment),Rs=l(),Q=h("p"),Q.textContent=$e,Fs=l(),p(Y.$$.fragment),xs=l(),W=h("p"),W.innerHTML=be,Qs=l(),p(N.$$.fragment),Ys=l(),p(S.$$.fragment),Ws=l(),V=h("p"),V.innerHTML=Ie,Ns=l(),p(H.$$.fragment),Ss=l(),E=h("p"),E.innerHTML=Ce,Vs=l(),p(L.$$.fragment),Hs=l(),p(z.$$.fragment),Es=l(),A=h("p"),A.textContent=Ze,Ls=l(),p(P.$$.fragment),zs=l(),K=h("p"),K.textContent=Ge,As=l(),p(D.$$.fragment),Ps=l(),O=h("p"),O.innerHTML=ke,Ks=l(),p(ss.$$.fragment),Ds=l(),p(es.$$.fragment),Os=l(),as=h("p"),as.innerHTML=qe,se=l(),p(ts.$$.fragment),ee=l(),ls=h("p"),ls.innerHTML=Be,ae=l(),p(ns.$$.fragment),te=l(),p(ps.$$.fragment),le=l(),is=h("p"),is.innerHTML=ve,ne=l(),w=h("blockquote"),w.innerHTML=_e,pe=l(),p(os.$$.fragment),ie=l(),p(rs.$$.fragment),oe=l(),p(ds.$$.fragment),re=l(),p(ms.$$.fragment),de=l(),p(hs.$$.fragment),me=l(),cs=h("p"),this.h()},l(s){const e=We("svelte-u9bgzb",document.head);M=y(e,"META",{name:!0,content:!0}),e.forEach(a),us=n(s),ys=y(s,"P",{}),Xe(ys).forEach(a),fs=n(s),i(U.$$.fragment,s),Ms=n(s),i(g.$$.fragment,s),ws=n(s),J=y(s,"P",{"data-svelte-h":!0}),c(J)!=="svelte-1d65wub"&&(J.innerHTML=ue),Us=n(s),i(T.$$.fragment,s),gs=n(s),j=y(s,"P",{"data-svelte-h":!0}),c(j)!=="svelte-1110j0n"&&(j.textContent=fe),Js=n(s),i($.$$.fragment,s),Ts=n(s),b=y(s,"P",{"data-svelte-h":!0}),c(b)!=="svelte-yuajzf"&&(b.innerHTML=Me),js=n(s),i(I.$$.fragment,s),$s=n(s),C=y(s,"P",{"data-svelte-h":!0}),c(C)!=="svelte-smuvdf"&&(C.innerHTML=we),bs=n(s),i(Z.$$.fragment,s),Is=n(s),G=y(s,"P",{"data-svelte-h":!0}),c(G)!=="svelte-1ec8wgz"&&(G.innerHTML=Ue),Cs=n(s),k=y(s,"P",{"data-svelte-h":!0}),c(k)!=="svelte-17ov0yg"&&(k.innerHTML=ge),Zs=n(s),i(q.$$.fragment,s),Gs=n(s),B=y(s,"P",{"data-svelte-h":!0}),c(B)!=="svelte-wfb25d"&&(B.innerHTML=Je),ks=n(s),i(v.$$.fragment,s),qs=n(s),_=y(s,"P",{"data-svelte-h":!0}),c(_)!=="svelte-1j7afyv"&&(_.innerHTML=Te),Bs=n(s),i(X.$$.fragment,s),vs=n(s),i(R.$$.fragment,s),_s=n(s),F=y(s,"P",{"data-svelte-h":!0}),c(F)!=="svelte-i1x932"&&(F.textContent=je),Xs=n(s),i(x.$$.fragment,s),Rs=n(s),Q=y(s,"P",{"data-svelte-h":!0}),c(Q)!=="svelte-1wo91tc"&&(Q.textContent=$e),Fs=n(s),i(Y.$$.fragment,s),xs=n(s),W=y(s,"P",{"data-svelte-h":!0}),c(W)!=="svelte-1claf2y"&&(W.innerHTML=be),Qs=n(s),i(N.$$.fragment,s),Ys=n(s),i(S.$$.fragment,s),Ws=n(s),V=y(s,"P",{"data-svelte-h":!0}),c(V)!=="svelte-u4wtbg"&&(V.innerHTML=Ie),Ns=n(s),i(H.$$.fragment,s),Ss=n(s),E=y(s,"P",{"data-svelte-h":!0}),c(E)!=="svelte-1alvzik"&&(E.innerHTML=Ce),Vs=n(s),i(L.$$.fragment,s),Hs=n(s),i(z.$$.fragment,s),Es=n(s),A=y(s,"P",{"data-svelte-h":!0}),c(A)!=="svelte-14ssghx"&&(A.textContent=Ze),Ls=n(s),i(P.$$.fragment,s),zs=n(s),K=y(s,"P",{"data-svelte-h":!0}),c(K)!=="svelte-9uthgh"&&(K.textContent=Ge),As=n(s),i(D.$$.fragment,s),Ps=n(s),O=y(s,"P",{"data-svelte-h":!0}),c(O)!=="svelte-mbpxd8"&&(O.innerHTML=ke),Ks=n(s),i(ss.$$.fragment,s),Ds=n(s),i(es.$$.fragment,s),Os=n(s),as=y(s,"P",{"data-svelte-h":!0}),c(as)!=="svelte-1b7187v"&&(as.innerHTML=qe),se=n(s),i(ts.$$.fragment,s),ee=n(s),ls=y(s,"P",{"data-svelte-h":!0}),c(ls)!=="svelte-h7f8qe"&&(ls.innerHTML=Be),ae=n(s),i(ns.$$.fragment,s),te=n(s),i(ps.$$.fragment,s),le=n(s),is=y(s,"P",{"data-svelte-h":!0}),c(is)!=="svelte-1l7xk3n"&&(is.innerHTML=ve),ne=n(s),w=y(s,"BLOCKQUOTE",{class:!0,"data-svelte-h":!0}),c(w)!=="svelte-1q19yys"&&(w.innerHTML=_e),pe=n(s),i(os.$$.fragment,s),ie=n(s),i(rs.$$.fragment,s),oe=n(s),i(ds.$$.fragment,s),re=n(s),i(ms.$$.fragment,s),de=n(s),i(hs.$$.fragment,s),me=n(s),cs=y(s,"P",{}),Xe(cs).forEach(a),this.h()},h(){ye(M,"name","hf:doc:metadata"),ye(M,"content",Ee),ye(w,"class","tip")},m(s,e){Ne(document.head,M),t(s,us,e),t(s,ys,e),t(s,fs,e),o(U,s,e),t(s,Ms,e),o(g,s,e),t(s,ws,e),t(s,J,e),t(s,Us,e),o(T,s,e),t(s,gs,e),t(s,j,e),t(s,Js,e),o($,s,e),t(s,Ts,e),t(s,b,e),t(s,js,e),o(I,s,e),t(s,$s,e),t(s,C,e),t(s,bs,e),o(Z,s,e),t(s,Is,e),t(s,G,e),t(s,Cs,e),t(s,k,e),t(s,Zs,e),o(q,s,e),t(s,Gs,e),t(s,B,e),t(s,ks,e),o(v,s,e),t(s,qs,e),t(s,_,e),t(s,Bs,e),o(X,s,e),t(s,vs,e),o(R,s,e),t(s,_s,e),t(s,F,e),t(s,Xs,e),o(x,s,e),t(s,Rs,e),t(s,Q,e),t(s,Fs,e),o(Y,s,e),t(s,xs,e),t(s,W,e),t(s,Qs,e),o(N,s,e),t(s,Ys,e),o(S,s,e),t(s,Ws,e),t(s,V,e),t(s,Ns,e),o(H,s,e),t(s,Ss,e),t(s,E,e),t(s,Vs,e),o(L,s,e),t(s,Hs,e),o(z,s,e),t(s,Es,e),t(s,A,e),t(s,Ls,e),o(P,s,e),t(s,zs,e),t(s,K,e),t(s,As,e),o(D,s,e),t(s,Ps,e),t(s,O,e),t(s,Ks,e),o(ss,s,e),t(s,Ds,e),o(es,s,e),t(s,Os,e),t(s,as,e),t(s,se,e),o(ts,s,e),t(s,ee,e),t(s,ls,e),t(s,ae,e),o(ns,s,e),t(s,te,e),o(ps,s,e),t(s,le,e),t(s,is,e),t(s,ne,e),t(s,w,e),t(s,pe,e),o(os,s,e),t(s,ie,e),o(rs,s,e),t(s,oe,e),o(ds,s,e),t(s,re,e),o(ms,s,e),t(s,de,e),o(hs,s,e),t(s,me,e),t(s,cs,e),he=!0},p:Fe,i(s){he||(r(U.$$.fragment,s),r(g.$$.fragment,s),r(T.$$.fragment,s),r($.$$.fragment,s),r(I.$$.fragment,s),r(Z.$$.fragment,s),r(q.$$.fragment,s),r(v.$$.fragment,s),r(X.$$.fragment,s),r(R.$$.fragment,s),r(x.$$.fragment,s),r(Y.$$.fragment,s),r(N.$$.fragment,s),r(S.$$.fragment,s),r(H.$$.fragment,s),r(L.$$.fragment,s),r(z.$$.fragment,s),r(P.$$.fragment,s),r(D.$$.fragment,s),r(ss.$$.fragment,s),r(es.$$.fragment,s),r(ts.$$.fragment,s),r(ns.$$.fragment,s),r(ps.$$.fragment,s),r(os.$$.fragment,s),r(rs.$$.fragment,s),r(ds.$$.fragment,s),r(ms.$$.fragment,s),r(hs.$$.fragment,s),he=!0)},o(s){d(U.$$.fragment,s),d(g.$$.fragment,s),d(T.$$.fragment,s),d($.$$.fragment,s),d(I.$$.fragment,s),d(Z.$$.fragment,s),d(q.$$.fragment,s),d(v.$$.fragment,s),d(X.$$.fragment,s),d(R.$$.fragment,s),d(x.$$.fragment,s),d(Y.$$.fragment,s),d(N.$$.fragment,s),d(S.$$.fragment,s),d(H.$$.fragment,s),d(L.$$.fragment,s),d(z.$$.fragment,s),d(P.$$.fragment,s),d(D.$$.fragment,s),d(ss.$$.fragment,s),d(es.$$.fragment,s),d(ts.$$.fragment,s),d(ns.$$.fragment,s),d(ps.$$.fragment,s),d(os.$$.fragment,s),d(rs.$$.fragment,s),d(ds.$$.fragment,s),d(ms.$$.fragment,s),d(hs.$$.fragment,s),he=!1},d(s){s&&(a(us),a(ys),a(fs),a(Ms),a(ws),a(J),a(Us),a(gs),a(j),a(Js),a(Ts),a(b),a(js),a($s),a(C),a(bs),a(Is),a(G),a(Cs),a(k),a(Zs),a(Gs),a(B),a(ks),a(qs),a(_),a(Bs),a(vs),a(_s),a(F),a(Xs),a(Rs),a(Q),a(Fs),a(xs),a(W),a(Qs),a(Ys),a(Ws),a(V),a(Ns),a(Ss),a(E),a(Vs),a(Hs),a(Es),a(A),a(Ls),a(zs),a(K),a(As),a(Ps),a(O),a(Ks),a(Ds),a(Os),a(as),a(se),a(ee),a(ls),a(ae),a(te),a(le),a(is),a(ne),a(w),a(pe),a(ie),a(oe),a(re),a(de),a(me),a(cs)),a(M),m(U,s),m(g,s),m(T,s),m($,s),m(I,s),m(Z,s),m(q,s),m(v,s),m(X,s),m(R,s),m(x,s),m(Y,s),m(N,s),m(S,s),m(H,s),m(L,s),m(z,s),m(P,s),m(D,s),m(ss,s),m(es,s),m(ts,s),m(ns,s),m(ps,s),m(os,s),m(rs,s),m(ds,s),m(ms,s),m(hs,s)}}}const Ee='{"title":"Pandas","local":"pandas","sections":[{"title":"Load a DataFrame","local":"load-a-dataframe","sections":[],"depth":2},{"title":"Save a DataFrame","local":"save-a-dataframe","sections":[],"depth":2},{"title":"Use Images","local":"use-images","sections":[{"title":"Image methods and Parquet","local":"image-methods-and-parquet","sections":[],"depth":3}],"depth":2},{"title":"Use Audios","local":"use-audios","sections":[{"title":"Audio methods and Parquet","local":"audio-methods-and-parquet","sections":[],"depth":3}],"depth":2},{"title":"Use Transformers","local":"use-transformers","sections":[{"title":"Text Classification","local":"text-classification","sections":[],"depth":3},{"title":"Text Generation","local":"text-generation","sections":[],"depth":3}],"depth":2}],"depth":1}';function Le(ce){return xe(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Oe extends Qe{constructor(M){super(),Ye(this,M,Le,He,Re,{})}}export{Oe as component};
