import{s as Et,n as Xt,o as kt}from"../chunks/scheduler.9cda857a.js";import{S as xt,i as zt,e as o,s as a,c as i,h as St,a as c,d as l,b as n,f as Ft,g as M,j as y,k as wt,l as Qt,m as s,n as J,t as p,o as m,p as T}from"../chunks/index.003612b0.js";import{C as Vt}from"../chunks/CopyLLMTxtMenu.44d08a39.js";import{C as r}from"../chunks/CodeBlock.a90632f1.js";import{H as w,E as _t}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.e5cfda8c.js";function Ht(rt){let f,re,ye,fe,I,ue,U,Ie,d,ft='<a href="https://github.com/typedef-ai/fenic" rel="nofollow">fenic</a> is a PySpark-inspired DataFrame framework designed for building production AI and agentic applications. fenic provides support for reading datasets directly from the Hugging Face Hub.',Ue,u,ut='<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/fenic_hf.png"/>',de,j,je,g,It="To get started, pip install <code>fenic</code>:",ge,C,Ce,h,he,b,Ut="Instantiate a fenic session with the default configuration (sufficient for reading datasets and other non-semantic operations):",be,Z,Ze,$,$e,B,dt="fenic is an opinionated data processing framework that combines:",Be,W,jt="<li><strong>DataFrame API</strong>: PySpark-inspired operations for familiar data manipulation</li> <li><strong>Semantic Operations</strong>: Built-in AI/LLM operations including semantic functions, embeddings, and clustering</li> <li><strong>Model Integration</strong>: Native support for AI providers (Anthropic, OpenAI, Cohere, Google)</li> <li><strong>Query Optimization</strong>: Automatic optimization through logical plan transformations</li>",We,G,Ge,A,gt="fenic can read datasets directly from the Hugging Face Hub using the <code>hf://</code> protocol. This functionality is built into fenic’s DataFrameReader interface.",Ae,R,Re,v,Ct="fenic supports reading the following formats from Hugging Face:",ve,q,ht="<li><strong>Parquet files</strong> (<code>.parquet</code>)</li> <li><strong>CSV files</strong> (<code>.csv</code>)</li>",qe,Y,Ye,N,bt="To read a dataset from the Hugging Face Hub:",Ne,F,Fe,E,Ee,X,Xe,k,Zt='<p><strong>Note:</strong> In fenic, a schema is the set of column names and their data types. When you enable <code>merge_schemas</code>, fenic tries to reconcile differences across files by filling missing columns with nulls and widening types where it can. Some layouts still cannot be merged—consult the fenic docs for <a href="https://docs.fenic.ai/latest/reference/fenic/?h=parquet#fenic.DataFrameReader.csv" rel="nofollow">CSV schema merging limitations</a> and <a href="https://docs.fenic.ai/latest/reference/fenic/?h=parquet#fenic.DataFrameReader.parquet" rel="nofollow">Parquet schema merging limitations</a>.</p>',ke,x,xe,z,$t="To read private datasets, you need to set your Hugging Face token as an environment variable:",ze,S,Se,Q,Qe,V,Bt="The Hugging Face path format in fenic follows this structure:",Ve,_,_e,H,Wt="You can also specify dataset revisions or versions:",He,L,Le,D,Gt="Features:",De,P,At=`<li>Supports glob patterns (<code>*</code>, <code>**</code>)</li> <li>Dataset revisions/versions using <code>@</code> notation:
<ul><li>Specific commit: <code>@d50d8923b5934dc8e74b66e6e4b0e2cd85e9142e</code></li> <li>Branch: <code>@refs/convert/parquet</code></li> <li>Branch alias: <code>@~parquet</code></li></ul></li> <li>Requires <code>HF_TOKEN</code> environment variable for private datasets</li>`,Pe,K,Ke,O,Rt="fenic allows you to combine multiple data sources in a single read operation, including mixing different protocols:",Oe,ee,et,te,vt="This flexibility allows you to seamlessly combine data from Hugging Face Hub and local files in your data processing pipeline.",tt,le,lt,se,qt="Once loaded from Hugging Face, you can use fenic’s full DataFrame API:",st,ae,at,ne,nt,ie,it,Me,Yt="To use semantic and embedding operations, configure language and embedding models in your SessionConfig. Once configured:",Mt,Je,Jt,pe,pt,me,mt,Te,Tt,oe,Nt='<li><a href="https://github.com/typedef-ai/fenic" rel="nofollow">fenic GitHub Repository</a></li> <li><a href="https://docs.fenic.ai/latest/" rel="nofollow">fenic Documentation</a></li>',ot,ce,ct,we,yt;return I=new Vt({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),U=new w({props:{title:"fenic",local:"fenic",headingTag:"h1"}}),j=new w({props:{title:"Getting Started",local:"getting-started",headingTag:"h2"}}),C=new r({props:{code:"cGlwJTIwaW5zdGFsbCUyMGZlbmlj",highlighted:"pip install fenic",wrap:!1}}),h=new w({props:{title:"Create a Session",local:"create-a-session",headingTag:"h3"}}),Z=new r({props:{code:"aW1wb3J0JTIwZmVuaWMlMjBhcyUyMGZjJTBBJTBBc2Vzc2lvbiUyMCUzRCUyMGZjLlNlc3Npb24uZ2V0X29yX2NyZWF0ZShmYy5TZXNzaW9uQ29uZmlnKCkp",highlighted:`<span class="hljs-keyword">import</span> fenic <span class="hljs-keyword">as</span> fc

session = fc.Session.get_or_create(fc.SessionConfig())`,wrap:!1}}),$=new w({props:{title:"Overview",local:"overview",headingTag:"h2"}}),G=new w({props:{title:"Read from Hugging Face Hub",local:"read-from-hugging-face-hub",headingTag:"h2"}}),R=new w({props:{title:"Supported Formats",local:"supported-formats",headingTag:"h3"}}),Y=new w({props:{title:"Reading Datasets",local:"reading-datasets",headingTag:"h3"}}),F=new r({props:{code:"aW1wb3J0JTIwZmVuaWMlMjBhcyUyMGZjJTBBJTBBc2Vzc2lvbiUyMCUzRCUyMGZjLlNlc3Npb24uZ2V0X29yX2NyZWF0ZShmYy5TZXNzaW9uQ29uZmlnKCkpJTBBJTBBJTIzJTIwUmVhZCUyMGElMjBDU1YlMjBmaWxlJTIwZnJvbSUyMGElMjBwdWJsaWMlMjBkYXRhc2V0JTBBZGYlMjAlM0QlMjBzZXNzaW9uLnJlYWQuY3N2KCUyMmhmJTNBJTJGJTJGZGF0YXNldHMlMkZkYXRhc2V0cy1leGFtcGxlcyUyRmRvYy1mb3JtYXRzLWNzdi0xJTJGZGF0YS5jc3YlMjIpJTBBJTBBJTIzJTIwUmVhZCUyMFBhcnF1ZXQlMjBmaWxlcyUyMHVzaW5nJTIwZ2xvYiUyMHBhdHRlcm5zJTBBZGYlMjAlM0QlMjBzZXNzaW9uLnJlYWQucGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGY2FpcyUyRm1tbHUlMkZhc3Ryb25vbXklMkYqLnBhcnF1ZXQlMjIpJTBBJTBBJTIzJTIwUmVhZCUyMGZyb20lMjBhJTIwc3BlY2lmaWMlMjBkYXRhc2V0JTIwcmV2aXNpb24lMEFkZiUyMCUzRCUyMHNlc3Npb24ucmVhZC5wYXJxdWV0KCUyMmhmJTNBJTJGJTJGZGF0YXNldHMlMkZkYXRhc2V0cy1leGFtcGxlcyUyRmRvYy1mb3JtYXRzLWNzdi0xJTQwfnBhcnF1ZXQlMkYqKiUyRioucGFycXVldCUyMik=",highlighted:`<span class="hljs-keyword">import</span> fenic <span class="hljs-keyword">as</span> fc

session = fc.Session.get_or_create(fc.SessionConfig())

<span class="hljs-comment"># Read a CSV file from a public dataset</span>
df = session.read.csv(<span class="hljs-string">&quot;hf://datasets/datasets-examples/doc-formats-csv-1/data.csv&quot;</span>)

<span class="hljs-comment"># Read Parquet files using glob patterns</span>
df = session.read.parquet(<span class="hljs-string">&quot;hf://datasets/cais/mmlu/astronomy/*.parquet&quot;</span>)

<span class="hljs-comment"># Read from a specific dataset revision</span>
df = session.read.parquet(<span class="hljs-string">&quot;hf://datasets/datasets-examples/doc-formats-csv-1@~parquet/**/*.parquet&quot;</span>)`,wrap:!1}}),E=new w({props:{title:"Reading with Schema Management",local:"reading-with-schema-management",headingTag:"h3"}}),X=new r({props:{code:"JTIzJTIwUmVhZCUyMG11bHRpcGxlJTIwQ1NWJTIwZmlsZXMlMjB3aXRoJTIwc2NoZW1hJTIwbWVyZ2luZyUwQWRmJTIwJTNEJTIwc2Vzc2lvbi5yZWFkLmNzdiglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZkYXRhc2V0X25hbWUlMkYqLmNzdiUyMiUyQyUyMG1lcmdlX3NjaGVtYXMlM0RUcnVlKSUwQSUwQSUyMyUyMFJlYWQlMjBtdWx0aXBsZSUyMFBhcnF1ZXQlMjBmaWxlcyUyMHdpdGglMjBzY2hlbWElMjBtZXJnaW5nJTBBZGYlMjAlM0QlMjBzZXNzaW9uLnJlYWQucGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGdXNlcm5hbWUlMkZkYXRhc2V0X25hbWUlMkYqLnBhcnF1ZXQlMjIlMkMlMjBtZXJnZV9zY2hlbWFzJTNEVHJ1ZSk=",highlighted:`<span class="hljs-comment"># Read multiple CSV files with schema merging</span>
df = session.read.csv(<span class="hljs-string">&quot;hf://datasets/username/dataset_name/*.csv&quot;</span>, merge_schemas=<span class="hljs-literal">True</span>)

<span class="hljs-comment"># Read multiple Parquet files with schema merging</span>
df = session.read.parquet(<span class="hljs-string">&quot;hf://datasets/username/dataset_name/*.parquet&quot;</span>, merge_schemas=<span class="hljs-literal">True</span>)`,wrap:!1}}),x=new w({props:{title:"Authentication",local:"authentication",headingTag:"h3"}}),S=new r({props:{code:"ZXhwb3J0JTIwSEZfVE9LRU4lM0QlMjJ5b3VyX2h1Z2dpbmdfZmFjZV90b2tlbl9oZXJlJTIy",highlighted:"export HF_TOKEN=&quot;your_hugging_face_token_here&quot;",wrap:!1}}),Q=new w({props:{title:"Path Format",local:"path-format",headingTag:"h3"}}),_=new r({props:{code:"aGYlM0ElMkYlMkYlN0JyZXBvX3R5cGUlN0QlMkYlN0JyZXBvX2lkJTdEJTJGJTdCcGF0aF90b19maWxlJTdE",highlighted:'hf:<span class="hljs-regexp">//</span>{repo_type}<span class="hljs-regexp">/{repo_id}/</span>{path_to_file}',wrap:!1}}),L=new r({props:{code:"aGYlM0ElMkYlMkYlN0JyZXBvX3R5cGUlN0QlMkYlN0JyZXBvX2lkJTdEJTQwJTdCcmV2aXNpb24lN0QlMkYlN0JwYXRoX3RvX2ZpbGUlN0Q=",highlighted:'<span class="hljs-title">hf</span>://{repo_<span class="hljs-keyword">type</span>}/{repo_id}@{revision}/{path_to_file}',wrap:!1}}),K=new w({props:{title:"Mixing Data Sources",local:"mixing-data-sources",headingTag:"h3"}}),ee=new r({props:{code:"JTIzJTIwTWl4JTIwSEYlMjBhbmQlMjBsb2NhbCUyMGZpbGVzJTIwaW4lMjBvbmUlMjByZWFkJTIwY2FsbCUwQWRmJTIwJTNEJTIwc2Vzc2lvbi5yZWFkLnBhcnF1ZXQoJTVCJTBBJTIwJTIwJTIwJTIwJTIyaGYlM0ElMkYlMkZkYXRhc2V0cyUyRmNhaXMlMkZtbWx1JTJGYXN0cm9ub215JTJGKi5wYXJxdWV0JTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIyZmlsZSUzQSUyRiUyRiUyRmxvY2FsJTJGZGF0YSUyRioucGFycXVldCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMi4lMkZyZWxhdGl2ZSUyRnBhdGglMkZkYXRhLnBhcnF1ZXQlMjIlMEElNUQp",highlighted:`<span class="hljs-comment"># Mix HF and local files in one read call</span>
df = session.read.parquet([
    <span class="hljs-string">&quot;hf://datasets/cais/mmlu/astronomy/*.parquet&quot;</span>,
    <span class="hljs-string">&quot;file:///local/data/*.parquet&quot;</span>,
    <span class="hljs-string">&quot;./relative/path/data.parquet&quot;</span>
])`,wrap:!1}}),le=new w({props:{title:"Processing Data from Hugging Face",local:"processing-data-from-hugging-face",headingTag:"h2"}}),ae=new w({props:{title:"Basic DataFrame Operations",local:"basic-dataframe-operations",headingTag:"h3"}}),ne=new r({props:{code:"aW1wb3J0JTIwZmVuaWMlMjBhcyUyMGZjJTBBJTBBc2Vzc2lvbiUyMCUzRCUyMGZjLlNlc3Npb24uZ2V0X29yX2NyZWF0ZShmYy5TZXNzaW9uQ29uZmlnKCkpJTBBJTBBJTIzJTIwTG9hZCUyMElNREIlMjBkYXRhc2V0JTIwZnJvbSUyMEh1Z2dpbmclMjBGYWNlJTBBZGYlMjAlM0QlMjBzZXNzaW9uLnJlYWQucGFycXVldCglMjJoZiUzQSUyRiUyRmRhdGFzZXRzJTJGaW1kYiUyRnBsYWluX3RleHQlMkZ0cmFpbi0qLnBhcnF1ZXQlMjIpJTBBJTBBJTIzJTIwRmlsdGVyJTIwYW5kJTIwc2VsZWN0JTBBcG9zaXRpdmVfcmV2aWV3cyUyMCUzRCUyMGRmLmZpbHRlcihmYy5jb2woJTIybGFiZWwlMjIpJTIwJTNEJTNEJTIwMSkuc2VsZWN0KCUyMnRleHQlMjIlMkMlMjAlMjJsYWJlbCUyMiklMEElMEElMjMlMjBHcm91cCUyMGJ5JTIwYW5kJTIwYWdncmVnYXRlJTBBbGFiZWxfY291bnRzJTIwJTNEJTIwZGYuZ3JvdXBfYnkoJTIybGFiZWwlMjIpLmFnZyglMEElMjAlMjAlMjAlMjBmYy5jb3VudCglMjIqJTIyKS5hbGlhcyglMjJjb3VudCUyMiklMEEp",highlighted:`<span class="hljs-keyword">import</span> fenic <span class="hljs-keyword">as</span> fc

session = fc.Session.get_or_create(fc.SessionConfig())

<span class="hljs-comment"># Load IMDB dataset from Hugging Face</span>
df = session.read.parquet(<span class="hljs-string">&quot;hf://datasets/imdb/plain_text/train-*.parquet&quot;</span>)

<span class="hljs-comment"># Filter and select</span>
positive_reviews = df.<span class="hljs-built_in">filter</span>(fc.col(<span class="hljs-string">&quot;label&quot;</span>) == <span class="hljs-number">1</span>).select(<span class="hljs-string">&quot;text&quot;</span>, <span class="hljs-string">&quot;label&quot;</span>)

<span class="hljs-comment"># Group by and aggregate</span>
label_counts = df.group_by(<span class="hljs-string">&quot;label&quot;</span>).agg(
    fc.count(<span class="hljs-string">&quot;*&quot;</span>).alias(<span class="hljs-string">&quot;count&quot;</span>)
)`,wrap:!1}}),ie=new w({props:{title:"AI-Powered Operations",local:"ai-powered-operations",headingTag:"h3"}}),Je=new r({props:{code:"aW1wb3J0JTIwZmVuaWMlMjBhcyUyMGZjJTBBJTBBJTIzJTIwUmVxdWlyZXMlMjBPUEVOQUlfQVBJX0tFWSUyMHRvJTIwYmUlMjBzZXQlMjBmb3IlMjBsYW5ndWFnZSUyMGFuZCUyMGVtYmVkZGluZyUyMGNhbGxzJTBBc2Vzc2lvbiUyMCUzRCUyMGZjLlNlc3Npb24uZ2V0X29yX2NyZWF0ZSglMEElMjAlMjAlMjAlMjBmYy5TZXNzaW9uQ29uZmlnKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNlbWFudGljJTNEZmMuU2VtYW50aWNDb25maWcoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbGFuZ3VhZ2VfbW9kZWxzJTNEJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyZ3B0LTRvLW1pbmklMjIlM0ElMjBmYy5PcGVuQUlMYW5ndWFnZU1vZGVsKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1vZGVsX25hbWUlM0QlMjJncHQtNG8tbWluaSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJwbSUzRDYwJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdHBtJTNENjAwMDAlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZW1iZWRkaW5nX21vZGVscyUzRCU3QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMnRleHQtZW1iZWRkaW5nLTMtc21hbGwlMjIlM0ElMjBmYy5PcGVuQUlFbWJlZGRpbmdNb2RlbCglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBtb2RlbF9uYW1lJTNEJTIydGV4dC1lbWJlZGRpbmctMy1zbWFsbCUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJwbSUzRDYwJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdHBtJTNENjAwMDAlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUwQSUyMCUyMCUyMCUyMCklMEEpJTBBJTBBJTIzJTIwTG9hZCUyMGElMjB0ZXh0JTIwZGF0YXNldCUyMGZyb20lMjBIdWdnaW5nJTIwRmFjZSUwQWRmJTIwJTNEJTIwc2Vzc2lvbi5yZWFkLnBhcnF1ZXQoJTIyaGYlM0ElMkYlMkZkYXRhc2V0cyUyRmltZGIlMkZwbGFpbl90ZXh0JTJGdHJhaW4tMDAwMDAtb2YtMDAwMDEucGFycXVldCUyMiklMEElMEElMjMlMjBBZGQlMjBlbWJlZGRpbmdzJTIwdG8lMjB0ZXh0JTIwY29sdW1ucyUwQWRmX3dpdGhfZW1iZWRkaW5ncyUyMCUzRCUyMGRmLnNlbGVjdCglMEElMjAlMjAlMjAlMjAlMjIqJTIyJTJDJTBBJTIwJTIwJTIwJTIwZmMuc2VtYW50aWMuZW1iZWQoZmMuY29sKCUyMnRleHQlMjIpKS5hbGlhcyglMjJlbWJlZGRpbmclMjIpJTBBKSUwQSUwQSUyMyUyMEFwcGx5JTIwc2VtYW50aWMlMjBmdW5jdGlvbnMlMjBmb3IlMjBzZW50aW1lbnQlMjBhbmFseXNpcyUwQWRmX2FuYWx5emVkJTIwJTNEJTIwZGZfd2l0aF9lbWJlZGRpbmdzLnNlbGVjdCglMEElMjAlMjAlMjAlMjAlMjIqJTIyJTJDJTBBJTIwJTIwJTIwJTIwZmMuc2VtYW50aWMuYW5hbHl6ZV9zZW50aW1lbnQoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZmMuY29sKCUyMnRleHQlMjIpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbW9kZWxfYWxpYXMlM0QlMjJncHQtNG8tbWluaSUyMiUyQyUyMCUyMCUyMyUyME9wdGlvbmFsJTNBJTIwc3BlY2lmeSUyMG1vZGVsJTBBJTIwJTIwJTIwJTIwKS5hbGlhcyglMjJzZW50aW1lbnQlMjIpJTBBKQ==",highlighted:`<span class="hljs-keyword">import</span> fenic <span class="hljs-keyword">as</span> fc

<span class="hljs-comment"># Requires OPENAI_API_KEY to be set for language and embedding calls</span>
session = fc.Session.get_or_create(
    fc.SessionConfig(
        semantic=fc.SemanticConfig(
            language_models={
                <span class="hljs-string">&quot;gpt-4o-mini&quot;</span>: fc.OpenAILanguageModel(
                    model_name=<span class="hljs-string">&quot;gpt-4o-mini&quot;</span>,
                    rpm=<span class="hljs-number">60</span>,
                    tpm=<span class="hljs-number">60000</span>,
                )
            },
            embedding_models={
                <span class="hljs-string">&quot;text-embedding-3-small&quot;</span>: fc.OpenAIEmbeddingModel(
                    model_name=<span class="hljs-string">&quot;text-embedding-3-small&quot;</span>,
                    rpm=<span class="hljs-number">60</span>,
                    tpm=<span class="hljs-number">60000</span>,
                )
            },
        )
    )
)

<span class="hljs-comment"># Load a text dataset from Hugging Face</span>
df = session.read.parquet(<span class="hljs-string">&quot;hf://datasets/imdb/plain_text/train-00000-of-00001.parquet&quot;</span>)

<span class="hljs-comment"># Add embeddings to text columns</span>
df_with_embeddings = df.select(
    <span class="hljs-string">&quot;*&quot;</span>,
    fc.semantic.embed(fc.col(<span class="hljs-string">&quot;text&quot;</span>)).alias(<span class="hljs-string">&quot;embedding&quot;</span>)
)

<span class="hljs-comment"># Apply semantic functions for sentiment analysis</span>
df_analyzed = df_with_embeddings.select(
    <span class="hljs-string">&quot;*&quot;</span>,
    fc.semantic.analyze_sentiment(
        fc.col(<span class="hljs-string">&quot;text&quot;</span>),
        model_alias=<span class="hljs-string">&quot;gpt-4o-mini&quot;</span>,  <span class="hljs-comment"># Optional: specify model</span>
    ).alias(<span class="hljs-string">&quot;sentiment&quot;</span>)
)`,wrap:!1}}),pe=new w({props:{title:"Example: Analyzing MMLU Dataset",local:"example-analyzing-mmlu-dataset",headingTag:"h2"}}),me=new r({props:{code:"aW1wb3J0JTIwZmVuaWMlMjBhcyUyMGZjJTBBJTBBJTIzJTIwUmVxdWlyZXMlMjBPUEVOQUlfQVBJX0tFWSUyMHRvJTIwYmUlMjBzZXQlMjBmb3IlMjBzZW1hbnRpYyUyMGNhbGxzJTBBc2Vzc2lvbiUyMCUzRCUyMGZjLlNlc3Npb24uZ2V0X29yX2NyZWF0ZSglMEElMjAlMjAlMjAlMjBmYy5TZXNzaW9uQ29uZmlnKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHNlbWFudGljJTNEZmMuU2VtYW50aWNDb25maWcoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbGFuZ3VhZ2VfbW9kZWxzJTNEJTdCJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyZ3B0LTRvLW1pbmklMjIlM0ElMjBmYy5PcGVuQUlMYW5ndWFnZU1vZGVsKCUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMG1vZGVsX25hbWUlM0QlMjJncHQtNG8tbWluaSUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMHJwbSUzRDYwJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwdHBtJTNENjAwMDAlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjApJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTdEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKSUwQSUyMCUyMCUyMCUyMCklMEEpJTBBJTBBJTIzJTIwTG9hZCUyME1NTFUlMjBhc3Ryb25vbXklMjBzdWJzZXQlMjBmcm9tJTIwSHVnZ2luZyUyMEZhY2UlMEFkZiUyMCUzRCUyMHNlc3Npb24ucmVhZC5wYXJxdWV0KCUyMmhmJTNBJTJGJTJGZGF0YXNldHMlMkZjYWlzJTJGbW1sdSUyRmFzdHJvbm9teSUyRioucGFycXVldCUyMiklMEElMEElMjMlMjBQcm9jZXNzJTIwdGhlJTIwZGF0YSUwQXByb2Nlc3NlZF9kZiUyMCUzRCUyMChkZiUwQSUyMCUyMCUyMCUyMCUyMyUyMEZpbHRlciUyMGZvciUyMHNwZWNpZmljJTIwY3JpdGVyaWElMEElMjAlMjAlMjAlMjAuZmlsdGVyKGZjLmNvbCglMjJzdWJqZWN0JTIyKSUyMCUzRCUzRCUyMCUyMmFzdHJvbm9teSUyMiklMEElMjAlMjAlMjAlMjAlMjMlMjBTZWxlY3QlMjByZWxldmFudCUyMGNvbHVtbnMlMEElMjAlMjAlMjAlMjAuc2VsZWN0KCUyMnF1ZXN0aW9uJTIyJTJDJTIwJTIyY2hvaWNlcyUyMiUyQyUyMCUyMmFuc3dlciUyMiklMEElMjAlMjAlMjAlMjAlMjMlMjBBZGQlMjBkaWZmaWN1bHR5JTIwYW5hbHlzaXMlMjB1c2luZyUyMHNlbWFudGljLm1hcCUwQSUyMCUyMCUyMCUyMC5zZWxlY3QoJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIyKiUyMiUyQyUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMGZjLnNlbWFudGljLm1hcCglMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjJSYXRlJTIwdGhlJTIwZGlmZmljdWx0eSUyMG9mJTIwdGhpcyUyMHF1ZXN0aW9uJTIwZnJvbSUyMDEtNSUzQSUyMCU3QiU3QnF1ZXN0aW9uJTdEJTdEJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwcXVlc3Rpb24lM0RmYy5jb2woJTIycXVlc3Rpb24lMjIpJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwbW9kZWxfYWxpYXMlM0QlMjJncHQtNG8tbWluaSUyMiUyMCUyMCUyMyUyME9wdGlvbmFsJTNBJTIwc3BlY2lmeSUyMG1vZGVsJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwKS5hbGlhcyglMjJkaWZmaWN1bHR5JTIyKSUwQSUyMCUyMCUyMCUyMCklMEEpJTBBJTBBJTIzJTIwU2hvdyUyMHJlc3VsdHMlMEFwcm9jZXNzZWRfZGYuc2hvdygp",highlighted:`<span class="hljs-keyword">import</span> fenic <span class="hljs-keyword">as</span> fc

<span class="hljs-comment"># Requires OPENAI_API_KEY to be set for semantic calls</span>
session = fc.Session.get_or_create(
    fc.SessionConfig(
        semantic=fc.SemanticConfig(
            language_models={
                <span class="hljs-string">&quot;gpt-4o-mini&quot;</span>: fc.OpenAILanguageModel(
                    model_name=<span class="hljs-string">&quot;gpt-4o-mini&quot;</span>,
                    rpm=<span class="hljs-number">60</span>,
                    tpm=<span class="hljs-number">60000</span>,
                )
            },
        )
    )
)

<span class="hljs-comment"># Load MMLU astronomy subset from Hugging Face</span>
df = session.read.parquet(<span class="hljs-string">&quot;hf://datasets/cais/mmlu/astronomy/*.parquet&quot;</span>)

<span class="hljs-comment"># Process the data</span>
processed_df = (df
    <span class="hljs-comment"># Filter for specific criteria</span>
    .<span class="hljs-built_in">filter</span>(fc.col(<span class="hljs-string">&quot;subject&quot;</span>) == <span class="hljs-string">&quot;astronomy&quot;</span>)
    <span class="hljs-comment"># Select relevant columns</span>
    .select(<span class="hljs-string">&quot;question&quot;</span>, <span class="hljs-string">&quot;choices&quot;</span>, <span class="hljs-string">&quot;answer&quot;</span>)
    <span class="hljs-comment"># Add difficulty analysis using semantic.map</span>
    .select(
        <span class="hljs-string">&quot;*&quot;</span>,
        fc.semantic.<span class="hljs-built_in">map</span>(
            <span class="hljs-string">&quot;Rate the difficulty of this question from 1-5: {{question}}&quot;</span>,
            question=fc.col(<span class="hljs-string">&quot;question&quot;</span>),
            model_alias=<span class="hljs-string">&quot;gpt-4o-mini&quot;</span>  <span class="hljs-comment"># Optional: specify model</span>
        ).alias(<span class="hljs-string">&quot;difficulty&quot;</span>)
    )
)

<span class="hljs-comment"># Show results</span>
processed_df.show()`,wrap:!1}}),Te=new w({props:{title:"Resources",local:"resources",headingTag:"h2"}}),ce=new _t({props:{source:"https://github.com/huggingface/hub-docs/blob/main/docs/hub/datasets-fenic.md"}}),{c(){f=o("meta"),re=a(),ye=o("p"),fe=a(),i(I.$$.fragment),ue=a(),i(U.$$.fragment),Ie=a(),d=o("p"),d.innerHTML=ft,Ue=a(),u=o("div"),u.innerHTML=ut,de=a(),i(j.$$.fragment),je=a(),g=o("p"),g.innerHTML=It,ge=a(),i(C.$$.fragment),Ce=a(),i(h.$$.fragment),he=a(),b=o("p"),b.textContent=Ut,be=a(),i(Z.$$.fragment),Ze=a(),i($.$$.fragment),$e=a(),B=o("p"),B.textContent=dt,Be=a(),W=o("ul"),W.innerHTML=jt,We=a(),i(G.$$.fragment),Ge=a(),A=o("p"),A.innerHTML=gt,Ae=a(),i(R.$$.fragment),Re=a(),v=o("p"),v.textContent=Ct,ve=a(),q=o("ul"),q.innerHTML=ht,qe=a(),i(Y.$$.fragment),Ye=a(),N=o("p"),N.textContent=bt,Ne=a(),i(F.$$.fragment),Fe=a(),i(E.$$.fragment),Ee=a(),i(X.$$.fragment),Xe=a(),k=o("blockquote"),k.innerHTML=Zt,ke=a(),i(x.$$.fragment),xe=a(),z=o("p"),z.textContent=$t,ze=a(),i(S.$$.fragment),Se=a(),i(Q.$$.fragment),Qe=a(),V=o("p"),V.textContent=Bt,Ve=a(),i(_.$$.fragment),_e=a(),H=o("p"),H.textContent=Wt,He=a(),i(L.$$.fragment),Le=a(),D=o("p"),D.textContent=Gt,De=a(),P=o("ul"),P.innerHTML=At,Pe=a(),i(K.$$.fragment),Ke=a(),O=o("p"),O.textContent=Rt,Oe=a(),i(ee.$$.fragment),et=a(),te=o("p"),te.textContent=vt,tt=a(),i(le.$$.fragment),lt=a(),se=o("p"),se.textContent=qt,st=a(),i(ae.$$.fragment),at=a(),i(ne.$$.fragment),nt=a(),i(ie.$$.fragment),it=a(),Me=o("p"),Me.textContent=Yt,Mt=a(),i(Je.$$.fragment),Jt=a(),i(pe.$$.fragment),pt=a(),i(me.$$.fragment),mt=a(),i(Te.$$.fragment),Tt=a(),oe=o("ul"),oe.innerHTML=Nt,ot=a(),i(ce.$$.fragment),ct=a(),we=o("p"),this.h()},l(e){const t=St("svelte-u9bgzb",document.head);f=c(t,"META",{name:!0,content:!0}),t.forEach(l),re=n(e),ye=c(e,"P",{}),Ft(ye).forEach(l),fe=n(e),M(I.$$.fragment,e),ue=n(e),M(U.$$.fragment,e),Ie=n(e),d=c(e,"P",{"data-svelte-h":!0}),y(d)!=="svelte-vn66qz"&&(d.innerHTML=ft),Ue=n(e),u=c(e,"DIV",{class:!0,"data-svelte-h":!0}),y(u)!=="svelte-1ujdj8h"&&(u.innerHTML=ut),de=n(e),M(j.$$.fragment,e),je=n(e),g=c(e,"P",{"data-svelte-h":!0}),y(g)!=="svelte-ug8u64"&&(g.innerHTML=It),ge=n(e),M(C.$$.fragment,e),Ce=n(e),M(h.$$.fragment,e),he=n(e),b=c(e,"P",{"data-svelte-h":!0}),y(b)!=="svelte-nie6dm"&&(b.textContent=Ut),be=n(e),M(Z.$$.fragment,e),Ze=n(e),M($.$$.fragment,e),$e=n(e),B=c(e,"P",{"data-svelte-h":!0}),y(B)!=="svelte-125c6zc"&&(B.textContent=dt),Be=n(e),W=c(e,"UL",{"data-svelte-h":!0}),y(W)!=="svelte-1aj6uq0"&&(W.innerHTML=jt),We=n(e),M(G.$$.fragment,e),Ge=n(e),A=c(e,"P",{"data-svelte-h":!0}),y(A)!=="svelte-slp14x"&&(A.innerHTML=gt),Ae=n(e),M(R.$$.fragment,e),Re=n(e),v=c(e,"P",{"data-svelte-h":!0}),y(v)!=="svelte-c06s8f"&&(v.textContent=Ct),ve=n(e),q=c(e,"UL",{"data-svelte-h":!0}),y(q)!=="svelte-1j6s0ic"&&(q.innerHTML=ht),qe=n(e),M(Y.$$.fragment,e),Ye=n(e),N=c(e,"P",{"data-svelte-h":!0}),y(N)!=="svelte-d49cqg"&&(N.textContent=bt),Ne=n(e),M(F.$$.fragment,e),Fe=n(e),M(E.$$.fragment,e),Ee=n(e),M(X.$$.fragment,e),Xe=n(e),k=c(e,"BLOCKQUOTE",{"data-svelte-h":!0}),y(k)!=="svelte-kl0uyj"&&(k.innerHTML=Zt),ke=n(e),M(x.$$.fragment,e),xe=n(e),z=c(e,"P",{"data-svelte-h":!0}),y(z)!=="svelte-d5pabx"&&(z.textContent=$t),ze=n(e),M(S.$$.fragment,e),Se=n(e),M(Q.$$.fragment,e),Qe=n(e),V=c(e,"P",{"data-svelte-h":!0}),y(V)!=="svelte-bvh4z0"&&(V.textContent=Bt),Ve=n(e),M(_.$$.fragment,e),_e=n(e),H=c(e,"P",{"data-svelte-h":!0}),y(H)!=="svelte-bhurm7"&&(H.textContent=Wt),He=n(e),M(L.$$.fragment,e),Le=n(e),D=c(e,"P",{"data-svelte-h":!0}),y(D)!=="svelte-fn4rp3"&&(D.textContent=Gt),De=n(e),P=c(e,"UL",{"data-svelte-h":!0}),y(P)!=="svelte-1lgwrby"&&(P.innerHTML=At),Pe=n(e),M(K.$$.fragment,e),Ke=n(e),O=c(e,"P",{"data-svelte-h":!0}),y(O)!=="svelte-1syu550"&&(O.textContent=Rt),Oe=n(e),M(ee.$$.fragment,e),et=n(e),te=c(e,"P",{"data-svelte-h":!0}),y(te)!=="svelte-1m42bp7"&&(te.textContent=vt),tt=n(e),M(le.$$.fragment,e),lt=n(e),se=c(e,"P",{"data-svelte-h":!0}),y(se)!=="svelte-1s7nkz5"&&(se.textContent=qt),st=n(e),M(ae.$$.fragment,e),at=n(e),M(ne.$$.fragment,e),nt=n(e),M(ie.$$.fragment,e),it=n(e),Me=c(e,"P",{"data-svelte-h":!0}),y(Me)!=="svelte-1wiw36t"&&(Me.textContent=Yt),Mt=n(e),M(Je.$$.fragment,e),Jt=n(e),M(pe.$$.fragment,e),pt=n(e),M(me.$$.fragment,e),mt=n(e),M(Te.$$.fragment,e),Tt=n(e),oe=c(e,"UL",{"data-svelte-h":!0}),y(oe)!=="svelte-2gsla3"&&(oe.innerHTML=Nt),ot=n(e),M(ce.$$.fragment,e),ct=n(e),we=c(e,"P",{}),Ft(we).forEach(l),this.h()},h(){wt(f,"name","hf:doc:metadata"),wt(f,"content",Lt),wt(u,"class","flex justify-center")},m(e,t){Qt(document.head,f),s(e,re,t),s(e,ye,t),s(e,fe,t),J(I,e,t),s(e,ue,t),J(U,e,t),s(e,Ie,t),s(e,d,t),s(e,Ue,t),s(e,u,t),s(e,de,t),J(j,e,t),s(e,je,t),s(e,g,t),s(e,ge,t),J(C,e,t),s(e,Ce,t),J(h,e,t),s(e,he,t),s(e,b,t),s(e,be,t),J(Z,e,t),s(e,Ze,t),J($,e,t),s(e,$e,t),s(e,B,t),s(e,Be,t),s(e,W,t),s(e,We,t),J(G,e,t),s(e,Ge,t),s(e,A,t),s(e,Ae,t),J(R,e,t),s(e,Re,t),s(e,v,t),s(e,ve,t),s(e,q,t),s(e,qe,t),J(Y,e,t),s(e,Ye,t),s(e,N,t),s(e,Ne,t),J(F,e,t),s(e,Fe,t),J(E,e,t),s(e,Ee,t),J(X,e,t),s(e,Xe,t),s(e,k,t),s(e,ke,t),J(x,e,t),s(e,xe,t),s(e,z,t),s(e,ze,t),J(S,e,t),s(e,Se,t),J(Q,e,t),s(e,Qe,t),s(e,V,t),s(e,Ve,t),J(_,e,t),s(e,_e,t),s(e,H,t),s(e,He,t),J(L,e,t),s(e,Le,t),s(e,D,t),s(e,De,t),s(e,P,t),s(e,Pe,t),J(K,e,t),s(e,Ke,t),s(e,O,t),s(e,Oe,t),J(ee,e,t),s(e,et,t),s(e,te,t),s(e,tt,t),J(le,e,t),s(e,lt,t),s(e,se,t),s(e,st,t),J(ae,e,t),s(e,at,t),J(ne,e,t),s(e,nt,t),J(ie,e,t),s(e,it,t),s(e,Me,t),s(e,Mt,t),J(Je,e,t),s(e,Jt,t),J(pe,e,t),s(e,pt,t),J(me,e,t),s(e,mt,t),J(Te,e,t),s(e,Tt,t),s(e,oe,t),s(e,ot,t),J(ce,e,t),s(e,ct,t),s(e,we,t),yt=!0},p:Xt,i(e){yt||(p(I.$$.fragment,e),p(U.$$.fragment,e),p(j.$$.fragment,e),p(C.$$.fragment,e),p(h.$$.fragment,e),p(Z.$$.fragment,e),p($.$$.fragment,e),p(G.$$.fragment,e),p(R.$$.fragment,e),p(Y.$$.fragment,e),p(F.$$.fragment,e),p(E.$$.fragment,e),p(X.$$.fragment,e),p(x.$$.fragment,e),p(S.$$.fragment,e),p(Q.$$.fragment,e),p(_.$$.fragment,e),p(L.$$.fragment,e),p(K.$$.fragment,e),p(ee.$$.fragment,e),p(le.$$.fragment,e),p(ae.$$.fragment,e),p(ne.$$.fragment,e),p(ie.$$.fragment,e),p(Je.$$.fragment,e),p(pe.$$.fragment,e),p(me.$$.fragment,e),p(Te.$$.fragment,e),p(ce.$$.fragment,e),yt=!0)},o(e){m(I.$$.fragment,e),m(U.$$.fragment,e),m(j.$$.fragment,e),m(C.$$.fragment,e),m(h.$$.fragment,e),m(Z.$$.fragment,e),m($.$$.fragment,e),m(G.$$.fragment,e),m(R.$$.fragment,e),m(Y.$$.fragment,e),m(F.$$.fragment,e),m(E.$$.fragment,e),m(X.$$.fragment,e),m(x.$$.fragment,e),m(S.$$.fragment,e),m(Q.$$.fragment,e),m(_.$$.fragment,e),m(L.$$.fragment,e),m(K.$$.fragment,e),m(ee.$$.fragment,e),m(le.$$.fragment,e),m(ae.$$.fragment,e),m(ne.$$.fragment,e),m(ie.$$.fragment,e),m(Je.$$.fragment,e),m(pe.$$.fragment,e),m(me.$$.fragment,e),m(Te.$$.fragment,e),m(ce.$$.fragment,e),yt=!1},d(e){e&&(l(re),l(ye),l(fe),l(ue),l(Ie),l(d),l(Ue),l(u),l(de),l(je),l(g),l(ge),l(Ce),l(he),l(b),l(be),l(Ze),l($e),l(B),l(Be),l(W),l(We),l(Ge),l(A),l(Ae),l(Re),l(v),l(ve),l(q),l(qe),l(Ye),l(N),l(Ne),l(Fe),l(Ee),l(Xe),l(k),l(ke),l(xe),l(z),l(ze),l(Se),l(Qe),l(V),l(Ve),l(_e),l(H),l(He),l(Le),l(D),l(De),l(P),l(Pe),l(Ke),l(O),l(Oe),l(et),l(te),l(tt),l(lt),l(se),l(st),l(at),l(nt),l(it),l(Me),l(Mt),l(Jt),l(pt),l(mt),l(Tt),l(oe),l(ot),l(ct),l(we)),l(f),T(I,e),T(U,e),T(j,e),T(C,e),T(h,e),T(Z,e),T($,e),T(G,e),T(R,e),T(Y,e),T(F,e),T(E,e),T(X,e),T(x,e),T(S,e),T(Q,e),T(_,e),T(L,e),T(K,e),T(ee,e),T(le,e),T(ae,e),T(ne,e),T(ie,e),T(Je,e),T(pe,e),T(me,e),T(Te,e),T(ce,e)}}}const Lt='{"title":"fenic","local":"fenic","sections":[{"title":"Getting Started","local":"getting-started","sections":[{"title":"Create a Session","local":"create-a-session","sections":[],"depth":3}],"depth":2},{"title":"Overview","local":"overview","sections":[],"depth":2},{"title":"Read from Hugging Face Hub","local":"read-from-hugging-face-hub","sections":[{"title":"Supported Formats","local":"supported-formats","sections":[],"depth":3},{"title":"Reading Datasets","local":"reading-datasets","sections":[],"depth":3},{"title":"Reading with Schema Management","local":"reading-with-schema-management","sections":[],"depth":3},{"title":"Authentication","local":"authentication","sections":[],"depth":3},{"title":"Path Format","local":"path-format","sections":[],"depth":3},{"title":"Mixing Data Sources","local":"mixing-data-sources","sections":[],"depth":3}],"depth":2},{"title":"Processing Data from Hugging Face","local":"processing-data-from-hugging-face","sections":[{"title":"Basic DataFrame Operations","local":"basic-dataframe-operations","sections":[],"depth":3},{"title":"AI-Powered Operations","local":"ai-powered-operations","sections":[],"depth":3}],"depth":2},{"title":"Example: Analyzing MMLU Dataset","local":"example-analyzing-mmlu-dataset","sections":[],"depth":2},{"title":"Resources","local":"resources","sections":[],"depth":2}],"depth":1}';function Dt(rt){return kt(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class ll extends xt{constructor(f){super(),zt(this,f,Dt,Ht,Et,{})}}export{ll as component};
