Create app.py

app.py

@@ -0,0 +1,72 @@
+# AUTOGENERATED! DO NOT EDIT! File to edit: Project_CUDA_Enabled (1).ipynb.
+
+# %% auto 0
+__all__ = ['model_checkpoint', 'model', 'tokenizer', 'dataset', 'train_dataset', 'iface', 'transform', 'cls_pooling',
+           'search_arxiv']
+
+# %% Project_CUDA_Enabled (1).ipynb 52
+import gradio as gr
+from datasets import load_dataset
+import numpy as np
+from transformers import AutoTokenizer, AutoModel
+
+
+model_checkpoint = 'sentence-transformers/all-MiniLM-L6-v2'
+model = AutoModel.from_pretrained(model_checkpoint)
+tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)
+
+
+# Load the dataset from Hugging Face
+dataset = load_dataset('Tarun-1999M/arxiv_cs_lg_embeddings')
+train_dataset = dataset['train']
+
+
+
+# Ensure embeddings are converted to NumPy arrays on-the-fly using set_transform
+def transform(example):
+    example['embeddings'] = np.array(example['embeddings'], dtype=np.float32)
+    return example
+
+
+
+
+def cls_pooling(model_output):
+    return model_output.last_hidden_state[:,0]
+
+train_dataset.set_transform(transform)
+
+# Add FAISS index
+train_dataset.add_faiss_index(column='embeddings')
+
+
+# Function to search the ArXiv papers
+def search_arxiv(query):
+    # Get the embedding for the query
+    question_embedding = get_embeddings([query]).cpu().detach().numpy()
+
+    # Search for similar papers
+    scores, samples = train_dataset.get_nearest_examples("embeddings", question_embedding, k=5)
+
+     # Sort the results by scores in descending order
+    sorted_results = sorted(zip(scores, samples['title'], samples['abstract']), reverse=True)
+
+    # Prepare and format the results for display
+    results = []
+    for score, title, abstract in sorted_results:
+        result = f"\n**Title:** {title}\n**Abstract:** {abstract}\n**Score:** {score:.4f}"
+        results.append(result)
+
+    return "\n\n".join(results)
+
+# Create the Gradio interface
+iface = gr.Interface(
+    fn=search_arxiv,
+    inputs=gr.components.Textbox(lines=1, placeholder="Enter your query..."),
+    outputs="markdown",
+    title="Semantic Search in ArXiv ML Papers",
+    description="Enter a query to find relevant ML papers from the ArXiv dataset."
+)
+
+# Launch the interface
+iface.launch(share=True)
+