Update ingest.py

ingest.py

@@ -1,37 +1,32 @@
-from langchain.document_loaders import PyPDFLoader
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-from langchain.embeddings import SentenceTransformerEmbeddings
-from langchain.vectorstores import Chroma
-import base64
-import io
+from langchain.document_loaders import PyPDFLoader, DirectoryLoader, PDFMinerLoader 
+from langchain.text_splitter import RecursiveCharacterTextSplitter 
+from langchain.embeddings import SentenceTransformerEmbeddings 
+from langchain.vectorstores import Chroma 
+import os 
+from constants import CHROMA_SETTINGS
 
 persist_directory = "db"
 
-def process_pdf_content(pdf_content):
-    # Process the PDF content here and generate a brief summary.
-    # You can use libraries like PyPDF2, pdfminer, or other PDF processing tools.
-
-    # For now, let's assume we have extracted the text from the PDF.
-    pdf_text = "This is a brief summary of the PDF content."
-
-    return pdf_text
-
-def create_embeddings(pdf_text):
-    print("Loading Sentence Transformers model")
+def main():
+    for root, dirs, files in os.walk("docs"):
+        for file in files:
+            if file.endswith(".pdf"):
+                print(file)
+                loader = PyPDFLoader(os.path.join(root, file))
+    documents = loader.load()
+    print("splitting into chunks")
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100)
+    texts = text_splitter.split_documents(documents)
+    #create embeddings here
+    print("Loading sentence transformers model")
     embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")
-    print("Creating embeddings. This may take some time...")
-    db = Chroma.from_documents([pdf_text], embeddings, persist_directory=persist_directory)
+    #create vector store here
+    print(f"Creating embeddings. May take some minutes...")
+    db = Chroma.from_documents(texts, embeddings, persist_directory=persist_directory, client_settings=CHROMA_SETTINGS)
     db.persist()
-    print("Embeddings creation complete!")
+    db=None 
 
-def main(uploaded_file):
-    if uploaded_file is not None:
-        with io.BytesIO(uploaded_file.read()) as pdf_buffer:
-            pdf_content = pdf_buffer.read()
-            pdf_summary = process_pdf_content(pdf_content)
-            create_embeddings(pdf_summary)
+    print(f"Ingestion complete! You can now run privateGPT.py to query your documents")
 
 if __name__ == "__main__":
-    # Replace None with the uploaded PDF file
-    uploaded_pdf_file = None  # Replace with the actual uploaded PDF file
-    main(uploaded_pdf_file)
+    main()