ingest.py

import streamlit as st
from langchain.document_loaders import PyPDFLoader
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.embeddings import SentenceTransformerEmbeddings
from langchain.vectorstores import Chroma
from constants import CHROMA_SETTINGS

persist_directory = "db"

def main():
    st.title("PDF Processor")

    uploaded_file = st.file_uploader("Upload a PDF file", type=["pdf"])

    if uploaded_file is not None:
        st.write("Processing PDF...")
        loader = PyPDFLoader(uploaded_file)
        documents = loader.load()

        st.write("Splitting into chunks")
        text_splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=100)
        texts = text_splitter.split_documents(documents)

        st.write("Loading sentence transformers model")
        embeddings = SentenceTransformerEmbeddings(model_name="all-MiniLM-L6-v2")

        st.write("Creating embeddings. This may take some time...")
        db = Chroma.from_documents(texts, embeddings, persist_directory=persist_directory, client_settings=CHROMA_SETTINGS)
        db.persist()
        db = None

        st.success("Ingestion complete! You can now run privateGPT.py to query your documents")

if __name__ == "__main__":
    main()