Upload folder using huggingface_hub

.gitattributes

@@ -1,35 +1,36 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+faiss_index_hnsw_new/index.faiss filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,14 +1,14 @@
----
-title: Aileeao
-emoji: 🏢
-colorFrom: purple
-colorTo: pink
-sdk: gradio
-sdk_version: 5.20.1
-app_file: app.py
-pinned: false
-license: mit
-short_description: ai李敖
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: Aileeao
+emoji: 🏢
+colorFrom: purple
+colorTo: pink
+sdk: gradio
+sdk_version: 5.20.1
+app_file: app.py
+pinned: false
+license: mit
+short_description: ai李敖
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,421 @@
+import os
+import gradio as gr
+from langchain_community.document_loaders import TextLoader, DirectoryLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import FAISS
+from langchain_openai import ChatOpenAI
+from langchain.prompts import PromptTemplate
+import requests
+import numpy as np
+import json
+import faiss
+from collections import deque
+from langchain_core.embeddings import Embeddings
+import threading
+import queue
+from langchain_core.messages import HumanMessage, AIMessage
+from sentence_transformers import SentenceTransformer
+import pickle
+import torch
+from langchain_core.documents import Document
+
+# 全局停止标志和输出队列
+stop_flag = threading.Event()
+output_queue = queue.Queue()
+
+# 自定义 SentenceTransformers 嵌入类
+class SentenceTransformerEmbeddings(Embeddings):
+    def __init__(self, model_name="BAAI/bge-m3"):
+        self.model = SentenceTransformer(model_name)
+        self.batch_size = 64
+
+    def embed_documents(self, texts):
+        embeddings_file = "embeddings_temp.npy"
+        total_chunks = len(texts)
+        embeddings_shape = (total_chunks, 1024)
+        
+        embeddings_array = np.memmap(embeddings_file, dtype='float32', mode='w+', shape=embeddings_shape)
+        with torch.cuda.amp.autocast():
+            for i in range(0, total_chunks, 1000):
+                batch = texts[i:i+1000]
+                batch_emb = self.model.encode(
+                    batch, 
+                    normalize_embeddings=True, 
+                    batch_size=self.batch_size,
+                    show_progress_bar=False
+                )
+                embeddings_array[i:i+len(batch)] = batch_emb
+                if (i + len(batch)) % 100 == 0:
+                    print(f"嵌入进度: {i+len(batch)} / {total_chunks}")
+                torch.cuda.empty_cache()
+        embeddings_array.flush()
+        return np.array(embeddings_array)
+
+    def embed_query(self, text):
+        with torch.cuda.amp.autocast():
+            return self.model.encode([text], normalize_embeddings=True, batch_size=1)[0]
+
+# SiliconFlow 重排序函数（保持不变）
+def rerank_documents(query, documents, api_key, top_n=10):
+    url = "https://api.siliconflow.cn/v1/rerank"
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json"
+    }
+    doc_texts = [doc.page_content for doc in documents]
+    payload = {
+        "model": "BAAI/bge-reranker-v2-m3",
+        "query": query,
+        "documents": doc_texts,
+        "top_n": top_n
+    }
+    response = requests.post(url, headers=headers, data=json.dumps(payload), timeout=30)
+    if response.status_code == 200:
+        result = response.json()
+        reranked_results = result.get("results", [])
+        if not reranked_results:
+            raise Exception("重排序结果为空")
+        reranked_docs_with_scores = [
+            (documents[res["index"]], res["relevance_score"])
+            for res in reranked_results
+        ]
+        return reranked_docs_with_scores
+    else:
+        raise Exception(f"重排序失败: {response.status_code}, {response.text}")
+
+# 设置 API Keys
+os.environ["SILICONFLOW_API_KEY"] = os.getenv("SILICONFLOW_API_KEY", "sk-cigytzyzghoziznvniugfihuicjcgmborusgodktydremtvd")
+os.environ["OPENROUTER_API_KEY"] = os.getenv("OPENROUTER_API_KEY", "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")
+
+# 初始化嵌入模型
+embeddings = SentenceTransformerEmbeddings(model_name="BAAI/bge-m3")
+
+# 构建 HNSW 索引
+def build_hnsw_index(knowledge_base_path, index_path):
+    print("开始加载文档...")
+    loader = DirectoryLoader(
+        knowledge_base_path,
+        glob="*.txt",
+        loader_cls=lambda path: TextLoader(path, encoding="utf-8"),
+        use_multithreading=True
+    )
+    documents = loader.load()
+    print(f"加载完成，共 {len(documents)} 个文档")
+    
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+    if not os.path.exists("chunks.pkl"):
+        print("开始分片...")
+        docs = text_splitter.split_documents(documents)
+        texts = [doc.page_content for doc in docs]
+        with open("chunks.pkl", "wb") as f:
+            pickle.dump(texts, f)
+        print(f"分片完成，共 {len(texts)} 个 chunk")
+    else:
+        with open("chunks.pkl", "rb") as f:
+            texts = pickle.load(f)
+        print(f"加载已有分片，共 {len(texts)} 个 chunk")
+    
+    embeddings_file = "embeddings_temp.npy"
+    if os.path.exists(embeddings_file):
+        os.remove(embeddings_file)
+    
+    if not os.path.exists("embeddings.npy"):
+        print("开始生成嵌入...")
+        embeddings_array = embeddings.embed_documents(texts)
+        np.save("embeddings.npy", embeddings_array)
+        if os.path.exists(embeddings_file):
+            os.remove(embeddings_file)
+        print(f"嵌入生成完成，维度: {embeddings_array.shape}")
+    else:
+        embeddings_array = np.load("embeddings.npy")
+        print(f"加载已有嵌入，维度: {embeddings_array.shape}")
+    
+    dimension = embeddings_array.shape[1]
+    index = faiss.IndexHNSWFlat(dimension, 16)
+    index.hnsw.efConstruction = 100
+    print("开始构建 HNSW 索引...")
+    
+    batch_size = 5000
+    total_vectors = embeddings_array.shape[0]
+    for i in range(0, total_vectors, batch_size):
+        batch = embeddings_array[i:i + batch_size]
+        index.add(batch)
+        print(f"索引构建进度: {min(i + batch_size, total_vectors)} / {total_vectors}")
+    
+    print("开始构造 FAISS 对象...")
+    # 使用 FAISS.from_texts 初始化基础结构，避免重复嵌入
+    dummy_texts = [texts[0]]  # 用一个样本初始化，避免嵌入所有文本
+    vector_store = FAISS.from_texts(dummy_texts, embeddings)
+    # 替换索引和文档存储
+    vector_store.index = index
+    vector_store.docstore._dict.clear()  # 清空默认的 docstore
+    vector_store.index_to_docstore_id.clear()  # 清空默认映射
+    
+    # 手动填充文档存储
+    for i, text in enumerate(texts):
+        doc_id = str(i)
+        vector_store.docstore._dict[doc_id] = Document(page_content=text)
+        vector_store.index_to_docstore_id[i] = doc_id
+    
+    print(f"构造后 vector_store 类型: {type(vector_store)}")
+    
+    print("开始保存索引...")
+    vector_store.save_local(index_path)
+    print(f"HNSW 索引已生成并保存到 '{index_path}'")
+    
+    # 验证保存结果
+    loaded_store = FAISS.load_local(index_path, embeddings=embeddings, allow_dangerous_deserialization=True)
+    print(f"加载后 vector_store 类型: {type(loaded_store)}")
+    return loaded_store
+
+# 将已有 faiss_index 转为 HNSW
+def convert_to_hnsw(existing_index_path, new_index_path):
+    old_vector_store = FAISS.load_local(existing_index_path, embeddings=embeddings, allow_dangerous_deserialization=True)
+    doc_texts = [doc.page_content for doc in old_vector_store.docstore._dict.values()]
+    embeddings_array = embeddings.embed_documents(doc_texts)
+    dimension = embeddings_array.shape[1]
+    index = faiss.IndexHNSWFlat(dimension, 8)
+    index.hnsw.efConstruction = 40
+    
+    batch_size = 5000
+    total_vectors = embeddings_array.shape[0]
+    for i in range(0, total_vectors, batch_size):
+        batch = embeddings_array[i:i + batch_size]
+        index.add(batch)
+        print(f"索引转换进度: {min(i + batch_size, total_vectors)} / {total_vectors}")
+    
+    print("开始构造 FAISS 对象...")
+    dummy_texts = [doc_texts[0]]
+    new_vector_store = FAISS.from_texts(dummy_texts, embeddings)
+    new_vector_store.index = index
+    new_vector_store.docstore._dict.clear()
+    new_vector_store.index_to_docstore_id.clear()
+    
+    for i, text in enumerate(doc_texts):
+        doc_id = str(i)
+        new_vector_store.docstore._dict[doc_id] = Document(page_content=text)
+        new_vector_store.index_to_docstore_id[i] = doc_id
+    
+    print(f"构造后 vector_store 类型: {type(new_vector_store)}")
+    
+    print("开始保存索引...")
+    new_vector_store.save_local(new_index_path)
+    print(f"已将 '{existing_index_path}' 转换为 HNSW 并保存到 '{new_index_path}'")
+    
+    loaded_store = FAISS.load_local(new_index_path, embeddings=embeddings, allow_dangerous_deserialization=True)
+    print(f"加载后 vector_store 类型: {type(loaded_store)}")
+    return loaded_store
+
+# 加载或生成索引
+index_path = "faiss_index_hnsw_new"
+knowledge_base_path = "knowledge_base"
+
+if not os.path.exists(index_path):
+    if os.path.exists("faiss_index"):
+        print("检测到已有 faiss_index，正在转换为 HNSW...")
+        vector_store = convert_to_hnsw("faiss_index", index_path)
+    elif os.path.exists(knowledge_base_path):
+        print("检测到 knowledge_base，正在生成 HNSW 索引...")
+        vector_store = build_hnsw_index(knowledge_base_path, index_path)
+    else:
+        raise FileNotFoundError("未找到 'faiss_index' 或 'knowledge_base'，请提供知识库数据")
+else:
+    vector_store = FAISS.load_local(index_path, embeddings=embeddings, allow_dangerous_deserialization=True)
+    vector_store.index.hnsw.efSearch = 300
+    print("已加载 HNSW 索引 'faiss_index_hnsw_new'，efSearch 设置为 300")
+    print(f"加载后 vector_store 类型: {type(vector_store)}")
+
+# 初始化 ChatOpenAI
+llm = ChatOpenAI(
+    model="deepseek/deepseek-r1:free",
+    api_key=os.environ["OPENROUTER_API_KEY"],
+    base_url="https://openrouter.ai/api/v1",
+    timeout=60,
+    temperature=0.3,
+    max_tokens=88888,
+    streaming=True
+)
+
+# 定义提示词模板（保持不变）
+prompt_template = PromptTemplate(
+    input_variables=["context", "question", "chat_history"],
+    template="""  
+    你是一个研究李敖的专家，根据用户提出的问题{question}、最近10轮对话历史{chat_history}以及从李敖相关书籍和评论中检索的内容{context}回答问题。  
+    在回答时，请注意以下几点：  
+    - 结合李敖的写作风格和思想，筛选出与问题和对话历史最相关的检索内容，避免无关信息。  
+    - 如果问题涉及李敖对某人或某事的评价，优先引用李敖的直接言论或文字，并说明出处。  
+    - 回答应结构化、分段落，确保逻辑清晰，语言生动，类似李敖的犀利风格。  
+    - 如果检索内容和历史不足以直接回答问题，可根据李敖的性格和观点推测其可能的看法，但需说明这是推测。  
+    - 列出引用的书籍或文章名称及章节（如有），如《李敖大全集》第X卷或具体书名。  
+    - 只能基于提供的知识库内容{context}和对话历史{chat_history}回答，不得引入外部信息。  
+    - 对于列举类问题，控制在10个要点以内，并优先提供最相关项。  
+    - 如果回答较长，结构化分段总结，分点作答控制在5个点以内。  
+    - 根据对话历史调整回答，避免重复或矛盾。  
+    """
+)
+
+# 对话历史管理（保持不变）
+class ConversationHistory:
+    def __init__(self, max_length=10):
+        self.history = deque(maxlen=max_length)
+
+    def add_turn(self, question, answer):
+        self.history.append((question, answer))
+
+    def get_history(self):
+        return [(turn[0], turn[1]) for turn in self.history]
+
+    def clear(self):
+        self.history.clear()
+
+conversation = ConversationHistory()
+
+# 计算余弦相似度函数（保持不变）
+def compute_cosine_similarity(query_embedding, doc_embeddings):
+    query_embedding = np.array(query_embedding)
+    doc_embeddings = np.array(doc_embeddings)
+    dot_product = np.dot(doc_embeddings, query_embedding)
+    query_norm = np.linalg.norm(query_embedding)
+    doc_norms = np.linalg.norm(doc_embeddings, axis=1)
+    similarities = dot_product / (query_norm * doc_norms + 1e-8)
+    return similarities
+
+# 生成回答的线程函数
+def generate_answer_thread(question, output_queue):
+    global stop_flag
+    stop_flag.clear()
+    try:
+        print(f"vector_store 类型: {type(vector_store)}")  # 调试
+        history_list = conversation.get_history()
+        history_text = "\n".join([f"问: {q}\n答: {a}" for q, a in history_list]) if history_list else ""
+        query_with_context = f"{history_text}\n当前问题: {question}" if history_text else question
+        initial_docs_with_scores = vector_store.similarity_search_with_score(query_with_context, k=50)
+        print(f"初始检索数量: {len(initial_docs_with_scores)}")
+        output_queue.put(f"初始检索数量: {len(initial_docs_with_scores)}\n")
+        
+        if stop_flag.is_set():
+            output_queue.put("生成已停止")
+            return
+        
+        query_embedding = embeddings.embed_query(query_with_context)
+        doc_embeddings = [embeddings.embed_query(doc.page_content) for doc, _ in initial_docs_with_scores]
+        similarities = compute_cosine_similarity(query_embedding, doc_embeddings)
+        print(f"余弦相似度范围: {min(similarities):.4f} - {max(similarities):.4f}")
+        output_queue.put(f"余弦相似度范围: {min(similarities):.4f} - {max(similarities):.4f}\n")
+        
+        if stop_flag.is_set():
+            output_queue.put("生成已停止")
+            return
+        
+        similarity_threshold = max(similarities) * 0.8
+        filtered_docs_with_scores = [
+            (doc, sim)
+            for (doc, _), sim in zip(initial_docs_with_scores, similarities)
+            if sim >= similarity_threshold
+        ]
+        if len(filtered_docs_with_scores) < 5:
+            filtered_docs_with_scores = [(doc, sim) for (doc, _), sim in zip(initial_docs_with_scores[:10], similarities[:10])]
+            print(f"过滤后数量不足，保留前 10 个文档")
+            output_queue.put("过滤后数量不足，保留前 10 个文档\n")
+        else:
+            print(f"过滤后数量: {len(filtered_docs_with_scores)}")
+            output_queue.put(f"过滤后数量: {len(filtered_docs_with_scores)}\n")
+
+        if stop_flag.is_set():
+            output_queue.put("生成已停止")
+            return
+
+        initial_docs = [doc for doc, _ in filtered_docs_with_scores]
+        vector_similarities = [sim for _, sim in filtered_docs_with_scores]
+        reranked_docs_with_scores = rerank_documents(query_with_context, initial_docs, os.environ["SILICONFLOW_API_KEY"], top_n=10)
+        reranked_docs = [doc for doc, score in reranked_docs_with_scores]
+        rerank_scores = [score for _, score in reranked_docs_with_scores]
+
+        if stop_flag.is_set():
+            output_queue.put("生成已停止")
+            return
+
+        combined_scores = [
+            0.2 * vector_similarities[i] + 0.8 * rerank_scores[i]
+            for i in range(len(reranked_docs))
+        ]
+        sorted_docs_with_scores = sorted(
+            zip(reranked_docs, combined_scores),
+            key=lambda x: x[1],
+            reverse=True
+        )
+        final_docs = [doc for doc, _ in sorted_docs_with_scores][:5]
+
+        if stop_flag.is_set():
+            output_queue.put("生成已停止")
+            return
+
+        context = "\n\n".join([doc.page_content for doc in final_docs])
+        chat_history = [HumanMessage(content=q) if i % 2 == 0 else AIMessage(content=a) 
+                        for i, (q, a) in enumerate(history_list)]
+        prompt = prompt_template.format(context=context, question=question, chat_history=history_text)
+
+        answer = ""
+        for chunk in llm.stream([HumanMessage(content=prompt)]):
+            if stop_flag.is_set():
+                output_queue.put(answer + "\n\n(生成已停止)")
+                return
+            answer += chunk.content
+            output_queue.put(answer)
+        
+        conversation.add_turn(question, answer)
+        output_queue.put(answer)
+
+    except Exception as e:
+        output_queue.put(f"Error: {str(e)}")
+
+# Gradio 接口函数（保持不变）
+def answer_question(question):
+    global stop_flag, output_queue
+    stop_flag.clear()
+    output_queue.queue.clear()
+    
+    thread = threading.Thread(target=generate_answer_thread, args=(question, output_queue))
+    thread.start()
+    
+    while thread.is_alive() or not output_queue.empty():
+        try:
+            output = output_queue.get(timeout=0.1)
+            yield output
+        except queue.Empty:
+            continue
+    
+    while not output_queue.empty():
+        yield output_queue.get()
+
+def stop_generation():
+    global stop_flag
+    stop_flag.set()
+    return "生成已停止，正在中止..."
+
+def clear_conversation():
+    conversation.clear()
+    return "对话历史已清空，请开始新的对话。"
+
+# 创建 Gradio 界面（保持不变）
+with gr.Blocks(title="AI李敖助手") as interface:
+    gr.Markdown("### AI李敖助手")
+    gr.Markdown("基于李敖163本相关书籍构建的知识库，支持上下文关联，记住最近10轮对话，输入问题以获取李敖风格的回答。")
+    
+    with gr.Row():
+        with gr.Column(scale=3):
+            question_input = gr.Textbox(label="请输入您的问题", placeholder="输入您的问题...")
+            submit_button = gr.Button("提交")
+        with gr.Column(scale=1):
+            clear_button = gr.Button("新建对话")
+            stop_button = gr.Button("停止生成")
+    
+    output_text = gr.Textbox(label="回答", interactive=False)
+    
+    submit_button.click(fn=answer_question, inputs=question_input, outputs=output_text)
+    clear_button.click(fn=clear_conversation, inputs=None, outputs=output_text)
+    stop_button.click(fn=stop_generation, inputs=None, outputs=output_text)
+
+# 启动应用
+if __name__ == "__main__":
+    interface.launch(share=True)

test.py

@@ -0,0 +1,274 @@
+<<<<<<< HEAD
+import torch
+print(torch.__version__)  # 如 2.4.0+cu118
+print(torch.cuda.is_available())  # 应返回 True
+print(torch.cuda.get_device_name(0))  # 应返回 GPU 型号
+=======
+import os
+import gradio as gr
+from langchain_community.document_loaders import TextLoader, DirectoryLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.vectorstores import FAISS
+from langchain_openai import ChatOpenAI
+from langchain.chains import RetrievalQA
+from langchain_core.embeddings import Embeddings
+from langchain.prompts import PromptTemplate
+import requests
+import numpy as np
+import json
+import faiss
+from langchain_community.embeddings import OllamaEmbeddings
+
+# 自定义 SiliconFlow 嵌入类
+class SiliconFlowEmbeddings(Embeddings):
+    def __init__(self, model="BAAI/bge-m3", api_key=None):
+        self.model = model
+        self.api_key = api_key
+
+    def embed_documents(self, texts):
+        return self._get_embeddings(texts)
+
+    def embed_query(self, text):
+        return self._get_embeddings([text])[0]
+
+    def _get_embeddings(self, texts):
+        url = "https://api.siliconflow.cn/v1/embeddings"
+        headers = {
+            "Authorization": f"Bearer {self.api_key}",
+            "Content-Type": "application/json"
+        }
+        payload = {
+            "model": self.model,
+            "input": texts
+        }
+        response = requests.post(url, json=payload, headers=headers, timeout=30)
+        if response.status_code == 200:
+            data = response.json()
+            return np.array([item["embedding"] for item in data["data"]])
+        else:
+            raise Exception(f"API 调用失败: {response.status_code}, {response.text}")
+
+# SiliconFlow 重排序函数
+def rerank_documents(query, documents, api_key, top_n=10):
+    url = "https://api.siliconflow.cn/v1/rerank"
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json"
+    }
+    doc_texts = [doc.page_content for doc in documents]
+    payload = {
+        "model": "BAAI/bge-reranker-v2-m3",
+        "query": query,
+        "documents": doc_texts,
+        "top_n": top_n
+    }
+    response = requests.post(url, headers=headers, data=json.dumps(payload), timeout=30)
+    if response.status_code == 200:
+        result = response.json()
+        reranked_results = result.get("results", [])
+        if not reranked_results:
+            raise Exception("重排序结果为空")
+        reranked_docs_with_scores = [
+            (documents[res["index"]], res["relevance_score"])
+            for res in reranked_results
+        ]
+        return reranked_docs_with_scores
+    else:
+        raise Exception(f"重排序失败: {response.status_code}, {response.text}")
+
+# 设置 API Keys
+os.environ["SILICONFLOW_API_KEY"] = os.getenv("SILICONFLOW_API_KEY", "sk-cigytzyzghoziznvniugfihuicjcgmborusgodktydremtvd")
+os.environ["OPENROUTER_API_KEY"] = os.getenv("OPENROUTER_API_KEY", "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")
+
+# 初始化嵌入模型
+embeddings = OllamaEmbeddings(model="bge-m3", base_url="http://localhost:11434")
+
+# 从 knowledge_base 生成 HNSW 索引
+def build_hnsw_index(knowledge_base_path, index_path):
+    loader = DirectoryLoader(
+        knowledge_base_path,
+        glob="*.txt",
+        loader_cls=lambda path: TextLoader(path, encoding="utf-8")
+    )
+    documents = loader.load()
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)
+    texts = text_splitter.split_documents(documents)
+    
+    # 使用 FAISS.from_documents 创建向量存储
+    vector_store = FAISS.from_documents(texts, embeddings)
+    
+    # 获取嵌入并转换为 HNSW
+    embeddings_array = np.array(embeddings.embed_documents([doc.page_content for doc in texts]))
+    dimension = embeddings_array.shape[1]
+    index = faiss.IndexHNSWFlat(dimension, 16)  # M=16
+    index.hnsw.efConstruction = 100
+    index.hnsw.efSearch = 50
+    index.add(embeddings_array)
+    
+    # 更新 FAISS 的索引
+    vector_store.index = index
+    vector_store.save_local(index_path)
+    print(f"HNSW 索引已生成并保存到 '{index_path}'")
+    return vector_store
+
+# 将已有 faiss_index 转为 HNSW
+def convert_to_hnsw(existing_index_path, new_index_path):
+    # 加载现有索引
+    old_vector_store = FAISS.load_local(existing_index_path, embeddings=embeddings, allow_dangerous_deserialization=True)
+    
+    # 获取文档内容
+    if hasattr(old_vector_store, 'docstore') and hasattr(old_vector_store.docstore, '_dict'):
+        docs = list(old_vector_store.docstore._dict.values())
+        doc_texts = [doc.page_content if hasattr(doc, 'page_content') else str(doc) for doc in docs]
+    else:
+        doc_ids = list(old_vector_store.index_to_docstore_id.keys())
+        doc_texts = [old_vector_store.docstore._dict[old_vector_store.index_to_docstore_id[i]].page_content 
+                     if hasattr(old_vector_store.docstore._dict[old_vector_store.index_to_docstore_id[i]], 'page_content') 
+                     else str(old_vector_store.docstore._dict[old_vector_store.index_to_docstore_id[i]]) 
+                     for i in doc_ids]
+    
+    # 使用全局 embeddings 对象生成嵌入
+    embeddings_array = np.array(embeddings.embed_documents(doc_texts))
+    
+    # 创建 HNSW 索引
+    dimension = embeddings_array.shape[1]
+    index = faiss.IndexHNSWFlat(dimension, 16)  # M=16
+    index.hnsw.efConstruction = 100
+    index.hnsw.efSearch = 50
+    index.add(embeddings_array)
+    
+    # 创建新的 FAISS 向量存储，注意不直接传递 index，而是稍后赋值
+    new_vector_store = FAISS.from_texts(doc_texts, embeddings)
+    new_vector_store.index = index  # 直接替换索引
+    new_vector_store.save_local(new_index_path)
+    print(f"已将 '{existing_index_path}' 转换为 HNSW 并保存到 '{new_index_path}'")
+    return new_vector_store
+
+# 加载或生成索引
+index_path = "faiss_index_hnsw"
+knowledge_base_path = "knowledge_base"
+
+if not os.path.exists(index_path):
+    if os.path.exists("faiss_index"):
+        print("检测到已有 faiss_index，正在转换为 HNSW...")
+        vector_store = convert_to_hnsw("faiss_index", index_path)
+    elif os.path.exists(knowledge_base_path):
+        print("检测到 knowledge_base，正在生成 HNSW 索引...")
+        vector_store = build_hnsw_index(knowledge_base_path, index_path)
+    else:
+        raise FileNotFoundError("未找到 'faiss_index' 或 'knowledge_base'，请提供知识库数据")
+else:
+    vector_store = FAISS.load_local(index_path, embeddings=embeddings, allow_dangerous_deserialization=True)
+    print("已加载 HNSW 索引 'faiss_index_hnsw'")
+
+# 初始化 ChatOpenAI 使用 OpenRouter
+llm = ChatOpenAI(
+    model="deepseek/deepseek-r1:free",
+    api_key=os.environ["OPENROUTER_API_KEY"],
+    base_url="https://openrouter.ai/api/v1",
+    timeout=60,
+    temperature=0.3,
+    max_tokens=88888,
+)
+
+# 定义提示词模板
+prompt_template = PromptTemplate(
+    input_variables=["context", "question"],
+    template="""  
+    你是一个研究李敖的专家，根据用户提出的问题{question}以及从李敖相关书籍和评论中检索的内容{context}回答问题。  
+
+    在回答时，请注意以下几点：  
+    - 结合李敖的写作风格和思想，筛选出与问题最相关的检索内容，避免无关信息。  
+    - 如果问题涉及李敖对某人或某事的评价，优先引用李敖的直接言论或文字，并说明出处。  
+    - 回答应结构化、分段落，确保逻辑清晰，语言生动，类似李敖的犀利风格。  
+    - 如果检索内容不足以直接回答问题，可根据李敖的性格和观点推测其可能的看法，但需说明这是推测。  
+    - 列出引用的书籍或文章名称及章节（如有），如《李敖大全集》第X卷或具体书名。  
+    - 只能基于提供的知识库内容{context}回答，不得引入外部信息。  
+    - 并非搜索结果的所有内容都与用户的问题密切相关，你需要结合问题，对搜索结果进行甄别、筛选。
+    - 对于列举类的问题（如列举所有航班信息），尽量将答案控制在10个要点以内，并告诉用户可以查看搜索来源、获得完整信息。优先提供信息完整、最相关的列举项；如非必要，不要主动告诉用户搜索结果未提供的内容。
+    - 如果回答很长，请尽量结构化、分段落总结。如果需要分点作答，尽量控制在5个点以内，并合并相关的内容。
+    - 对于客观类的问答，如果问题的答案非常简短，可以适当补充一到两句相关信息，以丰富内容。
+    - 你需要根据用户要求和回答内容选择合适、美观的回答格式，确保可读性强。
+    - 你的回答应该综合多个相关知识库内容来回答，不能重复引用一个知识库内容。
+    - 除非用户要求，否则你回答的语言需要和用户提问的语言保持一致。
+    """
+)
+
+# 创建检索问答链
+qa_chain = RetrievalQA.from_chain_type(
+    llm=llm,
+    chain_type="stuff",
+    retriever=vector_store.as_retriever(search_kwargs={"k": 30}),
+    return_source_documents=True,
+    chain_type_kwargs={"prompt": prompt_template}
+)
+
+# 定义 Gradio 接口函数
+def answer_question(question):
+    try:
+        # Step 1: FAISS 初始检索
+        initial_docs_with_scores = vector_store.similarity_search_with_score(question, k=30)
+        print(f"初始检索数量: {len(initial_docs_with_scores)}")
+        
+        # FAISS 返回的是距离，转换为相似度
+        similarities = [1 - score for _, score in initial_docs_with_scores]
+        print(f"相似度范围: {min(similarities):.4f} - {max(similarities):.4f}")
+        
+        # 打印前 5 个文档内容和相似度
+        for i, (doc, score) in enumerate(initial_docs_with_scores[:5]):
+            print(f"Top {i+1} - 相似度: {1 - score:.4f}, 内容: {doc.page_content[:100]}")
+
+        # Step 2: 动态阈值过滤
+        similarity_threshold = max(similarities) * 0.8
+        filtered_docs_with_scores = [
+            (doc, 1 - score)
+            for doc, score in initial_docs_with_scores
+            if (1 - score) >= similarity_threshold
+        ]
+        if len(filtered_docs_with_scores) < 5:
+            filtered_docs_with_scores = initial_docs_with_scores[:10]
+            print(f"过滤后数量不足，保留前 10 个文档")
+        else:
+            print(f"过滤后数量: {len(filtered_docs_with_scores)}")
+
+        initial_docs = [doc for doc, _ in filtered_docs_with_scores]
+        vector_similarities = [sim for _, sim in filtered_docs_with_scores]
+
+        # Step 3: 重排序
+        reranked_docs_with_scores = rerank_documents(question, initial_docs, os.environ["SILICONFLOW_API_KEY"], top_n=10)
+        reranked_docs = [doc for doc, score in reranked_docs_with_scores]
+        rerank_scores = [score for _, score in reranked_docs_with_scores]
+
+        # Step 4: 融合得分并排序
+        combined_scores = [
+            0.2 * vector_similarities[i] + 0.8 * rerank_scores[i]
+            for i in range(len(reranked_docs))
+        ]
+        sorted_docs_with_scores = sorted(
+            zip(reranked_docs, combined_scores),
+            key=lambda x: x[1],
+            reverse=True
+        )
+        final_docs = [doc for doc, _ in sorted_docs_with_scores][:5]
+
+        # Step 5: 生成回答
+        context = "\n\n".join([doc.page_content for doc in final_docs])
+        response = qa_chain.invoke({"query": question, "context": context})
+        
+        return response["result"]
+    except Exception as e:
+        return f"Error: {str(e)}"
+
+# 创建 Gradio 界面
+interface = gr.Interface(
+    fn=answer_question,
+    inputs=gr.Textbox(label="请输入您的问题"),
+    outputs=gr.Textbox(label="回答"),
+    title="AI李敖助手",
+    description="基于李敖163本相关书籍构建的知识库，输入问题以获取李敖风格的回答。"
+)
+
+# 启动应用
+if __name__ == "__main__":
+    interface.launch(share=True)
+>>>>>>> 921dc7e73a28368974490d7eba946303cf2129ba