import json import numpy as np from metrics.graph_matching import ( get_triple_match_f1, get_graph_match_accuracy, get_bert_score, get_bleu_rouge, split_to_edges, get_tokens, get_ged ) def load_data(gold_path, pred_path): ''' 数据加载处理： 只评估在预测数据中出现的文本对应的三元组 自动匹配真实数据和预测数据中的对应项 多维度评估： Triple Match F1：评估三元组的精确匹配程度 Graph Match Accuracy：评估图结构的匹配程度 BERT Score：评估语义相似度 BLEU & ROUGE：评估文本生成质量 图编辑距离(GED)：评估图结构差异 ''' # 加载真实数据 with open(gold_path, 'r', encoding='utf-8') as f: gold_data = json.load(f) # 加载预测数据 with open(pred_path, 'r', encoding='utf-8') as f: pred_data = json.load(f) # 提取三元组列表 gold_graphs = [] pred_graphs = [] # 确保只评估在预测数据中出现的文本对应的三元组 for pred_item in pred_data: pred_text = pred_item['text'] # 在gold_data中找到对应的文本 for gold_item in gold_data: if gold_item['text'] == pred_text: gold_graphs.append(gold_item['triple_list']) pred_graphs.append(pred_item['triple_list']) break return gold_graphs, pred_graphs def evaluate_triples(gold_graphs, pred_graphs): print("开始评估...") print("="*50) # 1. Triple Match F1 precision, recall, f1 = get_triple_match_f1(gold_graphs, pred_graphs) print("Triple Match") print(f"精确率: {precision:.4f}, 召回率: {recall:.4f}, F1: {f1:.4f}") # # 2. Graph Match Accuracy # graph_acc = get_graph_match_accuracy(pred_graphs, gold_graphs) # print(f"图匹配准确率: {graph_acc:.10f}") # 3. BERT Score gold_edges = split_to_edges(gold_graphs) pred_edges = split_to_edges(pred_graphs) precisions_BS, recalls_BS, f1s_BS = get_bert_score(gold_edges, pred_edges) print(f"BERT Score:") print(f"- Precision: {precisions_BS.mean():.4f}") print(f"- Recall: {recalls_BS.mean():.4f}") print(f"- F1: {f1s_BS.mean():.4f}") # # 4. BLEU & ROUGE # gold_tokens, pred_tokens = get_tokens(gold_edges, pred_edges) # p_rouge, r_rouge, f1_rouge, p_bleu, r_bleu, f1_bleu = get_bleu_rouge( # gold_tokens, pred_tokens, gold_edges, pred_edges # ) # print(f"\nBLEU分数:") # print(f"- Precision: {p_bleu.mean():.4f}") # print(f"- Recall: {r_bleu.mean():.4f}") # print(f"- F1: {f1_bleu.mean():.4f}") # print(f"\nROUGE分数:") # print(f"- Precision: {p_rouge.mean():.4f}") # print(f"- Recall: {r_rouge.mean():.4f}") # print(f"- F1: {f1_rouge.mean():.4f}") # # 5. 图编辑距离(GED) # total_ged = 0 # for gold, pred in zip(gold_graphs, pred_graphs): # ged = get_ged(gold, pred) # total_ged += ged # avg_ged = total_ged / len(gold_graphs) # print(f"\n平均图编辑距离: {avg_ged:.4f}") # 返回所有指标 return { 'triple_match': { 'precision': precision, 'recall': recall, 'f1': f1 }, # 'graph_acc': graph_acc, 'bert_score': { 'precision': precisions_BS.mean(), 'recall': recalls_BS.mean(), 'f1': f1s_BS.mean() }, # 'bleu': { # 'precision': p_bleu.mean(), # 'recall': r_bleu.mean(), # 'f1': f1_bleu.mean() # }, # 'rouge': { # 'precision': p_rouge.mean(), # 'recall': r_rouge.mean(), # 'f1': f1_rouge.mean() # }, # 'ged': avg_ged } if __name__ == '__main__': import pandas as pd # # 设置文件路径 # gold_path = './data/train_triples.json' # pred_path = './output/gpt.json' # # 加载数据 # gold_graphs, pred_graphs = load_data(gold_path, pred_path) # # 评估并打印结果 # results = evaluate_triples(gold_graphs, pred_graphs) # 加载地质描述文本，提取prompt和label with open('./data/train_triples.json', 'r', encoding='utf-8') as f: data = json.load(f) # 将data转换为DataFrame df = pd.DataFrame(data) # 提取prompt和label text = df['text'] label = df['triple_list'] # 设置文件路径 gold_path = './data/GT_500.json' model_paths = [ # # gpt-3.5 # 'F:/GeoLLM/output/output_result/Task1/nomal/zero_shot/old/gpt-3.5-turbo.json', # 零样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/one_shot/gpt-3p5-turbo.json', # 单样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/two_shot/gpt-3p5-turbo.json', # 双样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/three_shot/gpt-3p5-turbo.json', # 三样本 # 'F:/GeoLLM/output/output_result/Task1/knn/one_shot/gpt-3p5-turbo.json', # KNN单样本 # 'F:/GeoLLM/output/output_result/Task1/knn/two_shot/gpt-3p5-turbo.json', # KNN双样本 # 'F:/GeoLLM/output/output_result/Task1/knn/three_shot/gpt-3p5-turbo.json', # KNN三样本 # 'F:/GeoLLM/output/output_result/Task1/Knowledge-guided/one_shot/gpt-3p5-turbo.json', # 知识引导单样本 # # gpt-4o # 'F:/GeoLLM/output/output_result/Task1/nomal/zero_shot/old/gpt-4o.json', # 零样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/one_shot/gpt-4o.json', # 单样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/two_shot/gpt-4o.json', # 双样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/three_shot/gpt-4o.json', # 三样本 # 'F:/GeoLLM/output/output_result/Task1/knn/one_shot/gpt-4o.json', # KNN单样本 # 'F:/GeoLLM/output/output_result/Task1/knn/two_shot/gpt-4o.json', # KNN双样本 # 'F:/GeoLLM/output/output_result/Task1/knn/three_shot/gpt-4o.json', # KNN三样本 # 'F:/GeoLLM/output/output_result/Task1/Knowledge-guided/one_shot/gpt-4o.json', # 知识引导单样本 # # gemini-1p5-pro-002 # 'F:/GeoLLM/output/output_result/Task1/nomal/zero_shot/gemini-1p5-pro-002.json', # 零样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/one_shot/gemini-1p5-pro-002.json', # 单样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/two_shot/gemini-1p5-pro-002.json', # 双样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/three_shot/gemini-1p5-pro-002.json', # 三样本 # 'F:/GeoLLM/output/output_result/Task1/knn/one_shot/gemini-1p5-pro-002.json', # KNN单样本 # 'F:/GeoLLM/output/output_result/Task1/knn/two_shot/gemini-1p5-pro-002.json', # KNN双样本 # 'F:/GeoLLM/output/output_result/Task1/knn/three_shot/gemini-1p5-pro-002.json', # KNN三样本 # 'F:/GeoLLM/output/output_result/Task1/Knowledge-guided/one_shot/gemini-1p5-pro-002.json', # 知识引导单样本 # # claude-3-5-haiku-20241022 # 'F:/GeoLLM/output/output_result/Task1/nomal/zero_shot/claude-3-5-haiku-20241022.json', # 零样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/one_shot/claude-3-5-haiku-20241022.json', # 单样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/two_shot/claude-3-5-haiku-20241022.json', # 双样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/three_shot/claude-3-5-haiku-20241022.json', # 三样本 # 'F:/GeoLLM/output/output_result/Task1/knn/one_shot/claude-3-5-haiku-20241022.json', # KNN单样本 # 'F:/GeoLLM/output/output_result/Task1/knn/two_shot/claude-3-5-haiku-20241022.json', # KNN双样本 # 'F:/GeoLLM/output/output_result/Task1/knn/three_shot/claude-3-5-haiku-20241022.json', # KNN三样本 # 'F:/GeoLLM/output/output_result/Task1/Knowledge-guided/one_shot/claude-3-5-haiku-20241022.json', # 知识引导单样本 # # deepseek-ai # 'F:/GeoLLM/output/output_result/Task1/nomal/zero_shot/deepseek-ai/DeepSeek-V3.json', # 零样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/one_shot/deepseek-ai/DeepSeek-V3.json', # 单样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/two_shot/deepseek-ai/DeepSeek-V3.json', # 双样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/three_shot/deepseek-ai/DeepSeek-V3.json', # 三样本 # 'F:/GeoLLM/output/output_result/Task1/knn/one_shot/deepseek-ai/DeepSeek-V3.json', # KNN单样本 # 'F:/GeoLLM/output/output_result/Task1/knn/two_shot/deepseek-ai/DeepSeek-V3.json', # KNN双样本 # 'F:/GeoLLM/output/output_result/Task1/knn/three_shot/deepseek-ai/DeepSeek-V3.json', # KNN三样本 # 'F:/GeoLLM/output/output_result/Task1/Knowledge-guided/one_shot/deepseek-ai/DeepSeek-V3.json', # 知识引导单样本 # # R1 # 'F:/GeoLLM/output/output_result/Task1/nomal/zero_shot/deepseek-ai/DeepSeek-R1.json', # 零样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/one_shot/deepseek-ai/DeepSeek-R1.json', # 单样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/two_shot/deepseek-ai/DeepSeek-R1.json', # 双样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/three_shot/deepseek-ai/DeepSeek-R1.json', # 三样本 # 'F:/GeoLLM/output/output_result/Task1/knn/one_shot/deepseek-ai/DeepSeek-R1.json', # KNN单样本 # 'F:/GeoLLM/output/output_result/Task1/knn/two_shot/deepseek-ai/DeepSeek-R1.json', # KNN双样本 # 'F:/GeoLLM/output/output_result/Task1/knn/three_shot/deepseek-ai/DeepSeek-R1.json', # KNN三样本 # 'F:/GeoLLM/output/output_result/Task1/Knowledge-guided/one_shot/deepseek-ai/DeepSeek-R1.json', # 知识引导单样本 # # meta-llama # 'F:/GeoLLM/output/output_result/Task1/nomal/zero_shot/meta-llama/Meta-Llama-3p1-405B-Instruct.json', # 零样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/one_shot/meta-llama/Meta-Llama-3p1-405B-Instruct.json', # 单样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/two_shot/meta-llama/Meta-Llama-3p1-405B-Instruct.json', # 双样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/three_shot/meta-llama/Meta-Llama-3p1-405B-Instruct.json', # 三样本 # 'F:/GeoLLM/output/output_result/Task1/knn/one_shot/meta-llama/Meta-Llama-3p1-405B-Instruct.json', # KNN单样本 # 'F:/GeoLLM/output/output_result/Task1/knn/two_shot/meta-llama/Meta-Llama-3p1-405B-Instruct.json', # KNN双样本 # 'F:/GeoLLM/output/output_result/Task1/knn/three_shot/meta-llama/Meta-Llama-3p1-405B-Instruct.json', # KNN三样本 # 'F:/GeoLLM/output/output_result/Task1/Knowledge-guided/one_shot/meta-llama/Meta-Llama-3p1-405B-Instruct.json', # 知识引导单样本 # # Qwen # 'F:/GeoLLM/output/output_result/Task1/nomal/zero_shot/Qwen/Qwen2p5-72B-Instruct.json', # 零样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/one_shot/Qwen/Qwen2p5-72B-Instruct.json', # 单样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/two_shot/Qwen/Qwen2p5-72B-Instruct.json', # 双样本 # 'F:/GeoLLM/output/output_result/Task1/nomal/three_shot/Qwen/Qwen2p5-72B-Instruct.json', # 三样本 # 'F:/GeoLLM/output/output_result/Task1/knn/one_shot/Qwen/Qwen2p5-72B-Instruct.json', # KNN单样本 # 'F:/GeoLLM/output/output_result/Task1/knn/two_shot/Qwen/Qwen2p5-72B-Instruct.json', # KNN双样本 # 'F:/GeoLLM/output/output_result/Task1/knn/three_shot/Qwen/Qwen2p5-72B-Instruct.json', # KNN三样本 # 'F:/GeoLLM/output/output_result/Task1/Knowledge-guided/one_shot/Qwen/Qwen2p5-72B-Instruct.json', # 知识引导单样本 # 'F:/GeoLLM/output/Knowledge-guided_rerun/one_shot/gpt-3.5-turbo_one_shot.json', # 知识引导单样本 # 'F:/GeoLLM/output/Knowledge-guided_rerun/one_shot/gpt-3.5-turbo_0407.json', # 知识引导单样本 # 'F:/GeoLLM/output/Knowledge-guided_rerun/one_shot/gpt-3.5-turbo_old.json', # 知识引导单样本 # 'F:/GeoLLM/output/Knowledge-guided_rerun/one_shot/gpt-4o_konwledge_tri.json', # 知识引导单样本 # 'F:/GeoLLM/output/Knowledge-guided_rerun/one_shot/gemini-1.5-pro-002_one_shot.json', # 知识引导单样本 # 'F:/GeoLLM/output/Knowledge-guided_rerun/one_shot/claude-3-5-haiku-20241022_one_shot.json', # 知识引导单样本 # 'F:/GeoLLM/output/Knowledge-guided_rerun/one_shot/deepseek-ai/DeepSeek-V3_0420.json', # 知识引导单样本 # 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_only_tri/deepseek-ai/DeepSeek-V3.json', # 知识引导单样本 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_all/deepseek-ai/DeepSeek-R1_guide.json', # 知识引导单样本 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_all/deepseek-ai/DeepSeek-R1_one_shot.json', # 知识引导单样本 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_only_tri/deepseek-ai/DeepSeek-R1.json' # 'F:/GeoLLM/output/Knowledge-guided_rerun/one_shot/meta-llama/Meta-Llama-3.1-405B-Instruct_one_shot.json', # 知识引导单样本 # 'F:/GeoLLM/output/Knowledge-guided_rerun/one_shot/Qwen/Qwen2.5-72B-Instruct_one_shot.json', # 知识引导单样本 # 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_all/gpt-3.5-turbo_one_shot.json', # 知识引导单样本 # 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_all/gpt-4o_one_shot.json', # 知识引导单样本 # 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_all/gemini-1.5-pro-002_one_shot.json', # 知识引导单样本 # 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_all/claude-3-5-haiku-20241022_one_shot.json', # 知识引导单样本 # 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_all/deepseek-ai/DeepSeek-V3_one_shot.json', # 知识引导单样本 # # 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_all/deepseek-ai/DeepSeek-R1.json', # 知识引导单样本 # # 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_all/deepseek-ai/DeepSeek-R1_one_shot.json' # 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_all/meta-llama/Meta-Llama-3.1-405B-Instruct_one_shot.json', # 知识引导单样本 # 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_all/Qwen/Qwen2.5-72B-Instruct_one_shot.json', # 知识引导单样本 # 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_all/gpt-3.5-turbo.json', # 知识引导单样本 # 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_all/gpt-4o.json', # 知识引导单样本 # 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_all/gemini-1.5-pro-002.json', # 知识引导单样本 # 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_all/claude-3-5-haiku-20241022.json', # 知识引导单样本 # 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_all/deepseek-ai/DeepSeek-V3.json', # 知识引导单样本 # # 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_all/deepseek-ai/DeepSeek-R1.json', # 知识引导单样本 # # 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_all/deepseek-ai/DeepSeek-R1_one_shot.json' # 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_all/meta-llama/Meta-Llama-3.1-405B-Instruct.json', # 知识引导单样本 # 'F:/GeoLLM/output/Knowledge-guided_rerun/nomal_all/Qwen/Qwen2.5-72B-Instruct.json', # 知识引导单样本 ] # 对比不同模型的表现 print("各模型评估结果:") # 存储所有模型的结果 all_results = {} for pred_path in model_paths: model_name = pred_path.split('/')[-1].split('.')[0] print(f"\n{model_name}模型:") # 加载数据 gold_graphs, pred_graphs = load_data(gold_path, pred_path) # 评估并打印结果 results = evaluate_triples(gold_graphs, pred_graphs) all_results[model_name] = results # 连带model_paths和results一起保存为txt save_path = 'F:/GeoLLM/output/output_result/Task1/Result_Task1.txt' with open(save_path, 'a', encoding='utf-8') as f: f.write(f"{pred_path}: \n") f.write(f"Triple Match: \n") f.write(f"精确率: {results['triple_match']['precision']:.4f}, 召回率: {results['triple_match']['recall']:.4f}, F1: {results['triple_match']['f1']:.4f}\n") f.write(f"BERT Score: \n") f.write(f"- Precision: {results['bert_score']['precision']:.4f}\n") f.write(f"- Recall: {results['bert_score']['recall']:.4f}\n") f.write(f"- F1: {results['bert_score']['f1']:.4f}\n\n")