Update github_repo_analyzer.py

github_repo_analyzer.py

@@ -2,22 +2,25 @@ import os
 import sys
 import tempfile
 import shutil
-from urllib.parse import urlparse, quote
 import requests
-from github import Github
-from git import Repo
-from collections import defaultdict
 import time
-import numpy as np
-from sklearn.feature_extraction.text import TfidfVectorizer
-from sklearn.cluster import KMeans
-from sklearn.metrics.pairwise import cosine_similarity
 import subprocess
 import json
-from pathlib import Path
 import traceback
 import argparse
 import re
+import warnings
+import numpy as np
+
+from collections import defaultdict
+from pathlib import Path
+from urllib.parse import urlparse, quote
+from github import Github
+from git import Repo
+from sklearn.feature_extraction.text import TfidfVectorizer
+from sklearn.cluster import KMeans
+from sklearn.metrics.pairwise import cosine_similarity
+from sklearn.exceptions import ConvergenceWarning
 
 def run_semgrep(repo_path):
     try:
@@ -211,6 +214,12 @@ def parse_llm_response(response):
             return []
 
 def cluster_and_filter_items(items, n_clusters=5, n_items=10):
+    if len(items) == 0:
+        return []
+    
+    if len(items) <= n_items:
+        return items
+
     # Combine title and body for text analysis
     texts = [f"{item['title']} {item['body']}" for item in items]
     
@@ -218,27 +227,40 @@ def cluster_and_filter_items(items, n_clusters=5, n_items=10):
     vectorizer = TfidfVectorizer(stop_words='english', max_features=1000)
     tfidf_matrix = vectorizer.fit_transform(texts)
     
+    # Determine the number of clusters
+    n_clusters = min(n_clusters, len(items))
+    
     # Perform clustering
-    kmeans = KMeans(n_clusters=min(n_clusters, len(items)))
-    kmeans.fit(tfidf_matrix)
+    with warnings.catch_warnings():
+        warnings.filterwarnings("ignore", category=ConvergenceWarning)
+        kmeans = KMeans(n_clusters=n_clusters)
+        kmeans.fit(tfidf_matrix)
     
     # Get cluster centers
     cluster_centers = kmeans.cluster_centers_
     
     # Find items closest to cluster centers
     filtered_items = []
-    for i in range(min(n_clusters, len(items))):
+    for i in range(n_clusters):
         cluster_items = [item for item, label in zip(items, kmeans.labels_) if label == i]
         cluster_vectors = tfidf_matrix[kmeans.labels_ == i]
         
+        if cluster_vectors.shape[0] == 0:
+            continue
+        
         # Calculate similarities to cluster center
         similarities = cosine_similarity(cluster_vectors, cluster_centers[i].reshape(1, -1)).flatten()
         
         # Sort items by similarity and select top ones
         sorted_items = [x for _, x in sorted(zip(similarities, cluster_items), key=lambda pair: pair[0], reverse=True)]
-        filtered_items.extend(sorted_items[:min(n_items // n_clusters, len(sorted_items))])
+        filtered_items.extend(sorted_items[:max(1, n_items // n_clusters)])
+    
+    # If we didn't get enough items, add more from the original list
+    if len(filtered_items) < n_items:
+        remaining_items = [item for item in items if item not in filtered_items]
+        filtered_items.extend(remaining_items[:n_items - len(filtered_items)])
     
-    return filtered_items
+    return filtered_items[:n_items]
 
 def safe_filter_open_items(open_items, closed_patterns, n_items=10):
     try: