import gradio as gr import torch import os import sys from PIL import Image import uuid import huggingface_hub sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) from hpsv3.inference import HPSv3RewardInferencer try: import ImageReward as RM from hpsv2.src.open_clip import create_model_and_transforms, get_tokenizer except: RM = None create_model_and_transforms = None get_tokenizer = None print("ImageReward or HPSv2 dependencies not found. Skipping those models.") from transformers import AutoProcessor, AutoModel # --- Configuration --- DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu' DTYPE = torch.bfloat16 if DEVICE == 'cuda' else torch.float32 # --- Model Configuration --- MODEL_CONFIGS = { "HPSv3_7B": { "name": "HPSv3 7B", "type": "hpsv3" }, "HPSv2": { "name": "HPSv2", "checkpoint_path": "xswu/HPSv2/HPS_v2.1_compressed.pt", "type": "hpsv2" }, "ImageReward": { "name": "ImageReward v1.0", "checkpoint_path": "ImageReward-v1.0", "type": "imagereward" }, "PickScore": { "name": "PickScore", "checkpoint_path": "yuvalkirstain/PickScore_v1", "type": "pickscore" }, "CLIP": { "name": "CLIP ViT-H-14", "checkpoint_path": "laion/CLIP-ViT-H-14-laion2B-s32B-b79K", "type": "clip" } } # --- Global Model Storage --- current_models = {} current_model_name = None # --- Dynamic Model Loading Functions --- def load_model(model_key, update_status_fn=None): """Load the specified model based on the model key.""" global current_models, current_model_name if model_key == current_model_name and model_key in current_models: return current_models[model_key] if update_status_fn: update_status_fn(f"🔄 Loading {MODEL_CONFIGS[model_key]['name']}...") # Clear previous models to save memory current_models.clear() torch.cuda.empty_cache() config = MODEL_CONFIGS[model_key] try: if config["type"] == "hpsv3": checkpoint_path = huggingface_hub.hf_hub_download("MizzenAI/HPSv3", 'HPSv3.safetensors', repo_type='model') model = HPSv3RewardInferencer( device=DEVICE, checkpoint_path=checkpoint_path ) elif config["type"] == "hpsv2": model_obj, preprocess_train, preprocess_val = create_model_and_transforms( 'ViT-H-14', 'laion2B-s32B-b79K', precision='amp', device=DEVICE, jit=False, force_quick_gelu=False, force_custom_text=False, force_patch_dropout=False, force_image_size=None, pretrained_image=False, image_mean=None, image_std=None, light_augmentation=True, aug_cfg={}, output_dict=True, with_score_predictor=False, with_region_predictor=False ) checkpoint_path = huggingface_hub.hf_hub_download("xswu/HPSv2", 'HPS_v2.1_compressed.pt', repo_type='model') checkpoint = torch.load(checkpoint_path, map_location=DEVICE, weights_only=False) model_obj.load_state_dict(checkpoint['state_dict']) model_obj = model_obj.to(DEVICE).eval() tokenizer = get_tokenizer('ViT-H-14') model = {"model": model_obj, "preprocess_val": preprocess_val, "tokenizer": tokenizer} elif config["type"] == "imagereward": model = RM.load(config["checkpoint_path"]) elif config["type"] == "pickscore": processor = AutoProcessor.from_pretrained('/preflab/models/CLIP-ViT-H-14-laion2B-s32B-b79K') model_obj = AutoModel.from_pretrained(config["checkpoint_path"]).eval().to(DEVICE) model = {"model": model_obj, "processor": processor} elif config["type"] == "clip": model_obj = AutoModel.from_pretrained(config["checkpoint_path"]).to(DEVICE) processor = AutoProcessor.from_pretrained(config["checkpoint_path"]) model = {"model": model_obj, "processor": processor} else: raise ValueError(f"Unknown model type: {config['type']}") current_models[model_key] = model current_model_name = model_key if update_status_fn: update_status_fn(f"✅ {MODEL_CONFIGS[model_key]['name']} loaded successfully!") return model except Exception as e: error_msg = f"Error loading model {model_key}: {e}" print(error_msg) if update_status_fn: update_status_fn(f"❌ {error_msg}") return None def score_with_model(model_key, image_paths, prompts): """Score images using the specified model.""" model = load_model(model_key) if model is None: raise ValueError(f"Failed to load model {model_key}") config = MODEL_CONFIGS[model_key] if config["type"] == "hpsv3": rewards = model.reward(image_paths, prompts) return [reward[0].item() for reward in rewards] # HPSv3 returns tensor with multiple values, take first elif config["type"] == "hpsv2": return score_hpsv2_batch(model, image_paths, prompts) elif config["type"] == "imagereward": return [model.score(prompt, image_path) for prompt, image_path in zip(prompts, image_paths)] elif config["type"] == "pickscore": return score_pickscore_batch(prompts, image_paths, model["model"], model["processor"]) elif config["type"] == "clip": return score_clip_batch(model["model"], model["processor"], image_paths, prompts) else: raise ValueError(f"Unknown model type: {config['type']}") def score_hpsv2_batch(model_dict, image_paths, prompts): """Score using HPSv2 model.""" model = model_dict['model'] preprocess_val = model_dict['preprocess_val'] tokenizer = model_dict['tokenizer'] # 批量处理图片 images = [preprocess_val(Image.open(p)).unsqueeze(0)[:,:3,:,:] for p in image_paths] images = torch.cat(images, dim=0).to(device=DEVICE) texts = tokenizer(prompts).to(device=DEVICE) with torch.no_grad(): outputs = model(images, texts) image_features, text_features = outputs["image_features"], outputs["text_features"] logits_per_image = image_features @ text_features.T hps_scores = torch.diagonal(logits_per_image).cpu() return [score.item() for score in hps_scores] def score_pickscore_batch(prompts, image_paths, model, processor): """Score using PickScore model.""" pil_images = [Image.open(p) for p in image_paths] image_inputs = processor( images=pil_images, padding=True, truncation=True, max_length=77, return_tensors="pt", ).to(DEVICE) text_inputs = processor( text=prompts, padding=True, truncation=True, max_length=77, return_tensors="pt", ).to(DEVICE) with torch.no_grad(): image_embs = model.get_image_features(**image_inputs) image_embs = image_embs / torch.norm(image_embs, dim=-1, keepdim=True) text_embs = model.get_text_features(**text_inputs) text_embs = text_embs / torch.norm(text_embs, dim=-1, keepdim=True) scores = model.logit_scale.exp() * (text_embs @ image_embs.T) return [scores[i, i].cpu().item() for i in range(len(prompts))] def score_clip_batch(model, processor, image_paths, prompts): """Score using CLIP model.""" pil_images = [Image.open(p) for p in image_paths] image_inputs = processor( images=pil_images, padding=True, truncation=True, max_length=77, return_tensors="pt", ).to(DEVICE) text_inputs = processor( text=prompts, padding=True, truncation=True, max_length=77, return_tensors="pt", ).to(DEVICE) with torch.no_grad(): image_embs = model.get_image_features(**image_inputs) image_embs = image_embs / torch.norm(image_embs, dim=-1, keepdim=True) text_embs = model.get_text_features(**text_inputs) text_embs = text_embs / torch.norm(text_embs, dim=-1, keepdim=True) scores = image_embs @ text_embs.T return [scores[i, i].cpu().item() for i in range(len(prompts))] # Load default model print("Loading default HPSv3 model...") load_model("HPSv3_7B") print("Model loaded successfully.") # --- Helper Functions --- def get_score_interpretation(score): """Returns a color-coded qualitative interpretation of the score.""" if score is None: return "" if score < 0: color = "#ef4444" # Modern red bg_color = "rgba(239, 68, 68, 0.1)" icon = "❌" feedback = "Poor Quality" comment = "The image has significant quality issues or doesn't match the prompt well." elif score < 5: color = "#f59e0b" # Modern amber bg_color = "rgba(245, 158, 11, 0.1)" icon = "⚠️" feedback = "Needs Improvement" comment = "The image is acceptable but could be enhanced in quality or prompt alignment." elif score < 10: color = "#10b981" # Modern emerald bg_color = "rgba(16, 185, 129, 0.1)" icon = "✅" feedback = "Good Quality" comment = "A well-crafted image that aligns nicely with the given prompt." else: # score >= 10 color = "#06d6a0" # Vibrant teal bg_color = "rgba(6, 214, 160, 0.1)" icon = "⭐" feedback = "Excellent!" comment = "Outstanding quality and perfect alignment with the prompt." return f"""
{comment}
Evaluate image quality and alignment with prompts with multiple models.