add option for new user provided concepts

app.py

@@ -10,6 +10,7 @@ import open_clip
 from huggingface_hub import hf_hub_download
 from IP_Composer.IP_Adapter.ip_adapter import IPAdapterXL
 from IP_Composer.perform_swap import compute_dataset_embeds_svd, get_modified_images_embeds_composition
+from IP_Compoeser.generate_text_embeddings import load_descriptions, generate_embeddings 
 import spaces
 import random
 
@@ -32,6 +33,8 @@ ip_model = IPAdapterXL(pipe, image_encoder_repo, image_encoder_subfolder, ip_ckp
 # Initialize CLIP model
 clip_model, _, preprocess = open_clip.create_model_and_transforms('hf-hub:laion/CLIP-ViT-H-14-laion2B-s32B-b79K')
 clip_model.to(device)
+tokenizer = open_clip.get_tokenizer('hf-hub:laion/CLIP-ViT-H-14-laion2B-s32B-b79K')
+
 
 CONCEPTS_MAP={
     "age": "age_descriptions.npy",
@@ -120,6 +123,12 @@ def process_images(
     scale=1.0,
     seed=420,
     num_inference_steps=50,
+    concpet_from_file_1 = None,
+    concpet_from_file_2 = None,
+    concpet_from_file_3 = None,
+    use_concpet_from_file_1 = False,
+    use_concpet_from_file_2 = False,
+    use_concpet_from_file_3 = False
 ):
     """Process the base image and concept images to generate modified images"""
     # Process base image
@@ -129,23 +138,37 @@ def process_images(
     # Process concept images
     concept_images = []
     concept_descriptions = []
+
+    skip_load_concept =[False,False, False]
     
     # for demo purposes we allow for up to 3 different concepts and corresponding concept images 
     if concept_image1 is not None:
         concept_images.append(concept_image1)
-        concept_descriptions.append(CONCEPTS_MAP[concept_name1])
+        if use_concpet_from_file_1 and concpet_from_file_1: # if concept is new from user input
+            concept_descriptions.append(use_concpet_from_file_1)
+            skip_load_concept[0] = True
+        else:
+            concept_descriptions.append(CONCEPTS_MAP[concept_name1])
     else:
         return None, "Please upload at least one concept image"
     
     # Add second concept (optional)
     if concept_image2 is not None:
         concept_images.append(concept_image2)
-        concept_descriptions.append(CONCEPTS_MAP[concept_name2])
+        if use_concpet_from_file_2 and concpet_from_file_2: # if concept is new from user input
+            concept_descriptions.append(use_concpet_from_file_2)
+            skip_load_concept[1] = True
+        else:
+            concept_descriptions.append(CONCEPTS_MAP[concept_name2])
     
     # Add third concept (optional)
     if concept_image3 is not None:
         concept_images.append(concept_image3)
-        concept_descriptions.append(CONCEPTS_MAP[concept_name3])
+        if use_concpet_from_file_3 and concpet_from_file_3: # if concept is new from user input
+            concept_descriptions.append(use_concpet_from_file_3)
+            skip_load_concept[2] = True
+        else:
+            concept_descriptions.append(CONCEPTS_MAP[concept_name3])
     
     # Get all ranks
     ranks = [rank1]
@@ -159,12 +182,15 @@ def process_images(
     projection_matrices = []
     # for the demo, we assume 1 concept image per concept
     # for each concept image, we calculate it's image embeedings and load the concepts textual embeddings to copmpute the projection matrix over it
-    for i, concept_name in enumerate(concept_descriptions):
+    for i, concept in enumerate(concept_descriptions):
         img_pil = Image.fromarray(concept_images[i]).convert("RGB")
         concept_embeds.append(get_image_embeds(img_pil, clip_model, preprocess, device))
-        embeds_path = f"./IP_Composer/text_embeddings/{concept_name}"
-        with open(embeds_path, "rb") as f:
-            all_embeds_in = np.load(f)
+        if skip_load_concept[i]: # if concept is new from user input
+            all_embeds_in = concept        
+        else:
+            embeds_path = f"./IP_Composer/text_embeddings/{concept}"
+            with open(embeds_path, "rb") as f:
+                all_embeds_in = np.load(f)
         
         projection_matrix = compute_dataset_embeds_svd(all_embeds_in, ranks[i])
         projection_matrices.append(projection_matrix)
@@ -193,6 +219,13 @@ def process_images(
     
     return modified_images[0]
 
+@spaces.GPU
+def get_text_embeddings(concept_file):
+    descriptions = load_descriptions(concept_file) 
+    embeddings = generate_embeddings(descriptions, model, tokenizer, device, batch_size=100)
+    return embeddings, True
+    
+
 def process_and_display(
     base_image, 
     concept_image1, concept_name1="age",
@@ -236,7 +269,12 @@ following the algorithm proposed in [*IP-Composer: Semantic Composition of Visua
 
 [[project page](https://ip-composer.github.io/IP-Composer/)] [[arxiv](https://arxiv.org/pdf/2502.13951)]
         """)
-        
+        concpet_from_file_1 = gr.State()
+        concpet_from_file_2 = gr.State()
+        concpet_from_file_3 = gr.State()
+        use_concpet_from_file_1 = gr.State()
+        use_concpet_from_file_2 = gr.State()
+        use_concpet_from_file_3 = gr.State()
         with gr.Row():
             with gr.Column():
                 base_image = gr.Image(label="Base Image (Required)", type="numpy")
@@ -244,15 +282,19 @@ following the algorithm proposed in [*IP-Composer: Semantic Composition of Visua
                   with gr.Row():
                       with gr.Group():
                           concept_image1 = gr.Image(label="Concept Image 1", type="numpy")
+                          concept_file_1 = gradio.File(label="concept variations", file_types="text")
                           concept_name1 = gr.Dropdown(concept_options, label="concept 1", value=None, info="concept type")
                           
                 with gr.Tab("concept 2 - optional"):            
                      with gr.Group():    
                           concept_image2 = gr.Image(label="Concept Image 2", type="numpy")
+                          concept_file_2 = gradio.File(label="concept variations", file_types="text")
                           concept_name2 = gr.Dropdown(concept_options, label="concept 2", value=None, info="concept type")
+                
                 with gr.Tab("concept 3 - optional"):
                       with gr.Group():
                           concept_image3 = gr.Image(label="Concept Image 3", type="numpy")
+                          concept_file_3 = gradio.File(label="concept variations", file_types="text")
                           concept_name3 = gr.Dropdown(concept_options, label="concept 3", value= None, info="concept type")
                
             
@@ -284,6 +326,38 @@ following the algorithm proposed in [*IP-Composer: Semantic Composition of Visua
         fn=generate_examples,
         cache_examples=False
         )
+
+        concept_file_1.upload(
+            fn=get_text_embeddings,
+            inputs=[concept_file_1],
+            outputs=[concpet_from_file_1, use_concpet_from_file_1]
+        )
+        concept_file_2.upload(
+            fn=get_text_embeddings,
+            inputs=[concept_file_2],
+            outputs=[concpet_from_file_2, use_concpet_from_file_2]
+        )
+        concept_file_3.upload(
+            fn=get_text_embeddings,
+            inputs=[concept_file_3],
+            outputs=[concpet_from_file_3, use_concpet_from_file_3]
+        )
+
+        concept_file_1.delete(
+            fn=lambda x: False,
+            inputs=[concept_file_1],
+            outputs=[use_concpet_from_file_1]
+        )
+        concept_file_2.delete(
+            fn=lambda x: False,
+            inputs=[concept_file_2],
+            outputs=[use_concpet_from_file_2]
+        )
+        concept_file_3.delete(
+            fn=lambda x: False,
+            inputs=[concept_file_3],
+            outputs=[use_concpet_from_file_3]
+        )
         
         submit_btn.click(
             fn=randomize_seed_fn,
@@ -296,7 +370,13 @@ following the algorithm proposed in [*IP-Composer: Semantic Composition of Visua
                 concept_image2, concept_name2,
                 concept_image3, concept_name3,
                 rank1, rank2, rank3, 
-                prompt, scale, seed, num_inference_steps
+                prompt, scale, seed, num_inference_steps,
+                concpet_from_file_1,
+                concpet_from_file_2,
+                concpet_from_file_3,
+                use_concpet_from_file_1,
+                use_concpet_from_file_2,
+                use_concpet_from_file_3
             ],
             outputs=[output_image]
         )