app.py

import cv2
import torch
from PIL import Image, ImageOps
import numpy as np
import gradio as gr
import math
import os
import zipfile
import trimesh
import pygltflib
from scipy.ndimage import median_filter
import requests # Import requests for downloading


# Depth-Anything V2 model setup
from depth_anything_v2.dpt import DepthAnythingV2

DEVICE = 'cuda' if torch.cuda.is_available() else 'mps' if torch.backends.mps.is_available() else 'cpu'

model_configs = {
    'vits': {'encoder': 'vits', 'features': 64, 'out_channels': [48, 96, 192, 384]},
    'vitb': {'encoder': 'vitb', 'features': 128, 'out_channels': [96, 192, 384, 768]},
    'vitl': {'encoder': 'vitl', 'features': 256, 'out_channels': [256, 512, 1024, 1024]},
    'vitg': {'encoder': 'vitg', 'features': 384, 'out_channels': [1536, 1536, 1536, 1536]}
}

encoder = 'vitl' # or 'vits', 'vitb', 'vitg'

model = DepthAnythingV2(**model_configs[encoder])

# Define model directory and path
MODEL_DIR = "models"
os.makedirs(MODEL_DIR, exist_ok=True)
model_filename = f'depth_anything_v2_{encoder}.pth'
model_path = os.path.join(MODEL_DIR, model_filename)

# Add code to download model weights if not exists
if not os.path.exists(model_path):
    print(f"Downloading {model_path}...")
    url = f"https://huggingface.co/depth-anything/Depth-Anything-V2-Large/resolve/main/{model_filename}"
    response = requests.get(url, stream=True)
    with open(model_path, "wb") as f:
        for chunk in response.iter_content(chunk_size=8192):
            f.write(chunk)
    print("Download complete.")

model.load_state_dict(torch.load(model_path, map_location='cpu'))
model = model.to(DEVICE).eval()

# Helper functions (from your notebook)
def quaternion_multiply(q1, q2):
    x1, y1, z1, w1 = q1
    x2, y2, z2, w2 = q2
    return [
        w1 * x2 + x1 * w2 + y1 * z2 - z1 * y2,
        w1 * y2 - x1 * z2 + y1 * w2 + z1 * x2,
        w1 * z2 + x1 * y2 - y1 * x2 + z1 * w2,
        w1 * w2 - x1 * x2 - y1 * y2 - z1 * z2,
    ]


def glb_add_lights(path_input, path_output):
    """
    Adds directional lights in the horizontal plane to the glb file.
    :param path_input: path to input glb
    :param path_output: path to output glb
    :return: None
    """
    glb = pygltflib.GLTF2().load(path_input)

    N = 3  # default max num lights in Babylon.js is 4
    angle_step = 2 * math.pi / N
    elevation_angle = math.radians(75)

    light_colors = [
        [1.0, 0.0, 0.0],
        [0.0, 1.0, 0.0],
        [0.0, 0.0, 1.0],
    ]

    lights_extension = {
        "lights": [
            {"type": "directional", "color": light_colors[i], "intensity": 2.0}
            for i in range(N)
        ]
    }

    if "KHR_lights_punctual" not in glb.extensionsUsed:
        glb.extensionsUsed.append("KHR_lights_punctual")
    glb.extensions["KHR_lights_punctual"] = lights_extension

    light_nodes = []
    for i in range(N):
        angle = i * angle_step

        pos_rot = [0.0, 0.0, math.sin(angle / 2), math.cos(angle / 2)]
        elev_rot = [
            math.sin(elevation_angle / 2),
            0.0,
            0.0,
            math.cos(elevation_angle / 2),
        ]
        rotation = quaternion_multiply(pos_rot, elev_rot)

        node = {
            "rotation": rotation,
            "extensions": {"KHR_lights_punctual": {"light": i}},
        }
        light_nodes.append(node)

    light_node_indices = list(range(len(glb.nodes), len(glb.nodes) + N))
    glb.nodes.extend(light_nodes)

    root_node_index = glb.scenes[glb.scene].nodes[0]
    root_node = glb.nodes[root_node_index]
    if hasattr(root_node, "children"):
        root_node.children.extend(light_node_indices)
    else:
        root_node.children = light_node_indices

    glb.save(path_output)


def extrude_depth_3d(
    path_rgb,
    path_depth,
    path_out_base=None,
    alpha=1.0,
    invert=0,
    output_model_scale=100,
    filter_size=3,
    coef_near=0.0,
    coef_far=1.0,
    emboss=0.3,
    f_thic=0.05,
    f_near=-0.15,
    f_back=0.01,
    vertex_colors=True,
    scene_lights=True,
    prepare_for_3d_printing=False,
    zip_outputs=False,
    lift_height=0.0
):
    f_far_inner = -emboss
    f_far_outer = f_far_inner - f_back

    f_near = max(f_near, f_far_inner)

    depth_image = Image.open(path_depth)
    mono_image = Image.open(path_rgb).convert("L")

    if invert==1:
        mono_image = ImageOps.invert(mono_image)

    w, h = depth_image.size
    d_max = max(w, h)
    depth_image = np.array(depth_image).astype(np.double)
    mono_image = np.array(mono_image).astype(np.double)
    z_min, z_max = np.min(depth_image), np.max(depth_image)
    m_min, m_max = np.min(mono_image), np.max(mono_image)
    depth_image = (depth_image.astype(np.double) - z_min) / (z_max - z_min)
    depth_image[depth_image < coef_near] = coef_near
    depth_image[depth_image > coef_far] = coef_far
    z_min, z_max = np.min(depth_image), np.max(depth_image)
    depth_image = (depth_image - z_min) / (z_max - z_min)
    mono_image = median_filter(mono_image, size=5)
    mono_image = (mono_image.astype(np.double) - m_min) / (m_max - m_min)
    mono_image_new = np.where(depth_image == coef_far, 1, mono_image)
    m_min=np.min(mono_image_new)
    mono_image_new = np.where(depth_image == coef_far, 0, mono_image)
    m_max=np.max(mono_image_new)
    mono_image = np.where(depth_image == coef_far, m_min, mono_image)
    mono_image = (mono_image - m_min) / (m_max - m_min)
    depth_image = np.where(depth_image != 1.0, (1-alpha) * depth_image + alpha * mono_image, depth_image)
    #depth_image_new[depth_image < coef_near] = 0
    #depth_image_new[depth_image > coef_far] = 1
    #depth_image_new[depth_image_new < 0] = 0
    depth_image = median_filter(depth_image, size=filter_size)
    depth_image = emboss*(depth_image - np.min(depth_image)) / (np.max(depth_image) - np.min(depth_image))
    depth_image = np.where(depth_image != emboss, depth_image + lift_height, depth_image)
    Image.fromarray((depth_image * 255).astype(np.uint8)).convert("L").save(path_out_base+".png")
    rgb_image = np.array(
        Image.open(path_rgb).convert("RGB").resize((w, h), Image.Resampling.LANCZOS)
    )

    w_norm = w / float(d_max - 1)
    h_norm = h / float(d_max - 1)
    w_half = w_norm / 2
    h_half = h_norm / 2

    x, y = np.meshgrid(np.arange(w), np.arange(h))
    x = x / float(d_max - 1) - w_half  # [-w_half, w_half]
    y = -y / float(d_max - 1) + h_half  # [-h_half, h_half]
    z = -depth_image  # -depth_emboss (far) - 0 (near)
    vertices_2d = np.stack((x, y, z), axis=-1)
    vertices = vertices_2d.reshape(-1, 3)
    colors = rgb_image[:, :, :3].reshape(-1, 3) / 255.0

    faces = []
    for y in range(h - 1):
        for x in range(w - 1):
            idx = y * w + x
            faces.append([idx, idx + w, idx + 1])
            faces.append([idx + 1, idx + w, idx + 1 + w])

    # OUTER frame

    nv = len(vertices)
    vertices = np.append(
        vertices,
        [
            [-w_half - f_thic, -h_half - f_thic, f_near],  # 00
            [-w_half - f_thic, -h_half - f_thic, f_far_outer],  # 01
            [w_half + f_thic, -h_half - f_thic, f_near],  # 02
            [w_half + f_thic, -h_half - f_thic, f_far_outer],  # 03
            [w_half + f_thic, h_half + f_thic, f_near],  # 04
            [w_half + f_thic, h_half + f_thic, f_far_outer],  # 05
            [-w_half - f_thic, h_half + f_thic, f_near],  # 06
            [-w_half - f_thic, h_half + f_thic, f_far_outer],  # 07
        ],
        axis=0,
    )
    faces.extend(
        [
            [nv + 0, nv + 1, nv + 2],
            [nv + 2, nv + 1, nv + 3],
            [nv + 2, nv + 3, nv + 4],
            [nv + 4, nv + 3, nv + 5],
            [nv + 4, nv + 5, nv + 6],
            [nv + 6, nv + 5, nv + 7],
            [nv + 6, nv + 7, nv + 0],
            [nv + 0, nv + 7, nv + 1],
        ]
    )
    colors = np.append(colors, [[0.5, 0.5, 0.5]] * 8, axis=0)

    # INNER frame

    nv = len(vertices)
    vertices_left_data = vertices_2d[:, 0]  # H x 3
    vertices_left_frame = vertices_2d[:, 0].copy()  # H x 3
    vertices_left_frame[:, 2] = f_near
    vertices = np.append(vertices, vertices_left_data, axis=0)
    vertices = np.append(vertices, vertices_left_frame, axis=0)
    colors = np.append(colors, [[0.5, 0.5, 0.5]] * (2 * h), axis=0)
    for i in range(h - 1):
        nvi_d = nv + i
        nvi_f = nvi_d + h
        faces.append([nvi_d, nvi_f, nvi_d + 1])
        faces.append([nvi_d + 1, nvi_f, nvi_f + 1])

    nv = len(vertices)
    vertices_right_data = vertices_2d[:, -1]  # H x 3
    vertices_right_frame = vertices_2d[:, -1].copy()  # H x 3
    vertices_right_frame[:, 2] = f_near
    vertices = np.append(vertices, vertices_right_data, axis=0)
    vertices = np.append(vertices, vertices_right_frame, axis=0)
    colors = np.append(colors, [[0.5, 0.5, 0.5]] * (2 * h), axis=0)
    for i in range(h - 1):
        nvi_d = nv + i
        nvi_f = nvi_d + h
        faces.append([nvi_d, nvi_d + 1, nvi_f])
        faces.append([nvi_d + 1, nvi_f + 1, nvi_f])

    nv = len(vertices)
    vertices_top_data = vertices_2d[0, :]  # H x 3
    vertices_top_frame = vertices_2d[0, :].copy()  # H x 3
    vertices_top_frame[:, 2] = f_near
    vertices = np.append(vertices, vertices_top_data, axis=0)
    vertices = np.append(vertices, vertices_top_frame, axis=0)
    colors = np.append(colors, [[0.5, 0.5, 0.5]] * (2 * w), axis=0)
    for i in range(w - 1):
        nvi_d = nv + i
        nvi_f = nvi_d + w
        faces.append([nvi_d, nvi_d + 1, nvi_f])
        faces.append([nvi_d + 1, nvi_f + 1, nvi_f])

    nv = len(vertices)
    vertices_bottom_data = vertices_2d[-1, :]  # H x 3
    vertices_bottom_frame = vertices_2d[-1, :].copy()  # H x 3
    vertices_bottom_frame[:, 2] = f_near
    vertices = np.append(vertices, vertices_bottom_data, axis=0)
    vertices = np.append(vertices, vertices_bottom_frame, axis=0)
    colors = np.append(colors, [[0.5, 0.5, 0.5]] * (2 * w), axis=0)
    for i in range(w - 1):
        nvi_d = nv + i
        nvi_f = nvi_d + w
        faces.append([nvi_d, nvi_f, nvi_d + 1])
        faces.append([nvi_d + 1, nvi_f, nvi_f + 1])

    # FRONT frame

    nv = len(vertices)
    vertices = np.append(
        vertices,
        [
            [-w_half - f_thic, -h_half - f_thic, f_near],
            [-w_half - f_thic, h_half + f_thic, f_near],
        ],
        axis=0,
    )
    vertices = np.append(vertices, vertices_left_frame, axis=0)
    colors = np.append(colors, [[0.5, 0.5, 0.5]] * (2 + h), axis=0)
    for i in range(h - 1):
        faces.append([nv, nv + 2 + i + 1, nv + 2 + i])
    faces.append([nv, nv + 2, nv + 1])

    nv = len(vertices)
    vertices = np.append(
        vertices,
        [
            [w_half + f_thic, h_half + f_thic, f_near],
            [w_half + f_thic, -h_half - f_thic, f_near],
        ],
        axis=0,
    )
    vertices = np.append(vertices, vertices_right_frame, axis=0)
    colors = np.append(colors, [[0.5, 0.5, 0.5]] * (2 + h), axis=0)
    for i in range(h - 1):
        faces.append([nv, nv + 2 + i, nv + 2 + i + 1])
    faces.append([nv, nv + h + 1, nv + 1])

    nv = len(vertices)
    vertices = np.append(
        vertices,
        [
            [w_half + f_thic, h_half + f_thic, f_near],
            [-w_half - f_thic, h_half + f_thic, f_near],
        ],
        axis=0,
    )
    vertices = np.append(vertices, vertices_top_frame, axis=0)
    colors = np.append(colors, [[0.5, 0.5, 0.5]] * (2 + w), axis=0)
    for i in range(w - 1):
        faces.append([nv, nv + 2 + i, nv + 2 + i + 1])
    faces.append([nv, nv + 1, nv + 2])

    nv = len(vertices)
    vertices = np.append(
        vertices,
        [
            [-w_half - f_thic, -h_half - f_thic, f_near],
            [w_half + f_thic, -h_half - f_thic, f_near],
        ],
        axis=0,
    )
    vertices = np.append(vertices, vertices_bottom_frame, axis=0)
    colors = np.append(colors, [[0.5, 0.5, 0.5]] * (2 + w), axis=0)
    for i in range(w - 1):
        faces.append([nv, nv + 2 + i + 1, nv + 2 + i])
    faces.append([nv, nv + 1, nv + w + 1])

    # BACK frame

    nv = len(vertices)
    vertices = np.append(
        vertices,
        [
            [-w_half - f_thic, -h_half - f_thic, f_far_outer],  # 00
            [w_half + f_thic, -h_half - f_thic, f_far_outer],  # 01
            [w_half + f_thic, h_half + f_thic, f_far_outer],  # 02
            [-w_half - f_thic, h_half + f_thic, f_far_outer],  # 03
        ],
        axis=0,
    )
    faces.extend(
        [
            [nv + 0, nv + 2, nv + 1],
            [nv + 2, nv + 0, nv + 3],
        ]
    )
    colors = np.append(colors, [[0.5, 0.5, 0.5]] * 4, axis=0)

    trimesh_kwargs = {}
    if vertex_colors:
        trimesh_kwargs["vertex_colors"] = colors
    mesh = trimesh.Trimesh(vertices=vertices, faces=faces, **trimesh_kwargs)

    mesh.merge_vertices()

    current_max_dimension = max(mesh.extents)
    scaling_factor = output_model_scale / current_max_dimension
    mesh.apply_scale(scaling_factor)

    if prepare_for_3d_printing:
        rotation_mat = trimesh.transformations.rotation_matrix(
            np.radians(0), [0.5, 0, 0]
        )
        mesh.apply_transform(rotation_mat)

    if path_out_base is None:
        path_out_base = os.path.splitext(path_depth)[0].replace("_16bit", "")
    path_out_glb = path_out_base + ".glb"
    path_out_stl = path_out_base + ".stl"
    path_out_obj = path_out_base + ".obj"

    mesh.export(path_out_stl, file_type="stl")
    """
    mesh.export(path_out_glb, file_type="glb")
    if scene_lights:
        glb_add_lights(path_out_glb, path_out_glb)
    mesh.export(path_out_obj, file_type="obj")

    if zip_outputs:
        with zipfile.ZipFile(path_out_glb + ".zip", "w", zipfile.ZIP_DEFLATED) as zipf:
            arcname = os.path.basename(os.path.splitext(path_out_glb)[0]) + ".glb"
            zipf.write(path_out_glb, arcname=arcname)
            path_out_glb = path_out_glb + ".zip"
        with zipfile.ZipFile(path_out_stl + ".zip", "w", zipfile.ZIP_DEFLATED) as zipf:
            arcname = os.path.basename(os.path.splitext(path_out_stl)[0]) + ".stl"
            zipf.write(path_out_stl, arcname=arcname)
            path_out_stl = path_out_stl + ".zip"
        with zipfile.ZipFile(path_out_obj + ".zip", "w", zipfile.ZIP_DEFLATED) as zipf:
            arcname = os.path.basename(os.path.splitext(path_out_obj)[0]) + ".obj"
            zipf.write(path_out_obj, arcname=arcname)
            path_out_obj = path_out_obj + ".zip"
    """
    return path_out_glb, path_out_stl, path_out_obj

def scale_to_width(img, length):
      if img.width < img.height:
        width = length
        height = round(img.height * length / img.width)
      else:
        width = round(img.width * length / img.height)
        height = length
      return (width,height)


# Gradio Interface function
def process_image_and_generate_stl(image_input, depth_near, depth_far, thickness, alpha, backsheet, lift):
    # Depth Estimation
    raw_img = cv2.imread(image_input)
    depth = model.infer_image(raw_img) # HxW raw depth map in numpy

    # Save depth map temporarily
    depth_output_path = "output_depth.png"
    cv2.imwrite(depth_output_path, depth)

    # Prepare images for 3D model generation
    img_rgb = image_input
    img_depth = depth_output_path
    inv = 0 # Assuming no inversion for now, based on previous code
    # Image.open(img_rgb).convert("L").save("example_1_black.png") # This line might not be necessary for the final output
    size = scale_to_width(Image.open(img_rgb), 512)
    Image.open(img_rgb).resize(size, Image.Resampling.LANCZOS).save("one.png") # Use Resampling.LANCZOS
    if inv == 1:
        Image.open(img_depth).convert(mode="F").resize(size, Image.Resampling.BILINEAR).convert("I").save("two.png") # Use Resampling.BILINEAR
    else:
        img=Image.open(img_depth).convert(mode="F").resize(size, Image.Resampling.BILINEAR).convert("I") # Use Resampling.BILINEAR
        img = np.array(img).astype(np.double)
        im_max=np.max(img)
        im_min=np.min(img)
        img=(1-(img-im_min)/(im_max-im_min))*im_max
        img=Image.fromarray(img)
        img.convert("I").save("two.png")


    # 3D Model Generation
    output_path_base = "generated_relief"
    glb_path, stl_path, obj_path = extrude_depth_3d(
        "one.png",
        "two.png",
        alpha=alpha,
        invert=inv,
        path_out_base=output_path_base,
        output_model_scale=100,
        filter_size=5, # Using 5 based on previous code
        coef_near=depth_near,
        coef_far=depth_far,
        emboss=thickness,
        f_thic=0.0, # Using 0.0 based on previous code
        f_near=-thickness, # Using -thickness based on previous code
        f_back=backsheet, # Using 0.01 based on previous code
        vertex_colors=True,
        scene_lights=True,
        prepare_for_3d_printing=True,
        lift_height=-lift
    )
    preview_path = stl_path.replace(".stl", "_preview.stl")
    mesh = trimesh.load(stl_path)
    rot = trimesh.transformations.rotation_matrix(np.radians(180), [0, 0, 1])
    rot2 = trimesh.transformations.rotation_matrix(np.radians(-90),[1, 0, 0])
    mesh.apply_transform(rot)
    mesh.apply_transform(rot2)
    mesh.export(preview_path)
    return stl_path, preview_path # Return the path to the generated STL file


# Gradio Interface definition
iface = gr.Interface(
    fn=process_image_and_generate_stl,
    inputs=[
        gr.Image(type="filepath", label="Upload Image"),
        gr.Slider(minimum=0, maximum=1.0, value=0, label="Depth Near"),
        gr.Slider(minimum=0, maximum=1.0, value=1.0, label="Depth Far"),
        gr.Slider(minimum=0.1, maximum=1.0, value=0.3, label="Thickness"),
        gr.Slider(minimum=0, maximum=1.0, value=0.05, label="Alpha"),
        gr.Slider(minimum=0.01, maximum=0.1, value=0.01, label="BackSheet Thickness"),
        gr.Slider(minimum=0, maximum=0.1, value=0.0, label="lift"),
    ],
    outputs=[
        gr.File(label="Download STL File"),
        gr.Model3D(label="STL Preview")
    ], # Use gr.File() for file downloads
    title="Image to 2.5D Relief Model Generator",
    description="Upload an image, set parameters, and generate a 2.5D relief model (.stl file)."
)

# Launch the interface (for local testing)
if __name__ == "__main__":
    iface.launch(debug=True)