Spaces:
Runtime error
Runtime error
Update aduc_framework/engineers/deformes3D.py
Browse files
aduc_framework/engineers/deformes3D.py
CHANGED
|
@@ -2,18 +2,18 @@
|
|
| 2 |
#
|
| 3 |
# Copyright (C) August 4, 2025 Carlos Rodrigues dos Santos
|
| 4 |
#
|
| 5 |
-
# Versão 5.
|
| 6 |
#
|
| 7 |
-
# Esta versão implementa o executor do Diretor Autônomo
|
| 8 |
-
#
|
| 9 |
-
#
|
| 10 |
|
| 11 |
import os
|
| 12 |
import logging
|
| 13 |
import torch
|
| 14 |
from PIL import Image, ImageOps
|
| 15 |
import numpy as np
|
| 16 |
-
from typing import List, Dict, Any, Callable, Optional, Generator
|
| 17 |
|
| 18 |
from .deformes2D_thinker import deformes2d_thinker_singleton
|
| 19 |
from ..types import LatentConditioningItem
|
|
@@ -63,13 +63,12 @@ class Deformes3DEngine:
|
|
| 63 |
progress_callback: ProgressCallback = None
|
| 64 |
) -> Generator[List[Dict[str, Any]], None, None]:
|
| 65 |
"""
|
| 66 |
-
Ponto de entrada
|
| 67 |
emitindo a lista de keyframes a cada nova geração.
|
| 68 |
"""
|
| 69 |
if not self.workspace_dir:
|
| 70 |
raise RuntimeError("Deformes3DEngine não foi inicializado.")
|
| 71 |
|
| 72 |
-
# O 'yield from' passa adiante os yields do loop de gerenciamento.
|
| 73 |
yield from self._manage_directorial_loop(generation_state, progress_callback)
|
| 74 |
|
| 75 |
def _manage_directorial_loop(
|
|
@@ -95,7 +94,7 @@ class Deformes3DEngine:
|
|
| 95 |
|
| 96 |
while current_act_index < len(dynamic_script):
|
| 97 |
if progress_callback:
|
| 98 |
-
progress = current_act_index / len(dynamic_script)
|
| 99 |
progress_callback(progress, f"Diretor avaliando Ato {current_act_index + 1}")
|
| 100 |
|
| 101 |
context = {
|
|
@@ -122,7 +121,6 @@ class Deformes3DEngine:
|
|
| 122 |
)
|
| 123 |
all_keyframes.append(new_keyframe)
|
| 124 |
|
| 125 |
-
# EMITIR ATUALIZAÇÃO PARA A UI
|
| 126 |
yield all_keyframes
|
| 127 |
|
| 128 |
current_act_index += 1
|
|
@@ -132,23 +130,20 @@ class Deformes3DEngine:
|
|
| 132 |
|
| 133 |
def _generate_and_save_keyframe(self, decision: Dict, act_index: int, all_keyframes: List, resolution: int) -> Dict:
|
| 134 |
"""
|
| 135 |
-
SRP: Executa a geração de um único keyframe
|
| 136 |
"""
|
| 137 |
prompt = decision.get("prompt_proximo_keyframe", "Cena de transição cinematográfica.")
|
| 138 |
base_media_path = decision.get("midia_base_escolhida")
|
| 139 |
context_media_paths = decision.get("midias_contexto_escolhidas", [])
|
| 140 |
|
| 141 |
-
conditioning_items = self._prepare_ltx_conditioning(
|
| 142 |
act_index, all_keyframes, base_media_path, context_media_paths, resolution
|
| 143 |
)
|
| 144 |
|
| 145 |
generated_latents, _ = ltx_manager_singleton.generate_latent_fragment(
|
| 146 |
-
height=resolution, width=resolution,
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
video_total_frames=self._DIRECTOR_PARAMS["ltx_frames"],
|
| 150 |
-
video_fps=self._DIRECTOR_PARAMS["ltx_fps"],
|
| 151 |
-
guidance_scale=self._DIRECTOR_PARAMS["ltx_guidance_scale"],
|
| 152 |
num_inference_steps=self._DIRECTOR_PARAMS["ltx_inference_steps"]
|
| 153 |
)
|
| 154 |
new_latent = generated_latents[:, :, -1:, :, :].clone()
|
|
@@ -162,38 +157,46 @@ class Deformes3DEngine:
|
|
| 162 |
|
| 163 |
return {
|
| 164 |
"id": act_index, "caminho_pixel": pixel_path, "caminho_latent": latent_path,
|
| 165 |
-
"prompt_keyframe": prompt, "is_cut_point": decision.get("is_cut", False)
|
|
|
|
| 166 |
}
|
| 167 |
|
| 168 |
-
def _prepare_ltx_conditioning(self, act_index, keyframes, base_path, context_paths, res):
|
| 169 |
-
"""SRP: Constrói a lista de
|
| 170 |
-
items = []
|
| 171 |
res_tuple = (res, res)
|
| 172 |
weights = self._DIRECTOR_PARAMS["conditioning_weights"]
|
| 173 |
frames = self._DIRECTOR_PARAMS["conditioning_frames"]
|
| 174 |
|
| 175 |
-
def
|
| 176 |
pil = self._preprocess_image_for_latent_conversion(Image.open(path).convert("RGB"), res_tuple)
|
| 177 |
tensor = self._pil_to_pixel_tensor(pil)
|
| 178 |
return vae_manager_singleton.encode(tensor.to(self.device))
|
| 179 |
|
| 180 |
if not base_path:
|
| 181 |
logger.warning("Diretor não escolheu uma mídia base. A geração pode ser instável.")
|
| 182 |
-
return items
|
| 183 |
|
| 184 |
-
items.append(LatentConditioningItem(
|
|
|
|
| 185 |
|
| 186 |
-
if act_index > 0:
|
| 187 |
-
|
|
|
|
| 188 |
items.append(LatentConditioningItem(last_latent, frames["memory_last"], weights["memory_last"]))
|
| 189 |
-
|
| 190 |
-
|
|
|
|
|
|
|
|
|
|
| 191 |
items.append(LatentConditioningItem(penultimate_latent, frames["memory_penultimate"], weights["memory_penultimate"]))
|
|
|
|
| 192 |
|
| 193 |
for path in context_paths[:2]:
|
| 194 |
-
items.append(LatentConditioningItem(
|
|
|
|
| 195 |
|
| 196 |
-
return items
|
| 197 |
|
| 198 |
def _preprocess_image_for_latent_conversion(self, image: Image.Image, target_resolution: tuple) -> Image.Image:
|
| 199 |
return ImageOps.fit(image, target_resolution, Image.Resampling.LANCZOS) if image.size != target_resolution else image
|
|
@@ -208,4 +211,4 @@ class Deformes3DEngine:
|
|
| 208 |
image_np = (tensor.cpu().float().numpy() * 255).astype(np.uint8)
|
| 209 |
Image.fromarray(image_np).save(path)
|
| 210 |
|
| 211 |
-
deformes3d_engine_singleton = Deformes3DEngine()
|
|
|
|
| 2 |
#
|
| 3 |
# Copyright (C) August 4, 2025 Carlos Rodrigues dos Santos
|
| 4 |
#
|
| 5 |
+
# Versão 5.2.0 (Clean Architecture Executor with Streaming & DNA Provenance)
|
| 6 |
#
|
| 7 |
+
# Esta versão implementa o executor do Diretor Autônomo, emitindo atualizações
|
| 8 |
+
# em tempo real para a UI e registrando a proveniência de cada keyframe
|
| 9 |
+
# no DNA Digital da geração.
|
| 10 |
|
| 11 |
import os
|
| 12 |
import logging
|
| 13 |
import torch
|
| 14 |
from PIL import Image, ImageOps
|
| 15 |
import numpy as np
|
| 16 |
+
from typing import List, Dict, Any, Callable, Optional, Generator, Tuple
|
| 17 |
|
| 18 |
from .deformes2D_thinker import deformes2d_thinker_singleton
|
| 19 |
from ..types import LatentConditioningItem
|
|
|
|
| 63 |
progress_callback: ProgressCallback = None
|
| 64 |
) -> Generator[List[Dict[str, Any]], None, None]:
|
| 65 |
"""
|
| 66 |
+
Ponto de entrada que orquestra o ciclo de direção autônoma,
|
| 67 |
emitindo a lista de keyframes a cada nova geração.
|
| 68 |
"""
|
| 69 |
if not self.workspace_dir:
|
| 70 |
raise RuntimeError("Deformes3DEngine não foi inicializado.")
|
| 71 |
|
|
|
|
| 72 |
yield from self._manage_directorial_loop(generation_state, progress_callback)
|
| 73 |
|
| 74 |
def _manage_directorial_loop(
|
|
|
|
| 94 |
|
| 95 |
while current_act_index < len(dynamic_script):
|
| 96 |
if progress_callback:
|
| 97 |
+
progress = current_act_index / len(dynamic_script) if len(dynamic_script) > 0 else 0
|
| 98 |
progress_callback(progress, f"Diretor avaliando Ato {current_act_index + 1}")
|
| 99 |
|
| 100 |
context = {
|
|
|
|
| 121 |
)
|
| 122 |
all_keyframes.append(new_keyframe)
|
| 123 |
|
|
|
|
| 124 |
yield all_keyframes
|
| 125 |
|
| 126 |
current_act_index += 1
|
|
|
|
| 130 |
|
| 131 |
def _generate_and_save_keyframe(self, decision: Dict, act_index: int, all_keyframes: List, resolution: int) -> Dict:
|
| 132 |
"""
|
| 133 |
+
SRP: Executa a geração de um único keyframe e retorna seus metadados completos.
|
| 134 |
"""
|
| 135 |
prompt = decision.get("prompt_proximo_keyframe", "Cena de transição cinematográfica.")
|
| 136 |
base_media_path = decision.get("midia_base_escolhida")
|
| 137 |
context_media_paths = decision.get("midias_contexto_escolhidas", [])
|
| 138 |
|
| 139 |
+
conditioning_items, dna_data = self._prepare_ltx_conditioning(
|
| 140 |
act_index, all_keyframes, base_media_path, context_media_paths, resolution
|
| 141 |
)
|
| 142 |
|
| 143 |
generated_latents, _ = ltx_manager_singleton.generate_latent_fragment(
|
| 144 |
+
height=resolution, width=resolution, conditioning_items_data=conditioning_items,
|
| 145 |
+
motion_prompt=prompt, video_total_frames=self._DIRECTOR_PARAMS["ltx_frames"],
|
| 146 |
+
video_fps=self._DIRECTOR_PARAMS["ltx_fps"], guidance_scale=self._DIRECTOR_PARAMS["ltx_guidance_scale"],
|
|
|
|
|
|
|
|
|
|
| 147 |
num_inference_steps=self._DIRECTOR_PARAMS["ltx_inference_steps"]
|
| 148 |
)
|
| 149 |
new_latent = generated_latents[:, :, -1:, :, :].clone()
|
|
|
|
| 157 |
|
| 158 |
return {
|
| 159 |
"id": act_index, "caminho_pixel": pixel_path, "caminho_latent": latent_path,
|
| 160 |
+
"prompt_keyframe": prompt, "is_cut_point": decision.get("is_cut", False),
|
| 161 |
+
"entradas_latentes": dna_data
|
| 162 |
}
|
| 163 |
|
| 164 |
+
def _prepare_ltx_conditioning(self, act_index: int, keyframes: List[Dict], base_path: str, context_paths: List[str], res: int) -> Tuple[List[LatentConditioningItem], List[Dict]]:
|
| 165 |
+
"""SRP: Constrói a lista de condicionais para o LTX e os dados para o DNA."""
|
| 166 |
+
items, dna_data = [], []
|
| 167 |
res_tuple = (res, res)
|
| 168 |
weights = self._DIRECTOR_PARAMS["conditioning_weights"]
|
| 169 |
frames = self._DIRECTOR_PARAMS["conditioning_frames"]
|
| 170 |
|
| 171 |
+
def to_latent_tensor(path):
|
| 172 |
pil = self._preprocess_image_for_latent_conversion(Image.open(path).convert("RGB"), res_tuple)
|
| 173 |
tensor = self._pil_to_pixel_tensor(pil)
|
| 174 |
return vae_manager_singleton.encode(tensor.to(self.device))
|
| 175 |
|
| 176 |
if not base_path:
|
| 177 |
logger.warning("Diretor não escolheu uma mídia base. A geração pode ser instável.")
|
| 178 |
+
return items, dna_data
|
| 179 |
|
| 180 |
+
items.append(LatentConditioningItem(to_latent_tensor(base_path), frames["future_anchor"], weights["future_base"]))
|
| 181 |
+
dna_data.append({"caminho_origem": base_path, "frame_alvo": frames["future_anchor"], "forca_condicionamento": weights["future_base"]})
|
| 182 |
|
| 183 |
+
if act_index > 0 and keyframes:
|
| 184 |
+
last_kf_path = keyframes[-1]["caminho_latent"]
|
| 185 |
+
last_latent = torch.load(last_kf_path, map_location=self.device)
|
| 186 |
items.append(LatentConditioningItem(last_latent, frames["memory_last"], weights["memory_last"]))
|
| 187 |
+
dna_data.append({"caminho_origem": last_kf_path, "frame_alvo": frames["memory_last"], "forca_condicionamento": weights["memory_last"]})
|
| 188 |
+
|
| 189 |
+
if act_index > 1 and len(keyframes) >= 2:
|
| 190 |
+
penultimate_kf_path = keyframes[-2]["caminho_latent"]
|
| 191 |
+
penultimate_latent = torch.load(penultimate_kf_path, map_location=self.device)
|
| 192 |
items.append(LatentConditioningItem(penultimate_latent, frames["memory_penultimate"], weights["memory_penultimate"]))
|
| 193 |
+
dna_data.append({"caminho_origem": penultimate_kf_path, "frame_alvo": frames["memory_penultimate"], "forca_condicionamento": weights["memory_penultimate"]})
|
| 194 |
|
| 195 |
for path in context_paths[:2]:
|
| 196 |
+
items.append(LatentConditioningItem(to_latent_tensor(path), frames["future_anchor"], weights["future_context"]))
|
| 197 |
+
dna_data.append({"caminho_origem": path, "frame_alvo": frames["future_anchor"], "forca_condicionamento": weights["future_context"]})
|
| 198 |
|
| 199 |
+
return items, dna_data
|
| 200 |
|
| 201 |
def _preprocess_image_for_latent_conversion(self, image: Image.Image, target_resolution: tuple) -> Image.Image:
|
| 202 |
return ImageOps.fit(image, target_resolution, Image.Resampling.LANCZOS) if image.size != target_resolution else image
|
|
|
|
| 211 |
image_np = (tensor.cpu().float().numpy() * 255).astype(np.uint8)
|
| 212 |
Image.fromarray(image_np).save(path)
|
| 213 |
|
| 214 |
+
deformes3d_engine_singleton = Deformes3DEngine()```
|