Stable Fast 3D's own save node hands back the GLB as base64 in its UI output, which the gate does not read. LLMGWSaveMesh writes each mesh to output/ as a GLB and names it, as a picture's save node does. The prepare node also gives the object's mask now, which SF3D's sampler takes. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
133 lines
5.7 KiB
Python
133 lines
5.7 KiB
Python
"""llmgw's own ComfyUI nodes, put in every replica by the gate.
|
|
|
|
LLMGWPrepareForMesh is what an image-to-3D model wants of its picture, as
|
|
TripoSR's own pipeline (tsr/utils.py) does it: the background taken away —
|
|
by the picture's alpha where it has one, by rembg (MIT; u2net, Apache 2.0)
|
|
where it does not — the object cropped to its bounds, set square in the
|
|
middle at a ratio of the side, on grey.
|
|
"""
|
|
|
|
import numpy as np
|
|
import torch
|
|
from PIL import Image
|
|
|
|
_sessions = {}
|
|
|
|
|
|
def _rembg(img, model):
|
|
try:
|
|
import rembg
|
|
except ImportError as e: # a model that wants it names it in its nodes' requirements
|
|
raise RuntimeError("the picture has no alpha and rembg is not installed for this model: "
|
|
"add rembg and onnxruntime to its custom node requirements") from e
|
|
if model not in _sessions:
|
|
_sessions[model] = rembg.new_session(model)
|
|
return rembg.remove(img, session=_sessions[model])
|
|
|
|
|
|
def _fit(rgba, ratio):
|
|
"""tsr.utils.resize_foreground: crop to the alpha's bounds, pad square,
|
|
pad again so the object is ratio of the side."""
|
|
a = np.array(rgba)
|
|
ys, xs = np.where(a[..., 3] > 0)
|
|
if len(ys) == 0:
|
|
raise RuntimeError("nothing is left of the picture once its background is taken away")
|
|
fg = a[ys.min():ys.max() + 1, xs.min():xs.max() + 1]
|
|
size = max(fg.shape[0], fg.shape[1])
|
|
ph0, pw0 = (size - fg.shape[0]) // 2, (size - fg.shape[1]) // 2
|
|
sq = np.pad(fg, ((ph0, size - fg.shape[0] - ph0), (pw0, size - fg.shape[1] - pw0), (0, 0)))
|
|
new = int(size / ratio)
|
|
ph0, pw0 = (new - size) // 2, (new - size) // 2
|
|
return np.pad(sq, ((ph0, new - size - ph0), (pw0, new - size - pw0), (0, 0)))
|
|
|
|
|
|
class LLMGWPrepareForMesh:
|
|
@classmethod
|
|
def INPUT_TYPES(cls):
|
|
return {
|
|
"required": {
|
|
"image": ("IMAGE",),
|
|
"foreground_ratio": ("FLOAT", {"default": 0.85, "min": 0.3, "max": 1.0, "step": 0.01}),
|
|
"background": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}),
|
|
"remove_background": (["auto", "always", "never"],),
|
|
"rembg_model": (["u2net", "isnet-general-use", "u2netp"],),
|
|
},
|
|
"optional": {"mask": ("MASK",)},
|
|
}
|
|
|
|
RETURN_TYPES = ("IMAGE", "MASK")
|
|
RETURN_NAMES = ("image", "mask")
|
|
FUNCTION = "prepare"
|
|
CATEGORY = "llmgw"
|
|
|
|
def prepare(self, image, foreground_ratio, background, remove_background, rembg_model, mask=None):
|
|
out = []
|
|
for i in range(image.shape[0]):
|
|
rgb = (image[i].cpu().numpy() * 255).clip(0, 255).astype(np.uint8)
|
|
h, w = rgb.shape[:2]
|
|
alpha = None
|
|
# LoadImage's mask is 1 - alpha, and a 64x64 of zeros for a
|
|
# picture with none.
|
|
if mask is not None and remove_background != "always":
|
|
m = mask[min(i, mask.shape[0] - 1)].cpu().numpy()
|
|
if m.shape == (h, w) and m.max() > 0:
|
|
alpha = ((1.0 - m) * 255).clip(0, 255).astype(np.uint8)
|
|
if alpha is None and remove_background != "never":
|
|
rgba = np.array(_rembg(Image.fromarray(rgb), rembg_model).convert("RGBA"))
|
|
else:
|
|
if alpha is None:
|
|
alpha = np.full((h, w), 255, np.uint8)
|
|
rgba = np.dstack([rgb, alpha])
|
|
sq = _fit(rgba, foreground_ratio).astype(np.float32) / 255.0
|
|
a = sq[..., 3:4]
|
|
img = sq[..., :3] * a + (1.0 - a) * background
|
|
out.append(torch.from_numpy(np.dstack([img, a])))
|
|
# One size for the batch: the first picture's.
|
|
side = out[0].shape[0]
|
|
out = [o if o.shape[0] == side else torch.from_numpy(
|
|
np.array(Image.fromarray((o.numpy() * 255).astype(np.uint8)).resize((side, side)))).float() / 255.0
|
|
for o in out]
|
|
both = torch.stack(out)
|
|
# The picture on its background, and the object's alpha as a mask
|
|
# (1 where the object is) for a model that wants it, as Stable Fast
|
|
# 3D's sampler does.
|
|
return (both[..., :3].contiguous(), both[..., 3].contiguous())
|
|
|
|
|
|
class LLMGWSaveMesh:
|
|
"""Writes each mesh a 3D node made to output/ as a GLB and names the
|
|
files, which is how the gate finds a result. A model's own save node
|
|
may return the mesh as base64 (Stable Fast 3D's does) or as an OBJ
|
|
without its texture, and the gate reads neither."""
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(cls):
|
|
return {"required": {"mesh": ("MESH",), "filename_prefix": ("STRING", {"default": "mesh"})}}
|
|
|
|
RETURN_TYPES = ()
|
|
OUTPUT_NODE = True
|
|
FUNCTION = "save"
|
|
CATEGORY = "llmgw"
|
|
|
|
def save(self, mesh, filename_prefix="mesh"):
|
|
import os
|
|
import folder_paths
|
|
import trimesh
|
|
from trimesh.exchange import gltf
|
|
|
|
folder, name, counter, subfolder, _ = folder_paths.get_save_image_path(
|
|
filename_prefix, folder_paths.get_output_directory())
|
|
saved = []
|
|
for i, m in enumerate(mesh if isinstance(mesh, (list, tuple)) else [mesh]):
|
|
file = f"{name.replace('%batch_num%', str(i))}_{counter + i:05}_.glb"
|
|
scene = m if isinstance(m, trimesh.Scene) else trimesh.Scene(m)
|
|
with open(os.path.join(folder, file), "wb") as f:
|
|
f.write(gltf.export_glb(scene, include_normals=True))
|
|
saved.append({"filename": file, "type": "output", "subfolder": subfolder})
|
|
return {"ui": {"mesh": saved}}
|
|
|
|
|
|
NODE_CLASS_MAPPINGS = {"LLMGWPrepareForMesh": LLMGWPrepareForMesh, "LLMGWSaveMesh": LLMGWSaveMesh}
|
|
NODE_DISPLAY_NAME_MAPPINGS = {"LLMGWPrepareForMesh": "Prepare a picture for a 3D model (llmgw)",
|
|
"LLMGWSaveMesh": "Save a mesh as GLB (llmgw)"}
|