CI / test (push) Successful in 8m59s
Stable Fast 3D's own save node hands back the GLB as base64 in its UI output, which the gate does not read. LLMGWSaveMesh writes each mesh to output/ as a GLB and names it, as a picture's save node does. The prepare node also gives the object's mask now, which SF3D's sampler takes. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
133 lines
5.7 KiB
Python
133 lines
5.7 KiB
Python
"""llmgw's own ComfyUI nodes, put in every replica by the gate.
|
|
|
|
LLMGWPrepareForMesh is what an image-to-3D model wants of its picture, as
|
|
TripoSR's own pipeline (tsr/utils.py) does it: the background taken away —
|
|
by the picture's alpha where it has one, by rembg (MIT; u2net, Apache 2.0)
|
|
where it does not — the object cropped to its bounds, set square in the
|
|
middle at a ratio of the side, on grey.
|
|
"""
|
|
|
|
import numpy as np
|
|
import torch
|
|
from PIL import Image
|
|
|
|
_sessions = {}
|
|
|
|
|
|
def _rembg(img, model):
|
|
try:
|
|
import rembg
|
|
except ImportError as e: # a model that wants it names it in its nodes' requirements
|
|
raise RuntimeError("the picture has no alpha and rembg is not installed for this model: "
|
|
"add rembg and onnxruntime to its custom node requirements") from e
|
|
if model not in _sessions:
|
|
_sessions[model] = rembg.new_session(model)
|
|
return rembg.remove(img, session=_sessions[model])
|
|
|
|
|
|
def _fit(rgba, ratio):
|
|
"""tsr.utils.resize_foreground: crop to the alpha's bounds, pad square,
|
|
pad again so the object is ratio of the side."""
|
|
a = np.array(rgba)
|
|
ys, xs = np.where(a[..., 3] > 0)
|
|
if len(ys) == 0:
|
|
raise RuntimeError("nothing is left of the picture once its background is taken away")
|
|
fg = a[ys.min():ys.max() + 1, xs.min():xs.max() + 1]
|
|
size = max(fg.shape[0], fg.shape[1])
|
|
ph0, pw0 = (size - fg.shape[0]) // 2, (size - fg.shape[1]) // 2
|
|
sq = np.pad(fg, ((ph0, size - fg.shape[0] - ph0), (pw0, size - fg.shape[1] - pw0), (0, 0)))
|
|
new = int(size / ratio)
|
|
ph0, pw0 = (new - size) // 2, (new - size) // 2
|
|
return np.pad(sq, ((ph0, new - size - ph0), (pw0, new - size - pw0), (0, 0)))
|
|
|
|
|
|
class LLMGWPrepareForMesh:
|
|
@classmethod
|
|
def INPUT_TYPES(cls):
|
|
return {
|
|
"required": {
|
|
"image": ("IMAGE",),
|
|
"foreground_ratio": ("FLOAT", {"default": 0.85, "min": 0.3, "max": 1.0, "step": 0.01}),
|
|
"background": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}),
|
|
"remove_background": (["auto", "always", "never"],),
|
|
"rembg_model": (["u2net", "isnet-general-use", "u2netp"],),
|
|
},
|
|
"optional": {"mask": ("MASK",)},
|
|
}
|
|
|
|
RETURN_TYPES = ("IMAGE", "MASK")
|
|
RETURN_NAMES = ("image", "mask")
|
|
FUNCTION = "prepare"
|
|
CATEGORY = "llmgw"
|
|
|
|
def prepare(self, image, foreground_ratio, background, remove_background, rembg_model, mask=None):
|
|
out = []
|
|
for i in range(image.shape[0]):
|
|
rgb = (image[i].cpu().numpy() * 255).clip(0, 255).astype(np.uint8)
|
|
h, w = rgb.shape[:2]
|
|
alpha = None
|
|
# LoadImage's mask is 1 - alpha, and a 64x64 of zeros for a
|
|
# picture with none.
|
|
if mask is not None and remove_background != "always":
|
|
m = mask[min(i, mask.shape[0] - 1)].cpu().numpy()
|
|
if m.shape == (h, w) and m.max() > 0:
|
|
alpha = ((1.0 - m) * 255).clip(0, 255).astype(np.uint8)
|
|
if alpha is None and remove_background != "never":
|
|
rgba = np.array(_rembg(Image.fromarray(rgb), rembg_model).convert("RGBA"))
|
|
else:
|
|
if alpha is None:
|
|
alpha = np.full((h, w), 255, np.uint8)
|
|
rgba = np.dstack([rgb, alpha])
|
|
sq = _fit(rgba, foreground_ratio).astype(np.float32) / 255.0
|
|
a = sq[..., 3:4]
|
|
img = sq[..., :3] * a + (1.0 - a) * background
|
|
out.append(torch.from_numpy(np.dstack([img, a])))
|
|
# One size for the batch: the first picture's.
|
|
side = out[0].shape[0]
|
|
out = [o if o.shape[0] == side else torch.from_numpy(
|
|
np.array(Image.fromarray((o.numpy() * 255).astype(np.uint8)).resize((side, side)))).float() / 255.0
|
|
for o in out]
|
|
both = torch.stack(out)
|
|
# The picture on its background, and the object's alpha as a mask
|
|
# (1 where the object is) for a model that wants it, as Stable Fast
|
|
# 3D's sampler does.
|
|
return (both[..., :3].contiguous(), both[..., 3].contiguous())
|
|
|
|
|
|
class LLMGWSaveMesh:
|
|
"""Writes each mesh a 3D node made to output/ as a GLB and names the
|
|
files, which is how the gate finds a result. A model's own save node
|
|
may return the mesh as base64 (Stable Fast 3D's does) or as an OBJ
|
|
without its texture, and the gate reads neither."""
|
|
|
|
@classmethod
|
|
def INPUT_TYPES(cls):
|
|
return {"required": {"mesh": ("MESH",), "filename_prefix": ("STRING", {"default": "mesh"})}}
|
|
|
|
RETURN_TYPES = ()
|
|
OUTPUT_NODE = True
|
|
FUNCTION = "save"
|
|
CATEGORY = "llmgw"
|
|
|
|
def save(self, mesh, filename_prefix="mesh"):
|
|
import os
|
|
import folder_paths
|
|
import trimesh
|
|
from trimesh.exchange import gltf
|
|
|
|
folder, name, counter, subfolder, _ = folder_paths.get_save_image_path(
|
|
filename_prefix, folder_paths.get_output_directory())
|
|
saved = []
|
|
for i, m in enumerate(mesh if isinstance(mesh, (list, tuple)) else [mesh]):
|
|
file = f"{name.replace('%batch_num%', str(i))}_{counter + i:05}_.glb"
|
|
scene = m if isinstance(m, trimesh.Scene) else trimesh.Scene(m)
|
|
with open(os.path.join(folder, file), "wb") as f:
|
|
f.write(gltf.export_glb(scene, include_normals=True))
|
|
saved.append({"filename": file, "type": "output", "subfolder": subfolder})
|
|
return {"ui": {"mesh": saved}}
|
|
|
|
|
|
NODE_CLASS_MAPPINGS = {"LLMGWPrepareForMesh": LLMGWPrepareForMesh, "LLMGWSaveMesh": LLMGWSaveMesh}
|
|
NODE_DISPLAY_NAME_MAPPINGS = {"LLMGWPrepareForMesh": "Prepare a picture for a 3D model (llmgw)",
|
|
"LLMGWSaveMesh": "Save a mesh as GLB (llmgw)"}
|