Files
llmgw/internal/comfy/nodes/llmgw_nodes/__init__.py
T
Ilker ManapandClaude Opus 5.5 cfc636e328
CI / test (push) Failing after 6m28s
A mesh's base colour can be brought to the picture it was made from
Stable Fast 3D takes the light out of the picture it sees, and a dark
basalt (median 0.17) came back a light stone dimly lit (0.67). The save
node takes albedo_match, the picture and its mask, and scales the base
colour texture in linear light by the ratio of the object's medians, so
marks keep their contrast; texels nothing maps to are left alone. On the
game team's rock it is 0.18.

A workflow's {{name|false}} default now fills a BOOLEAN input with a
boolean rather than the word.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-14 00:51:06 +02:00

184 lines
8.0 KiB
Python

"""llmgw's own ComfyUI nodes, put in every replica by the gate.
LLMGWPrepareForMesh is what an image-to-3D model wants of its picture, as
TripoSR's own pipeline (tsr/utils.py) does it: the background taken away —
by the picture's alpha where it has one, by rembg (MIT; u2net, Apache 2.0)
where it does not — the object cropped to its bounds, set square in the
middle at a ratio of the side, on grey.
"""
import numpy as np
import torch
from PIL import Image
_sessions = {}
def _rembg(img, model):
try:
import rembg
except ImportError as e: # a model that wants it names it in its nodes' requirements
raise RuntimeError("the picture has no alpha and rembg is not installed for this model: "
"add rembg and onnxruntime to its custom node requirements") from e
if model not in _sessions:
_sessions[model] = rembg.new_session(model)
return rembg.remove(img, session=_sessions[model])
def _fit(rgba, ratio):
"""tsr.utils.resize_foreground: crop to the alpha's bounds, pad square,
pad again so the object is ratio of the side."""
a = np.array(rgba)
ys, xs = np.where(a[..., 3] > 0)
if len(ys) == 0:
raise RuntimeError("nothing is left of the picture once its background is taken away")
fg = a[ys.min():ys.max() + 1, xs.min():xs.max() + 1]
size = max(fg.shape[0], fg.shape[1])
ph0, pw0 = (size - fg.shape[0]) // 2, (size - fg.shape[1]) // 2
sq = np.pad(fg, ((ph0, size - fg.shape[0] - ph0), (pw0, size - fg.shape[1] - pw0), (0, 0)))
new = int(size / ratio)
ph0, pw0 = (new - size) // 2, (new - size) // 2
return np.pad(sq, ((ph0, new - size - ph0), (pw0, new - size - pw0), (0, 0)))
class LLMGWPrepareForMesh:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"image": ("IMAGE",),
"foreground_ratio": ("FLOAT", {"default": 0.85, "min": 0.3, "max": 1.0, "step": 0.01}),
"background": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}),
"remove_background": (["auto", "always", "never"],),
"rembg_model": (["u2net", "isnet-general-use", "u2netp"],),
},
"optional": {"mask": ("MASK",)},
}
RETURN_TYPES = ("IMAGE", "MASK")
RETURN_NAMES = ("image", "mask")
FUNCTION = "prepare"
CATEGORY = "llmgw"
def prepare(self, image, foreground_ratio, background, remove_background, rembg_model, mask=None):
out = []
for i in range(image.shape[0]):
rgb = (image[i].cpu().numpy() * 255).clip(0, 255).astype(np.uint8)
h, w = rgb.shape[:2]
alpha = None
# LoadImage's mask is 1 - alpha, and a 64x64 of zeros for a
# picture with none.
if mask is not None and remove_background != "always":
m = mask[min(i, mask.shape[0] - 1)].cpu().numpy()
if m.shape == (h, w) and m.max() > 0:
alpha = ((1.0 - m) * 255).clip(0, 255).astype(np.uint8)
if alpha is None and remove_background != "never":
rgba = np.array(_rembg(Image.fromarray(rgb), rembg_model).convert("RGBA"))
else:
if alpha is None:
alpha = np.full((h, w), 255, np.uint8)
rgba = np.dstack([rgb, alpha])
sq = _fit(rgba, foreground_ratio).astype(np.float32) / 255.0
a = sq[..., 3:4]
img = sq[..., :3] * a + (1.0 - a) * background
out.append(torch.from_numpy(np.dstack([img, a])))
# One size for the batch: the first picture's.
side = out[0].shape[0]
out = [o if o.shape[0] == side else torch.from_numpy(
np.array(Image.fromarray((o.numpy() * 255).astype(np.uint8)).resize((side, side)))).float() / 255.0
for o in out]
both = torch.stack(out)
# The picture on its background, and the object's alpha as a mask
# (1 where the object is) for a model that wants it, as Stable Fast
# 3D's sampler does.
return (both[..., :3].contiguous(), both[..., 3].contiguous())
def _to_linear(x):
return np.where(x <= 0.04045, x / 12.92, ((x + 0.055) / 1.055) ** 2.4)
def _to_srgb(x):
return np.where(x <= 0.0031308, x * 12.92, 1.055 * np.power(np.clip(x, 0, None), 1 / 2.4) - 0.055)
def _match_albedo(tex, reference, mask):
"""The texture's brightness brought to the picture's: Stable Fast 3D
takes the light out of what it sees, and a dark object it reads as a
light one dimly lit — a basalt of 0.17 came back 0.67. Scaled in linear
light by the ratio of the medians over the object, so its marks keep
their contrast; texels nothing maps to (black) are left as they are."""
t = np.asarray(tex.convert("RGB")).astype(np.float64) / 255.0
used = t.max(-1) > 2 / 255
ref = reference.cpu().numpy().astype(np.float64)
obj = mask.cpu().numpy() > 0.5 if mask is not None else np.ones(ref.shape[:2], bool)
if used.sum() < 16 or obj.sum() < 16:
return tex
lum = lambda a: a @ np.array([0.2126, 0.7152, 0.0722])
want = np.median(lum(_to_linear(ref[obj])))
have = np.median(lum(_to_linear(t[used])))
if have <= 0:
return tex
k = float(np.clip(want / have, 0.05, 20.0))
out = t.copy()
out[used] = np.clip(_to_srgb(_to_linear(t[used]) * k), 0, 1)
img = Image.fromarray((out * 255 + 0.5).astype(np.uint8))
img.format = tex.format
return img
class LLMGWSaveMesh:
"""Writes each mesh a 3D node made to output/ as a GLB and names the
files, which is how the gate finds a result. A model's own save node
may return the mesh as base64 (Stable Fast 3D's does) or as an OBJ
without its texture, and the gate reads neither.
With albedo_match and the picture the mesh was made from (and its
mask), the mesh's base colour is brought to the picture's brightness."""
@classmethod
def INPUT_TYPES(cls):
return {"required": {"mesh": ("MESH",), "filename_prefix": ("STRING", {"default": "mesh"})},
"optional": {"albedo_match": ("BOOLEAN", {"default": False}),
"reference": ("IMAGE",), "reference_mask": ("MASK",)}}
RETURN_TYPES = ()
OUTPUT_NODE = True
FUNCTION = "save"
CATEGORY = "llmgw"
def save(self, mesh, filename_prefix="mesh", albedo_match=False, reference=None, reference_mask=None):
import os
import folder_paths
import trimesh
from trimesh.exchange import gltf
folder, name, counter, subfolder, _ = folder_paths.get_save_image_path(
filename_prefix, folder_paths.get_output_directory())
saved = []
for i, m in enumerate(mesh if isinstance(mesh, (list, tuple)) else [mesh]):
if albedo_match and reference is not None:
_match_scene(m, reference[min(i, reference.shape[0] - 1)],
None if reference_mask is None else reference_mask[min(i, reference_mask.shape[0] - 1)])
file = f"{name.replace('%batch_num%', str(i))}_{counter + i:05}_.glb"
scene = m if isinstance(m, trimesh.Scene) else trimesh.Scene(m)
with open(os.path.join(folder, file), "wb") as f:
f.write(gltf.export_glb(scene, include_normals=True))
saved.append({"filename": file, "type": "output", "subfolder": subfolder})
return {"ui": {"mesh": saved}}
def _match_scene(m, reference, mask):
"""_match_albedo on every base colour texture of a mesh or scene."""
geoms = m.geometry.values() if hasattr(m, "geometry") else [m]
for g in geoms:
mat = getattr(getattr(g, "visual", None), "material", None)
tex = getattr(mat, "baseColorTexture", None)
if tex is not None:
mat.baseColorTexture = _match_albedo(tex, reference, mask)
NODE_CLASS_MAPPINGS = {"LLMGWPrepareForMesh": LLMGWPrepareForMesh, "LLMGWSaveMesh": LLMGWSaveMesh}
NODE_DISPLAY_NAME_MAPPINGS = {"LLMGWPrepareForMesh": "Prepare a picture for a 3D model (llmgw)",
"LLMGWSaveMesh": "Save a mesh as GLB (llmgw)"}