A picture is made ready for a 3D model; a model asleep is listed asleep

LLMGWPrepareForMesh, llmgw's own ComfyUI node, written by the gate into
every replica's custom_nodes from the binary: TripoSR's own preparation
(tsr/utils.py) — the background taken away, by the picture's alpha where
it has one and by rembg (MIT; u2net, Apache 2.0) where it does not, the
object cropped to its bounds, set square at a ratio of the side (0.85),
on grey. A picture sent as it was had a white or black shell round the
object, whatever was behind it made into the mesh.

/v1/models listed only models with a replica up, so one that loads when
asked — triposr, flux-schnell — was invisible to the clients that would
have asked for it. Those are listed now, with loaded: false; every entry
says loaded.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-10 20:43:20 +02:00
1 parent fd3c7bda32
commit b743b11c54
8 files changed
+197

No files matched your search

+4
View File
@@ -2370,6 +2370,10 @@
"owned_by": {
"type": "string"
},
"loaded": {
"type": "boolean",
"description": "Loaded is whether a replica is up now; a model that loads when asked is listed asleep too, and its first request waits for it to load."
},
"context_length": {
"type": "integer",
"description": "ContextLength and MaxOutputTokens, where the catalog says."
+3
View File
@@ -269,6 +269,9 @@ type ModelInfo struct {
Object string `json:"object"`
Created int64 `json:"created"`
OwnedBy string `json:"owned_by"`
// Loaded is whether a replica is up now; a model that loads when asked
// is listed asleep too, and its first request waits for it to load.
Loaded bool `json:"loaded"`
// ContextLength and MaxOutputTokens, where the catalog says.
ContextLength int `json:"context_length,omitempty"`
MaxOutputTokens int `json:"max_output_tokens,omitempty"`
+3
View File
@@ -99,6 +99,9 @@ func RunGate(ctx context.Context, cfg GateConfig) error {
if out == nil {
out = os.Stdout
}
if err := installOwnNodes(state); err != nil {
return fmt.Errorf("llmgw's own ComfyUI nodes: %w", err)
}
nodes, err := parseNodes(os.Getenv(NodesEnv))
if err != nil {
return err
+11
View File
@@ -250,3 +250,14 @@ func TestComfyUIKeepsWritingWhenWhatTheGateWritesToIsGone(t *testing.T) {
t.Fatalf("ComfyUI answered %d times; output it could not hand on stopped it", ok)
}
}
func TestLLMGWsOwnNodesArePutInEveryReplica(t *testing.T) {
state := t.TempDir()
if err := installOwnNodes(state); err != nil {
t.Fatal(err)
}
b, err := os.ReadFile(filepath.Join(state, "custom_nodes", "llmgw_nodes", "__init__.py"))
if err != nil || !strings.Contains(string(b), "LLMGWPrepareForMesh") {
t.Fatalf("%v %.80s", err, b)
}
}
+25
View File
@@ -2,8 +2,10 @@ package comfy
import (
"context"
"embed"
"encoding/json"
"fmt"
"io/fs"
"os"
"os/exec"
"path"
@@ -25,6 +27,29 @@ type Node struct {
Requirements []string `json:"requirements,omitempty"`
}
//go:embed all:nodes
var ownNodes embed.FS
// installOwnNodes writes llmgw's own nodes into a replica's custom_nodes:
// what every workflow may use, from the binary rather than a repository,
// written again at every start so a replica runs the gate's own version.
func installOwnNodes(state string) error {
return fs.WalkDir(ownNodes, "nodes", func(p string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
dst := filepath.Join(state, "custom_nodes", strings.TrimPrefix(p, "nodes"))
if d.IsDir() {
return os.MkdirAll(dst, 0o755)
}
b, err := ownNodes.ReadFile(p)
if err != nil {
return err
}
return os.WriteFile(dst, b, 0o644) //nolint:gosec // code ComfyUI reads, not secret
})
}
// NodesEnv is where the agent hands the gate a model's custom nodes.
const NodesEnv = "LLMGW_COMFYUI_NODES"
@@ -0,0 +1,93 @@
"""llmgw's own ComfyUI nodes, put in every replica by the gate.
LLMGWPrepareForMesh is what an image-to-3D model wants of its picture, as
TripoSR's own pipeline (tsr/utils.py) does it: the background taken away —
by the picture's alpha where it has one, by rembg (MIT; u2net, Apache 2.0)
where it does not — the object cropped to its bounds, set square in the
middle at a ratio of the side, on grey.
"""
import numpy as np
import torch
from PIL import Image
_sessions = {}
def _rembg(img, model):
try:
import rembg
except ImportError as e: # a model that wants it names it in its nodes' requirements
raise RuntimeError("the picture has no alpha and rembg is not installed for this model: "
"add rembg and onnxruntime to its custom node requirements") from e
if model not in _sessions:
_sessions[model] = rembg.new_session(model)
return rembg.remove(img, session=_sessions[model])
def _fit(rgba, ratio):
"""tsr.utils.resize_foreground: crop to the alpha's bounds, pad square,
pad again so the object is ratio of the side."""
a = np.array(rgba)
ys, xs = np.where(a[..., 3] > 0)
if len(ys) == 0:
raise RuntimeError("nothing is left of the picture once its background is taken away")
fg = a[ys.min():ys.max() + 1, xs.min():xs.max() + 1]
size = max(fg.shape[0], fg.shape[1])
ph0, pw0 = (size - fg.shape[0]) // 2, (size - fg.shape[1]) // 2
sq = np.pad(fg, ((ph0, size - fg.shape[0] - ph0), (pw0, size - fg.shape[1] - pw0), (0, 0)))
new = int(size / ratio)
ph0, pw0 = (new - size) // 2, (new - size) // 2
return np.pad(sq, ((ph0, new - size - ph0), (pw0, new - size - pw0), (0, 0)))
class LLMGWPrepareForMesh:
@classmethod
def INPUT_TYPES(cls):
return {
"required": {
"image": ("IMAGE",),
"foreground_ratio": ("FLOAT", {"default": 0.85, "min": 0.3, "max": 1.0, "step": 0.01}),
"background": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}),
"remove_background": (["auto", "always", "never"],),
"rembg_model": (["u2net", "isnet-general-use", "u2netp"],),
},
"optional": {"mask": ("MASK",)},
}
RETURN_TYPES = ("IMAGE",)
FUNCTION = "prepare"
CATEGORY = "llmgw"
def prepare(self, image, foreground_ratio, background, remove_background, rembg_model, mask=None):
out = []
for i in range(image.shape[0]):
rgb = (image[i].cpu().numpy() * 255).clip(0, 255).astype(np.uint8)
h, w = rgb.shape[:2]
alpha = None
# LoadImage's mask is 1 - alpha, and a 64x64 of zeros for a
# picture with none.
if mask is not None and remove_background != "always":
m = mask[min(i, mask.shape[0] - 1)].cpu().numpy()
if m.shape == (h, w) and m.max() > 0:
alpha = ((1.0 - m) * 255).clip(0, 255).astype(np.uint8)
if alpha is None and remove_background != "never":
rgba = np.array(_rembg(Image.fromarray(rgb), rembg_model).convert("RGBA"))
else:
if alpha is None:
alpha = np.full((h, w), 255, np.uint8)
rgba = np.dstack([rgb, alpha])
sq = _fit(rgba, foreground_ratio).astype(np.float32) / 255.0
a = sq[..., 3:4]
img = sq[..., :3] * a + (1.0 - a) * background
out.append(torch.from_numpy(img))
# One size for the batch: the first picture's.
side = out[0].shape[0]
out = [o if o.shape[0] == side else torch.from_numpy(
np.array(Image.fromarray((o.numpy() * 255).astype(np.uint8)).resize((side, side)))).float() / 255.0
for o in out]
return (torch.stack(out),)
NODE_CLASS_MAPPINGS = {"LLMGWPrepareForMesh": LLMGWPrepareForMesh}
NODE_DISPLAY_NAME_MAPPINGS = {"LLMGWPrepareForMesh": "Prepare a picture for a 3D model (llmgw)"}
+18
View File
@@ -10,6 +10,7 @@ import (
"fmt"
llmgw "gitea.mynodes.xyz/ilker/llmgw"
"io"
"maps"
"net/http"
"slices"
"strconv"
@@ -1108,6 +1109,16 @@ func (r *Router) handleModels(w http.ResponseWriter, req *http.Request) {
seen[m] = struct{}{}
}
}
// A model that loads when asked has no replica until somebody asks,
// and asking is how it gets one: listing only what runs hid it from
// exactly the clients that would have asked — a 3D model, a picture
// model, asleep between uses. Said with loaded: false.
loaded := maps.Clone(seen)
for name, m := range r.modelMap() {
if name == m.Name && m.Scaling.OnDemand() && !m.IsExternal() {
seen[name] = struct{}{}
}
}
// Catalog aliases are usable names too, so clients should see them.
for name := range r.modelMap() {
if _, ok := seen[r.modelMap()[name].Name]; ok {
@@ -1135,8 +1146,15 @@ func (r *Router) handleModels(w http.ResponseWriter, req *http.Request) {
if r.modelMap()[n].Simulated() {
owner = "llmgw-sim"
}
_, up := loaded[n]
if m := r.modelMap()[n]; !up && m.Name != "" {
_, up = loaded[m.Name]
}
entry := map[string]any{
"id": n, "object": "model", "created": time.Now().Unix(), "owned_by": owner,
// Not OpenAI's: whether a replica is up now, or the first
// request waits for one to load.
"loaded": up,
}
// Not OpenAI's either: what a federated peer reading this list needs
// to leave out — a second name for a model already listed, and a
+40
View File
@@ -361,3 +361,43 @@ func TestThe3DPageMakesAJobAndShowsItsMeshToItsOwnerOnly(t *testing.T) {
t.Errorf("another project's person read the job: %d", rec.Code)
}
}
func TestAModelAsleepUntilAskedIsListedAndSaidAsleep(t *testing.T) {
// triposr and flux-schnell load when asked and stop when idle; a client
// asking /v1/models which model makes 3D models found neither while
// they slept, and so never asked.
cfg := baseConfig()
asleep := catModel("triposr", nil)
asleep.Scaling = catalog.Scaling{Min: 0, Max: 1}
asleep.ComfyUI = &catalog.ComfyUI{Output: catalog.OutputMeshes, Workflow: "w.json"}
asleep.Profiles = []catalog.Profile{{Name: "gpu", Engine: catalog.EngineComfyUI, GPUClass: "6000-24g", TP: 1}}
cfg.Models = []catalog.Model{asleep, catModel("chat", nil)}
rt := newTestRouter(t, cfg)
e := newFakeEngine(t, "chat")
rt.SetEndpoints(t.Context(), []endpoint.Info{{ID: "c", URL: e.srv.URL, Models: []string{"chat"}}})
rec := httptest.NewRecorder()
rt.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/v1/models", nil))
var list struct {
Data []struct {
ID string `json:"id"`
Loaded bool `json:"loaded"`
Capabilities map[string]bool `json:"capabilities"`
} `json:"data"`
}
if err := json.Unmarshal(rec.Body.Bytes(), &list); err != nil {
t.Fatal(err)
}
got := map[string]bool{}
for _, m := range list.Data {
got[m.ID] = m.Loaded
if m.ID == "triposr" && !m.Capabilities["meshes"] {
t.Error("triposr is not said to make meshes")
}
}
if loaded, ok := got["triposr"]; !ok || loaded {
t.Errorf("triposr listed %v, loaded %v: wanted listed and asleep", ok, loaded)
}
if !got["chat"] {
t.Error("a model serving is not said loaded")
}
}