A picture is made ready for a 3D model; a model asleep is listed asleep
LLMGWPrepareForMesh, llmgw's own ComfyUI node, written by the gate into every replica's custom_nodes from the binary: TripoSR's own preparation (tsr/utils.py) — the background taken away, by the picture's alpha where it has one and by rembg (MIT; u2net, Apache 2.0) where it does not, the object cropped to its bounds, set square at a ratio of the side (0.85), on grey. A picture sent as it was had a white or black shell round the object, whatever was behind it made into the mesh. /v1/models listed only models with a replica up, so one that loads when asked — triposr, flux-schnell — was invisible to the clients that would have asked for it. Those are listed now, with loaded: false; every entry says loaded. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
fd3c7bda32
commit
b743b11c54
8 files changed
+197
No files matched your search
@@ -2370,6 +2370,10 @@
|
||||
"owned_by": {
|
||||
"type": "string"
|
||||
},
|
||||
"loaded": {
|
||||
"type": "boolean",
|
||||
"description": "Loaded is whether a replica is up now; a model that loads when asked is listed asleep too, and its first request waits for it to load."
|
||||
},
|
||||
"context_length": {
|
||||
"type": "integer",
|
||||
"description": "ContextLength and MaxOutputTokens, where the catalog says."
|
||||
|
||||
@@ -269,6 +269,9 @@ type ModelInfo struct {
|
||||
Object string `json:"object"`
|
||||
Created int64 `json:"created"`
|
||||
OwnedBy string `json:"owned_by"`
|
||||
// Loaded is whether a replica is up now; a model that loads when asked
|
||||
// is listed asleep too, and its first request waits for it to load.
|
||||
Loaded bool `json:"loaded"`
|
||||
// ContextLength and MaxOutputTokens, where the catalog says.
|
||||
ContextLength int `json:"context_length,omitempty"`
|
||||
MaxOutputTokens int `json:"max_output_tokens,omitempty"`
|
||||
|
||||
@@ -99,6 +99,9 @@ func RunGate(ctx context.Context, cfg GateConfig) error {
|
||||
if out == nil {
|
||||
out = os.Stdout
|
||||
}
|
||||
if err := installOwnNodes(state); err != nil {
|
||||
return fmt.Errorf("llmgw's own ComfyUI nodes: %w", err)
|
||||
}
|
||||
nodes, err := parseNodes(os.Getenv(NodesEnv))
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -250,3 +250,14 @@ func TestComfyUIKeepsWritingWhenWhatTheGateWritesToIsGone(t *testing.T) {
|
||||
t.Fatalf("ComfyUI answered %d times; output it could not hand on stopped it", ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLLMGWsOwnNodesArePutInEveryReplica(t *testing.T) {
|
||||
state := t.TempDir()
|
||||
if err := installOwnNodes(state); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(state, "custom_nodes", "llmgw_nodes", "__init__.py"))
|
||||
if err != nil || !strings.Contains(string(b), "LLMGWPrepareForMesh") {
|
||||
t.Fatalf("%v %.80s", err, b)
|
||||
}
|
||||
}
|
||||
@@ -2,8 +2,10 @@ package comfy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
@@ -25,6 +27,29 @@ type Node struct {
|
||||
Requirements []string `json:"requirements,omitempty"`
|
||||
}
|
||||
|
||||
//go:embed all:nodes
|
||||
var ownNodes embed.FS
|
||||
|
||||
// installOwnNodes writes llmgw's own nodes into a replica's custom_nodes:
|
||||
// what every workflow may use, from the binary rather than a repository,
|
||||
// written again at every start so a replica runs the gate's own version.
|
||||
func installOwnNodes(state string) error {
|
||||
return fs.WalkDir(ownNodes, "nodes", func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
dst := filepath.Join(state, "custom_nodes", strings.TrimPrefix(p, "nodes"))
|
||||
if d.IsDir() {
|
||||
return os.MkdirAll(dst, 0o755)
|
||||
}
|
||||
b, err := ownNodes.ReadFile(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(dst, b, 0o644) //nolint:gosec // code ComfyUI reads, not secret
|
||||
})
|
||||
}
|
||||
|
||||
// NodesEnv is where the agent hands the gate a model's custom nodes.
|
||||
const NodesEnv = "LLMGW_COMFYUI_NODES"
|
||||
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
"""llmgw's own ComfyUI nodes, put in every replica by the gate.
|
||||
|
||||
LLMGWPrepareForMesh is what an image-to-3D model wants of its picture, as
|
||||
TripoSR's own pipeline (tsr/utils.py) does it: the background taken away —
|
||||
by the picture's alpha where it has one, by rembg (MIT; u2net, Apache 2.0)
|
||||
where it does not — the object cropped to its bounds, set square in the
|
||||
middle at a ratio of the side, on grey.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
_sessions = {}
|
||||
|
||||
|
||||
def _rembg(img, model):
|
||||
try:
|
||||
import rembg
|
||||
except ImportError as e: # a model that wants it names it in its nodes' requirements
|
||||
raise RuntimeError("the picture has no alpha and rembg is not installed for this model: "
|
||||
"add rembg and onnxruntime to its custom node requirements") from e
|
||||
if model not in _sessions:
|
||||
_sessions[model] = rembg.new_session(model)
|
||||
return rembg.remove(img, session=_sessions[model])
|
||||
|
||||
|
||||
def _fit(rgba, ratio):
|
||||
"""tsr.utils.resize_foreground: crop to the alpha's bounds, pad square,
|
||||
pad again so the object is ratio of the side."""
|
||||
a = np.array(rgba)
|
||||
ys, xs = np.where(a[..., 3] > 0)
|
||||
if len(ys) == 0:
|
||||
raise RuntimeError("nothing is left of the picture once its background is taken away")
|
||||
fg = a[ys.min():ys.max() + 1, xs.min():xs.max() + 1]
|
||||
size = max(fg.shape[0], fg.shape[1])
|
||||
ph0, pw0 = (size - fg.shape[0]) // 2, (size - fg.shape[1]) // 2
|
||||
sq = np.pad(fg, ((ph0, size - fg.shape[0] - ph0), (pw0, size - fg.shape[1] - pw0), (0, 0)))
|
||||
new = int(size / ratio)
|
||||
ph0, pw0 = (new - size) // 2, (new - size) // 2
|
||||
return np.pad(sq, ((ph0, new - size - ph0), (pw0, new - size - pw0), (0, 0)))
|
||||
|
||||
|
||||
class LLMGWPrepareForMesh:
|
||||
@classmethod
|
||||
def INPUT_TYPES(cls):
|
||||
return {
|
||||
"required": {
|
||||
"image": ("IMAGE",),
|
||||
"foreground_ratio": ("FLOAT", {"default": 0.85, "min": 0.3, "max": 1.0, "step": 0.01}),
|
||||
"background": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}),
|
||||
"remove_background": (["auto", "always", "never"],),
|
||||
"rembg_model": (["u2net", "isnet-general-use", "u2netp"],),
|
||||
},
|
||||
"optional": {"mask": ("MASK",)},
|
||||
}
|
||||
|
||||
RETURN_TYPES = ("IMAGE",)
|
||||
FUNCTION = "prepare"
|
||||
CATEGORY = "llmgw"
|
||||
|
||||
def prepare(self, image, foreground_ratio, background, remove_background, rembg_model, mask=None):
|
||||
out = []
|
||||
for i in range(image.shape[0]):
|
||||
rgb = (image[i].cpu().numpy() * 255).clip(0, 255).astype(np.uint8)
|
||||
h, w = rgb.shape[:2]
|
||||
alpha = None
|
||||
# LoadImage's mask is 1 - alpha, and a 64x64 of zeros for a
|
||||
# picture with none.
|
||||
if mask is not None and remove_background != "always":
|
||||
m = mask[min(i, mask.shape[0] - 1)].cpu().numpy()
|
||||
if m.shape == (h, w) and m.max() > 0:
|
||||
alpha = ((1.0 - m) * 255).clip(0, 255).astype(np.uint8)
|
||||
if alpha is None and remove_background != "never":
|
||||
rgba = np.array(_rembg(Image.fromarray(rgb), rembg_model).convert("RGBA"))
|
||||
else:
|
||||
if alpha is None:
|
||||
alpha = np.full((h, w), 255, np.uint8)
|
||||
rgba = np.dstack([rgb, alpha])
|
||||
sq = _fit(rgba, foreground_ratio).astype(np.float32) / 255.0
|
||||
a = sq[..., 3:4]
|
||||
img = sq[..., :3] * a + (1.0 - a) * background
|
||||
out.append(torch.from_numpy(img))
|
||||
# One size for the batch: the first picture's.
|
||||
side = out[0].shape[0]
|
||||
out = [o if o.shape[0] == side else torch.from_numpy(
|
||||
np.array(Image.fromarray((o.numpy() * 255).astype(np.uint8)).resize((side, side)))).float() / 255.0
|
||||
for o in out]
|
||||
return (torch.stack(out),)
|
||||
|
||||
|
||||
NODE_CLASS_MAPPINGS = {"LLMGWPrepareForMesh": LLMGWPrepareForMesh}
|
||||
NODE_DISPLAY_NAME_MAPPINGS = {"LLMGWPrepareForMesh": "Prepare a picture for a 3D model (llmgw)"}
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"fmt"
|
||||
llmgw "gitea.mynodes.xyz/ilker/llmgw"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strconv"
|
||||
@@ -1108,6 +1109,16 @@ func (r *Router) handleModels(w http.ResponseWriter, req *http.Request) {
|
||||
seen[m] = struct{}{}
|
||||
}
|
||||
}
|
||||
// A model that loads when asked has no replica until somebody asks,
|
||||
// and asking is how it gets one: listing only what runs hid it from
|
||||
// exactly the clients that would have asked — a 3D model, a picture
|
||||
// model, asleep between uses. Said with loaded: false.
|
||||
loaded := maps.Clone(seen)
|
||||
for name, m := range r.modelMap() {
|
||||
if name == m.Name && m.Scaling.OnDemand() && !m.IsExternal() {
|
||||
seen[name] = struct{}{}
|
||||
}
|
||||
}
|
||||
// Catalog aliases are usable names too, so clients should see them.
|
||||
for name := range r.modelMap() {
|
||||
if _, ok := seen[r.modelMap()[name].Name]; ok {
|
||||
@@ -1135,8 +1146,15 @@ func (r *Router) handleModels(w http.ResponseWriter, req *http.Request) {
|
||||
if r.modelMap()[n].Simulated() {
|
||||
owner = "llmgw-sim"
|
||||
}
|
||||
_, up := loaded[n]
|
||||
if m := r.modelMap()[n]; !up && m.Name != "" {
|
||||
_, up = loaded[m.Name]
|
||||
}
|
||||
entry := map[string]any{
|
||||
"id": n, "object": "model", "created": time.Now().Unix(), "owned_by": owner,
|
||||
// Not OpenAI's: whether a replica is up now, or the first
|
||||
// request waits for one to load.
|
||||
"loaded": up,
|
||||
}
|
||||
// Not OpenAI's either: what a federated peer reading this list needs
|
||||
// to leave out — a second name for a model already listed, and a
|
||||
|
||||
@@ -361,3 +361,43 @@ func TestThe3DPageMakesAJobAndShowsItsMeshToItsOwnerOnly(t *testing.T) {
|
||||
t.Errorf("another project's person read the job: %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModelAsleepUntilAskedIsListedAndSaidAsleep(t *testing.T) {
|
||||
// triposr and flux-schnell load when asked and stop when idle; a client
|
||||
// asking /v1/models which model makes 3D models found neither while
|
||||
// they slept, and so never asked.
|
||||
cfg := baseConfig()
|
||||
asleep := catModel("triposr", nil)
|
||||
asleep.Scaling = catalog.Scaling{Min: 0, Max: 1}
|
||||
asleep.ComfyUI = &catalog.ComfyUI{Output: catalog.OutputMeshes, Workflow: "w.json"}
|
||||
asleep.Profiles = []catalog.Profile{{Name: "gpu", Engine: catalog.EngineComfyUI, GPUClass: "6000-24g", TP: 1}}
|
||||
cfg.Models = []catalog.Model{asleep, catModel("chat", nil)}
|
||||
rt := newTestRouter(t, cfg)
|
||||
e := newFakeEngine(t, "chat")
|
||||
rt.SetEndpoints(t.Context(), []endpoint.Info{{ID: "c", URL: e.srv.URL, Models: []string{"chat"}}})
|
||||
rec := httptest.NewRecorder()
|
||||
rt.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/v1/models", nil))
|
||||
var list struct {
|
||||
Data []struct {
|
||||
ID string `json:"id"`
|
||||
Loaded bool `json:"loaded"`
|
||||
Capabilities map[string]bool `json:"capabilities"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &list); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := map[string]bool{}
|
||||
for _, m := range list.Data {
|
||||
got[m.ID] = m.Loaded
|
||||
if m.ID == "triposr" && !m.Capabilities["meshes"] {
|
||||
t.Error("triposr is not said to make meshes")
|
||||
}
|
||||
}
|
||||
if loaded, ok := got["triposr"]; !ok || loaded {
|
||||
t.Errorf("triposr listed %v, loaded %v: wanted listed and asleep", ok, loaded)
|
||||
}
|
||||
if !got["chat"] {
|
||||
t.Error("a model serving is not said loaded")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user