diff --git a/docs/client-api.openapi.json b/docs/client-api.openapi.json index 689f465..441e81a 100644 --- a/docs/client-api.openapi.json +++ b/docs/client-api.openapi.json @@ -2370,6 +2370,10 @@ "owned_by": { "type": "string" }, + "loaded": { + "type": "boolean", + "description": "Loaded is whether a replica is up now; a model that loads when asked is listed asleep too, and its first request waits for it to load." + }, "context_length": { "type": "integer", "description": "ContextLength and MaxOutputTokens, where the catalog says." diff --git a/internal/clientapi/clientapi.go b/internal/clientapi/clientapi.go index cf22bf9..4002552 100644 --- a/internal/clientapi/clientapi.go +++ b/internal/clientapi/clientapi.go @@ -269,6 +269,9 @@ type ModelInfo struct { Object string `json:"object"` Created int64 `json:"created"` OwnedBy string `json:"owned_by"` + // Loaded is whether a replica is up now; a model that loads when asked + // is listed asleep too, and its first request waits for it to load. + Loaded bool `json:"loaded"` // ContextLength and MaxOutputTokens, where the catalog says. ContextLength int `json:"context_length,omitempty"` MaxOutputTokens int `json:"max_output_tokens,omitempty"` diff --git a/internal/comfy/gate.go b/internal/comfy/gate.go index 42bb80f..7e696d7 100644 --- a/internal/comfy/gate.go +++ b/internal/comfy/gate.go @@ -99,6 +99,9 @@ func RunGate(ctx context.Context, cfg GateConfig) error { if out == nil { out = os.Stdout } + if err := installOwnNodes(state); err != nil { + return fmt.Errorf("llmgw's own ComfyUI nodes: %w", err) + } nodes, err := parseNodes(os.Getenv(NodesEnv)) if err != nil { return err diff --git a/internal/comfy/gate_test.go b/internal/comfy/gate_test.go index 38aebd3..632be83 100644 --- a/internal/comfy/gate_test.go +++ b/internal/comfy/gate_test.go @@ -250,3 +250,14 @@ func TestComfyUIKeepsWritingWhenWhatTheGateWritesToIsGone(t *testing.T) { t.Fatalf("ComfyUI answered %d times; output it could not hand on stopped it", ok) } } + +func TestLLMGWsOwnNodesArePutInEveryReplica(t *testing.T) { + state := t.TempDir() + if err := installOwnNodes(state); err != nil { + t.Fatal(err) + } + b, err := os.ReadFile(filepath.Join(state, "custom_nodes", "llmgw_nodes", "__init__.py")) + if err != nil || !strings.Contains(string(b), "LLMGWPrepareForMesh") { + t.Fatalf("%v %.80s", err, b) + } +} diff --git a/internal/comfy/nodes.go b/internal/comfy/nodes.go index 1e0ffb1..4d4edaf 100644 --- a/internal/comfy/nodes.go +++ b/internal/comfy/nodes.go @@ -2,8 +2,10 @@ package comfy import ( "context" + "embed" "encoding/json" "fmt" + "io/fs" "os" "os/exec" "path" @@ -25,6 +27,29 @@ type Node struct { Requirements []string `json:"requirements,omitempty"` } +//go:embed all:nodes +var ownNodes embed.FS + +// installOwnNodes writes llmgw's own nodes into a replica's custom_nodes: +// what every workflow may use, from the binary rather than a repository, +// written again at every start so a replica runs the gate's own version. +func installOwnNodes(state string) error { + return fs.WalkDir(ownNodes, "nodes", func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + dst := filepath.Join(state, "custom_nodes", strings.TrimPrefix(p, "nodes")) + if d.IsDir() { + return os.MkdirAll(dst, 0o755) + } + b, err := ownNodes.ReadFile(p) + if err != nil { + return err + } + return os.WriteFile(dst, b, 0o644) //nolint:gosec // code ComfyUI reads, not secret + }) +} + // NodesEnv is where the agent hands the gate a model's custom nodes. const NodesEnv = "LLMGW_COMFYUI_NODES" diff --git a/internal/comfy/nodes/llmgw_nodes/__init__.py b/internal/comfy/nodes/llmgw_nodes/__init__.py new file mode 100644 index 0000000..baae17a --- /dev/null +++ b/internal/comfy/nodes/llmgw_nodes/__init__.py @@ -0,0 +1,93 @@ +"""llmgw's own ComfyUI nodes, put in every replica by the gate. + +LLMGWPrepareForMesh is what an image-to-3D model wants of its picture, as +TripoSR's own pipeline (tsr/utils.py) does it: the background taken away — +by the picture's alpha where it has one, by rembg (MIT; u2net, Apache 2.0) +where it does not — the object cropped to its bounds, set square in the +middle at a ratio of the side, on grey. +""" + +import numpy as np +import torch +from PIL import Image + +_sessions = {} + + +def _rembg(img, model): + try: + import rembg + except ImportError as e: # a model that wants it names it in its nodes' requirements + raise RuntimeError("the picture has no alpha and rembg is not installed for this model: " + "add rembg and onnxruntime to its custom node requirements") from e + if model not in _sessions: + _sessions[model] = rembg.new_session(model) + return rembg.remove(img, session=_sessions[model]) + + +def _fit(rgba, ratio): + """tsr.utils.resize_foreground: crop to the alpha's bounds, pad square, + pad again so the object is ratio of the side.""" + a = np.array(rgba) + ys, xs = np.where(a[..., 3] > 0) + if len(ys) == 0: + raise RuntimeError("nothing is left of the picture once its background is taken away") + fg = a[ys.min():ys.max() + 1, xs.min():xs.max() + 1] + size = max(fg.shape[0], fg.shape[1]) + ph0, pw0 = (size - fg.shape[0]) // 2, (size - fg.shape[1]) // 2 + sq = np.pad(fg, ((ph0, size - fg.shape[0] - ph0), (pw0, size - fg.shape[1] - pw0), (0, 0))) + new = int(size / ratio) + ph0, pw0 = (new - size) // 2, (new - size) // 2 + return np.pad(sq, ((ph0, new - size - ph0), (pw0, new - size - pw0), (0, 0))) + + +class LLMGWPrepareForMesh: + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "image": ("IMAGE",), + "foreground_ratio": ("FLOAT", {"default": 0.85, "min": 0.3, "max": 1.0, "step": 0.01}), + "background": ("FLOAT", {"default": 0.5, "min": 0.0, "max": 1.0, "step": 0.01}), + "remove_background": (["auto", "always", "never"],), + "rembg_model": (["u2net", "isnet-general-use", "u2netp"],), + }, + "optional": {"mask": ("MASK",)}, + } + + RETURN_TYPES = ("IMAGE",) + FUNCTION = "prepare" + CATEGORY = "llmgw" + + def prepare(self, image, foreground_ratio, background, remove_background, rembg_model, mask=None): + out = [] + for i in range(image.shape[0]): + rgb = (image[i].cpu().numpy() * 255).clip(0, 255).astype(np.uint8) + h, w = rgb.shape[:2] + alpha = None + # LoadImage's mask is 1 - alpha, and a 64x64 of zeros for a + # picture with none. + if mask is not None and remove_background != "always": + m = mask[min(i, mask.shape[0] - 1)].cpu().numpy() + if m.shape == (h, w) and m.max() > 0: + alpha = ((1.0 - m) * 255).clip(0, 255).astype(np.uint8) + if alpha is None and remove_background != "never": + rgba = np.array(_rembg(Image.fromarray(rgb), rembg_model).convert("RGBA")) + else: + if alpha is None: + alpha = np.full((h, w), 255, np.uint8) + rgba = np.dstack([rgb, alpha]) + sq = _fit(rgba, foreground_ratio).astype(np.float32) / 255.0 + a = sq[..., 3:4] + img = sq[..., :3] * a + (1.0 - a) * background + out.append(torch.from_numpy(img)) + # One size for the batch: the first picture's. + side = out[0].shape[0] + out = [o if o.shape[0] == side else torch.from_numpy( + np.array(Image.fromarray((o.numpy() * 255).astype(np.uint8)).resize((side, side)))).float() / 255.0 + for o in out] + return (torch.stack(out),) + + +NODE_CLASS_MAPPINGS = {"LLMGWPrepareForMesh": LLMGWPrepareForMesh} +NODE_DISPLAY_NAME_MAPPINGS = {"LLMGWPrepareForMesh": "Prepare a picture for a 3D model (llmgw)"} diff --git a/internal/router/handlers.go b/internal/router/handlers.go index 79bb59f..df88f09 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -10,6 +10,7 @@ import ( "fmt" llmgw "gitea.mynodes.xyz/ilker/llmgw" "io" + "maps" "net/http" "slices" "strconv" @@ -1108,6 +1109,16 @@ func (r *Router) handleModels(w http.ResponseWriter, req *http.Request) { seen[m] = struct{}{} } } + // A model that loads when asked has no replica until somebody asks, + // and asking is how it gets one: listing only what runs hid it from + // exactly the clients that would have asked — a 3D model, a picture + // model, asleep between uses. Said with loaded: false. + loaded := maps.Clone(seen) + for name, m := range r.modelMap() { + if name == m.Name && m.Scaling.OnDemand() && !m.IsExternal() { + seen[name] = struct{}{} + } + } // Catalog aliases are usable names too, so clients should see them. for name := range r.modelMap() { if _, ok := seen[r.modelMap()[name].Name]; ok { @@ -1135,8 +1146,15 @@ func (r *Router) handleModels(w http.ResponseWriter, req *http.Request) { if r.modelMap()[n].Simulated() { owner = "llmgw-sim" } + _, up := loaded[n] + if m := r.modelMap()[n]; !up && m.Name != "" { + _, up = loaded[m.Name] + } entry := map[string]any{ "id": n, "object": "model", "created": time.Now().Unix(), "owned_by": owner, + // Not OpenAI's: whether a replica is up now, or the first + // request waits for one to load. + "loaded": up, } // Not OpenAI's either: what a federated peer reading this list needs // to leave out — a second name for a model already listed, and a diff --git a/internal/router/meshes_test.go b/internal/router/meshes_test.go index a5673b7..e18dcc2 100644 --- a/internal/router/meshes_test.go +++ b/internal/router/meshes_test.go @@ -361,3 +361,43 @@ func TestThe3DPageMakesAJobAndShowsItsMeshToItsOwnerOnly(t *testing.T) { t.Errorf("another project's person read the job: %d", rec.Code) } } + +func TestAModelAsleepUntilAskedIsListedAndSaidAsleep(t *testing.T) { + // triposr and flux-schnell load when asked and stop when idle; a client + // asking /v1/models which model makes 3D models found neither while + // they slept, and so never asked. + cfg := baseConfig() + asleep := catModel("triposr", nil) + asleep.Scaling = catalog.Scaling{Min: 0, Max: 1} + asleep.ComfyUI = &catalog.ComfyUI{Output: catalog.OutputMeshes, Workflow: "w.json"} + asleep.Profiles = []catalog.Profile{{Name: "gpu", Engine: catalog.EngineComfyUI, GPUClass: "6000-24g", TP: 1}} + cfg.Models = []catalog.Model{asleep, catModel("chat", nil)} + rt := newTestRouter(t, cfg) + e := newFakeEngine(t, "chat") + rt.SetEndpoints(t.Context(), []endpoint.Info{{ID: "c", URL: e.srv.URL, Models: []string{"chat"}}}) + rec := httptest.NewRecorder() + rt.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/v1/models", nil)) + var list struct { + Data []struct { + ID string `json:"id"` + Loaded bool `json:"loaded"` + Capabilities map[string]bool `json:"capabilities"` + } `json:"data"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &list); err != nil { + t.Fatal(err) + } + got := map[string]bool{} + for _, m := range list.Data { + got[m.ID] = m.Loaded + if m.ID == "triposr" && !m.Capabilities["meshes"] { + t.Error("triposr is not said to make meshes") + } + } + if loaded, ok := got["triposr"]; !ok || loaded { + t.Errorf("triposr listed %v, loaded %v: wanted listed and asleep", ok, loaded) + } + if !got["chat"] { + t.Error("a model serving is not said loaded") + } +}