Files
llmgw/internal/router/meshes.go
T
Ilker ManapandClaude Opus 5.5 88781cfcfe A job says how far along it is; a ComfyUI model brings its custom nodes
G2: the gate keeps a websocket open to its ComfyUI under a client id of its
own, puts that id on every workflow it lets through — ComfyUI tells only
the client that queued a workflow how it goes — and keeps the last word on
each: the node running, its step of how many. It answers at
/progress/{prompt_id}. While a job's request waits, the router reads the
queue (queued, n ahead) and the gate (running, node, step, percent) and
writes it into the job, which a client reads back. Only into a job of the
same key: a tag naming another's job writes nothing.

G3, the rest of the plumbing: comfyui.nodes names custom nodes by https
repository and full commit, with the PyPI packages they need. The gate puts
each in the replica's own custom_nodes at that commit and installs the
packages beside the replica, constrained to the versions ComfyUI's
environment already has, which it never changes. The catalog refuses a
branch for a commit and anything but package specs for requirements. An
OBJ a node saves leaves the gateway as a GLB, colours kept, so every mesh
can carry its licence.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-10 18:46:44 +02:00

272 lines
8.6 KiB
Go

package router
import (
"bytes"
"context"
"crypto/rand"
"encoding/base64"
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net/http"
"strconv"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/comfy"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
)
// 3D models from a picture, made by ComfyUI. Clients ask for them as a job
// (jobs.go); the job worker replays the request through the ordinary path,
// and at a ComfyUI replica it arrives here: the picture is put in ComfyUI's
// input directory, the model's workflow is filled in with its name and run,
// and the mesh it saves comes back as the answer — the file the job keeps.
// meshRequest is what a request for a 3D model asks. The picture is the
// whole of the prompt: base64, or a data URL.
type meshRequest struct {
Model string `json:"model"`
Image string `json:"image"`
Seed *int64 `json:"seed"`
Steps int `json:"steps"`
// Resolution is the model's own grid, where it has a choice: TRELLIS.2
// makes 512, 1024 or 1536. Zero leaves it to the workflow.
Resolution int `json:"resolution"`
picture []byte
ext string
}
// maxPicture bounds the picture a mesh is made from.
const maxPicture = 16 << 20
func parseMeshRequest(body []byte) (meshRequest, error) {
var in meshRequest
if err := json.Unmarshal(body, &in); err != nil {
return in, err
}
s := strings.TrimSpace(in.Image)
if s == "" {
return in, errors.New("image is required: the picture to make a 3D model of, base64 or a data URL")
}
if strings.HasPrefix(s, "data:") {
_, after, ok := strings.Cut(s, ";base64,")
if !ok {
return in, errors.New("image is a data URL without base64 in it")
}
s = after
}
if base64.StdEncoding.DecodedLen(len(s)) > maxPicture+3 {
return in, fmt.Errorf("image is past %d MB", maxPicture>>20)
}
b, err := base64.StdEncoding.DecodeString(s)
if err != nil {
return in, errors.New("image is not base64: " + err.Error())
}
switch http.DetectContentType(b) {
case "image/png":
in.ext = ".png"
case "image/jpeg":
in.ext = ".jpg"
case "image/webp":
in.ext = ".webp"
default:
return in, errors.New("image is not a PNG, JPEG or WebP picture")
}
in.picture = b
if in.Steps < 0 || in.Steps > maxSteps {
return in, fmt.Errorf("steps is between 1 and %d", maxSteps)
}
switch in.Resolution {
case 0, 512, 1024, 1536:
default:
return in, fmt.Errorf("resolution %d is not 512, 1024 or 1536", in.Resolution)
}
return in, nil
}
// generateMeshes answers a request for a 3D model at a ComfyUI replica. It
// writes the mesh itself, with its type, which is what the job keeps.
func (r *Router) generateMeshes(ctx context.Context, w http.ResponseWriter, req *parsedRequest, ep endpoint.Info, body []byte) (proxy.Outcome, error) {
in, err := parseMeshRequest(body)
if err != nil {
return proxy.Outcome{}, err
}
tpl, err := comfy.Load(ep.Workflow)
if err != nil {
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
}
if t := r.cfg.Timeouts.Total; t > 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, t)
defer cancel()
}
start := time.Now()
fail := func(err error) (proxy.Outcome, error) {
var ce *comfy.Error
switch {
case errors.As(err, &ce):
return proxy.Outcome{Retryable: ce.Retryable}, err
case errors.Is(err, context.Canceled) && ctxCauseIsClient(ctx):
return proxy.Outcome{}, proxy.ErrClientGone
case errors.Is(err, context.DeadlineExceeded):
return proxy.Outcome{}, fmt.Errorf("%s made no 3D model within %s", ep.ID, r.cfg.Timeouts.Total)
}
return proxy.Outcome{}, err
}
// A name of its own: two requests' pictures must not overwrite each
// other in the replica's input directory.
rnd := make([]byte, 8)
_, _ = rand.Read(rnd)
name, err := r.comfy.Upload(ctx, ep.URL, ep.APIKey, "llmgw-"+hex.EncodeToString(rnd)+in.ext, in.picture)
if err != nil {
return fail(err)
}
seed := comfy.RandomSeed()
if in.Seed != nil {
seed = *in.Seed
}
var upstream struct {
Model string `json:"model"`
}
_ = json.Unmarshal(body, &upstream)
p := comfy.Params{Checkpoint: upstream.Model, Image: name, Seed: seed, Steps: in.Steps, Extra: map[string]any{}}
if in.Resolution > 0 {
p.Extra["resolution"] = in.Resolution
}
workflow, err := tpl.Fill(p)
if err != nil {
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
}
files, err := r.comfy.GenerateWatch(ctx, ep.URL, ep.APIKey, workflow, r.jobProgress(ctx, req))
if err != nil {
return fail(err)
}
var mesh *comfy.Image
for i := range files {
if isMeshType(files[i].ContentType) {
mesh = &files[i]
break
}
}
if mesh == nil {
return proxy.Outcome{}, fmt.Errorf("%s: the workflow saved no mesh (it saved %d other files): "+
"its last node should save a .glb", ep.ID, len(files))
}
data := mesh.Data
if mesh.ContentType == "model/obj" {
glb, err := objToGLB(data)
if err != nil {
return proxy.Outcome{}, fmt.Errorf("%s: the mesh it saved: %w", ep.ID, err)
}
data, mesh.ContentType = glb, "model/gltf-binary"
}
m := r.modelMap()[req.Model]
if m.License != nil && mesh.ContentType == "model/gltf-binary" {
if with, err := glbWithLicense(data, *m.License); err == nil {
data = with
} else {
// Refused rather than sent without it: a mesh travels without
// the API, and the notice is what the terms ask to travel too.
return proxy.Outcome{}, fmt.Errorf("%s: the licence could not be written into the mesh: %w", ep.ID, err)
}
}
w.Header().Set("x-llmgw-seed", strconv.FormatInt(seed, 10))
if m.License != nil {
w.Header().Set("x-llmgw-license", m.License.Name)
}
w.Header().Set("Content-Type", mesh.ContentType)
w.WriteHeader(http.StatusOK)
_, _ = w.Write(data)
took := time.Since(start)
return proxy.Outcome{Wrote: true, Status: http.StatusOK, Bytes: int64(len(data)), TTFT: took, Decode: took,
Meshes: 1}, nil
}
func isMeshType(ct string) bool {
return strings.HasPrefix(ct, "model/") || ct == "application/ply"
}
// glbWithLicense is a GLB with the licence in asset.extras.license, where a
// file's terms travel with it: glTF's own place for what an application
// adds, kept by every tool that loads and saves the file.
func glbWithLicense(glb []byte, lic catalog.License) ([]byte, error) {
if len(glb) < 20 || string(glb[:4]) != "glTF" || binary.LittleEndian.Uint32(glb[4:]) != 2 {
return nil, errors.New("not a glTF 2 binary")
}
n := int(binary.LittleEndian.Uint32(glb[12:]))
if string(glb[16:20]) != "JSON" || 20+n > len(glb) {
return nil, errors.New("its first chunk is not its JSON")
}
var doc map[string]any
d := json.NewDecoder(bytes.NewReader(glb[20 : 20+n]))
d.UseNumber() // every number as it was written
if err := d.Decode(&doc); err != nil {
return nil, err
}
asset, _ := doc["asset"].(map[string]any)
if asset == nil {
asset = map[string]any{"version": "2.0"}
}
extras, _ := asset["extras"].(map[string]any)
if extras == nil {
extras = map[string]any{}
}
l := map[string]any{"name": lic.Name}
if lic.URL != "" {
l["url"] = lic.URL
}
if lic.Notice != "" {
l["notice"] = lic.Notice
}
extras["license"] = l
asset["extras"] = extras
doc["asset"] = asset
js, err := json.Marshal(doc)
if err != nil {
return nil, err
}
// A chunk is padded to four bytes, the JSON one with spaces.
for len(js)%4 != 0 {
js = append(js, ' ')
}
rest := glb[20+n:]
out := make([]byte, 0, 20+len(js)+len(rest))
out = append(out, "glTF"...)
out = binary.LittleEndian.AppendUint32(out, 2)
out = binary.LittleEndian.AppendUint32(out, uint32(20+len(js)+len(rest))) //nolint:gosec // bounded by the file read
out = binary.LittleEndian.AppendUint32(out, uint32(len(js))) //nolint:gosec // bounded by the file read
out = append(out, "JSON"...)
out = append(out, js...)
return append(out, rest...), nil
}
// jobProgress is where a job's request says how far along it is: into the
// job, for whoever asks after it. Nil for a request that is no job's — and
// for one that only says it is: the job must be the same key's, so nobody
// writes into another's progress by naming it in a tag.
func (r *Router) jobProgress(ctx context.Context, req *parsedRequest) func(comfy.Progress) {
id := req.Tags["job"]
if id == "" || r.store == nil || id != req.RequestID {
return nil
}
j, err := r.store.JobByID(ctx, id)
if err != nil || j.KeyID != req.KeyID {
return nil
}
return func(p comfy.Progress) {
b, err := json.Marshal(p)
if err != nil {
return
}
if err := r.store.SetJobProgress(context.WithoutCancel(ctx), id, string(b), p.PromptID); err != nil {
r.log.Warn("a job's progress could not be kept", "job", id, "err", err)
}
}
}