Files
llmgw/internal/router/meshes.go
T
Ilker ManapandClaude Opus 5.5 aa531b1053 ComfyUI models of one environment share one replica
flux and sf3d on one card handed it to each other, each starting ComfyUI
and loading its weights from nothing (BACKLOG H3, placement side). An
environment of two or more models now gets a hidden catalog entry,
comfyui-env-<name>: on its first member's profiles with the most memory
any member needs, on demand for the longest idle timeout, carrying every
member's weights, repositories and the environment's custom nodes. The
members get no replicas of their own; asking for one wants the entry.

The agent fetches each member's weights; the gate links all of them,
refusing two weights files of one name, links each member's Hugging Face
repository, and answers /v1/models for every member. The control plane
offers the replica under the members' names with each one's workflow and
checkpoint, which the router sends by the model asked for. A member's
sibling never evicts the replica serving it, and the members' load counts
as the entry's.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-14 13:55:34 +02:00

389 lines
13 KiB
Go

package router
import (
"bytes"
"context"
"crypto/rand"
"encoding/base64"
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/comfy"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
)
// 3D models from a picture, made by ComfyUI. Clients ask for them as a job
// (jobs.go); the job worker replays the request through the ordinary path,
// and at a ComfyUI replica it arrives here: the picture is put in ComfyUI's
// input directory, the model's workflow is filled in with its name and run,
// and the mesh it saves comes back as the answer — the file the job keeps.
// meshRequest is what a request for a 3D model asks. The picture is the
// whole of the prompt: base64, or a data URL.
type meshRequest struct {
Model string `json:"model"`
Image string `json:"image"`
Seed *int64 `json:"seed"`
Steps int `json:"steps"`
// Resolution is the model's own grid, where it has a choice: TripoSR's
// marching cubes take 256 by default, TRELLIS.2 512 to 1536. Zero leaves
// it to the workflow.
Resolution int `json:"resolution"`
picture []byte
ext string
}
// maxPicture bounds the picture a mesh is made from.
const maxPicture = 16 << 20
func parseMeshRequest(body []byte) (meshRequest, error) {
var in meshRequest
if err := json.Unmarshal(body, &in); err != nil {
return in, err
}
s := strings.TrimSpace(in.Image)
if s == "" {
return in, errors.New("image is required: the picture to make a 3D model of, base64 or a data URL")
}
if strings.HasPrefix(s, "data:") {
_, after, ok := strings.Cut(s, ";base64,")
if !ok {
return in, errors.New("image is a data URL without base64 in it")
}
s = after
}
if base64.StdEncoding.DecodedLen(len(s)) > maxPicture+3 {
return in, fmt.Errorf("image is past %d MB", maxPicture>>20)
}
b, err := base64.StdEncoding.DecodeString(s)
if err != nil {
return in, errors.New("image is not base64: " + err.Error())
}
switch http.DetectContentType(b) {
case "image/png":
in.ext = ".png"
case "image/jpeg":
in.ext = ".jpg"
case "image/webp":
in.ext = ".webp"
default:
return in, errors.New("image is not a PNG, JPEG or WebP picture")
}
in.picture = b
if in.Steps < 0 || in.Steps > maxSteps {
return in, fmt.Errorf("steps is between 1 and %d", maxSteps)
}
if r := in.Resolution; r != 0 && (r < 64 || r > 2048 || r%32 != 0) {
return in, fmt.Errorf("resolution %d is not a multiple of 32 from 64 to 2048", r)
}
return in, nil
}
// generateMeshes answers a request for a 3D model at a ComfyUI replica. It
// writes the mesh itself, with its type, which is what the job keeps.
func (r *Router) generateMeshes(ctx context.Context, w http.ResponseWriter, req *parsedRequest, ep endpoint.Info, body []byte) (proxy.Outcome, error) {
in, err := parseMeshRequest(body)
if err != nil {
return proxy.Outcome{}, err
}
wf, _ := ep.ComfyFor(req.Model)
tpl, err := comfy.Load(wf)
if err != nil {
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
}
if t := r.cfg.Timeouts.Total; t > 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, t)
defer cancel()
}
start := time.Now()
fail := func(err error) (proxy.Outcome, error) {
var ce *comfy.Error
switch {
case errors.As(err, &ce):
return proxy.Outcome{Retryable: ce.Retryable}, err
case errors.Is(err, context.Canceled) && ctxCauseIsClient(ctx):
return proxy.Outcome{}, proxy.ErrClientGone
case errors.Is(err, context.DeadlineExceeded):
return proxy.Outcome{}, fmt.Errorf("%s made no 3D model within %s", ep.ID, r.cfg.Timeouts.Total)
}
return proxy.Outcome{}, err
}
// A name of its own: two requests' pictures must not overwrite each
// other in the replica's input directory.
rnd := make([]byte, 8)
_, _ = rand.Read(rnd)
name, err := r.comfy.Upload(ctx, ep.URL, ep.APIKey, "llmgw-"+hex.EncodeToString(rnd)+in.ext, in.picture)
if err != nil {
return fail(err)
}
seed := comfy.RandomSeed()
if in.Seed != nil {
seed = *in.Seed
}
var upstream struct {
Model string `json:"model"`
}
_ = json.Unmarshal(body, &upstream)
p := comfy.Params{Checkpoint: upstream.Model, Image: name, Seed: seed, Steps: in.Steps, Extra: map[string]any{}}
if in.Resolution > 0 {
p.Extra["resolution"] = in.Resolution
}
workflow, err := tpl.Fill(p)
if err != nil {
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
}
files, err := r.comfy.GenerateWatch(ctx, ep.URL, ep.APIKey, workflow, r.jobProgress(ctx, req))
if err != nil {
return fail(err)
}
var mesh *comfy.Image
for i := range files {
if isMeshType(files[i].ContentType) {
mesh = &files[i]
break
}
}
if mesh == nil {
return proxy.Outcome{}, fmt.Errorf("%s: the workflow saved no mesh (it saved %d other files): "+
"its last node should save a .glb", ep.ID, len(files))
}
data := mesh.Data
if mesh.ContentType == "model/obj" {
glb, err := objToGLB(data)
if err != nil {
return proxy.Outcome{}, fmt.Errorf("%s: the mesh it saved: %w", ep.ID, err)
}
data, mesh.ContentType = glb, "model/gltf-binary"
}
m := r.modelMap()[req.Model]
if m.License != nil && mesh.ContentType == "model/gltf-binary" {
if with, err := glbWithLicense(data, *m.License); err == nil {
data = with
} else {
// Refused rather than sent without it: a mesh travels without
// the API, and the notice is what the terms ask to travel too.
return proxy.Outcome{}, fmt.Errorf("%s: the licence could not be written into the mesh: %w", ep.ID, err)
}
}
w.Header().Set("x-llmgw-seed", strconv.FormatInt(seed, 10))
if m.License != nil {
w.Header().Set("x-llmgw-license", m.License.Name)
}
w.Header().Set("Content-Type", mesh.ContentType)
w.WriteHeader(http.StatusOK)
_, _ = w.Write(data)
took := time.Since(start)
return proxy.Outcome{Wrote: true, Status: http.StatusOK, Bytes: int64(len(data)), TTFT: took, Decode: took,
Meshes: 1}, nil
}
func isMeshType(ct string) bool {
return strings.HasPrefix(ct, "model/") || ct == "application/ply"
}
// glbWithLicense is a GLB with the licence in asset.extras.license, where a
// file's terms travel with it: glTF's own place for what an application
// adds, kept by every tool that loads and saves the file.
func glbWithLicense(glb []byte, lic catalog.License) ([]byte, error) {
if len(glb) < 20 || string(glb[:4]) != "glTF" || binary.LittleEndian.Uint32(glb[4:]) != 2 {
return nil, errors.New("not a glTF 2 binary")
}
n := int(binary.LittleEndian.Uint32(glb[12:]))
if string(glb[16:20]) != "JSON" || 20+n > len(glb) {
return nil, errors.New("its first chunk is not its JSON")
}
var doc map[string]any
d := json.NewDecoder(bytes.NewReader(glb[20 : 20+n]))
d.UseNumber() // every number as it was written
if err := d.Decode(&doc); err != nil {
return nil, err
}
asset, _ := doc["asset"].(map[string]any)
if asset == nil {
asset = map[string]any{"version": "2.0"}
}
extras, _ := asset["extras"].(map[string]any)
if extras == nil {
extras = map[string]any{}
}
// The whole licence, as /v1/models gives it: its id, whether it allows
// commercial use and where it grants nothing are what a pipeline that
// sorts assets by licence reads, and the name alone is for people.
var l map[string]any
b, err := json.Marshal(lic)
if err != nil {
return nil, err
}
if err := json.Unmarshal(b, &l); err != nil {
return nil, err
}
extras["license"] = l
asset["extras"] = extras
doc["asset"] = asset
js, err := json.Marshal(doc)
if err != nil {
return nil, err
}
// A chunk is padded to four bytes, the JSON one with spaces.
for len(js)%4 != 0 {
js = append(js, ' ')
}
rest := glb[20+n:]
out := make([]byte, 0, 20+len(js)+len(rest))
out = append(out, "glTF"...)
out = binary.LittleEndian.AppendUint32(out, 2)
out = binary.LittleEndian.AppendUint32(out, uint32(20+len(js)+len(rest))) //nolint:gosec // bounded by the file read
out = binary.LittleEndian.AppendUint32(out, uint32(len(js))) //nolint:gosec // bounded by the file read
out = append(out, "JSON"...)
out = append(out, js...)
return append(out, rest...), nil
}
// jobProgress is where a job's request says how far along it is: into the
// job, for whoever asks after it. Nil for a request that is no job's — and
// for one that only says it is: the job must be the same key's, so nobody
// writes into another's progress by naming it in a tag.
func (r *Router) jobProgress(ctx context.Context, req *parsedRequest) func(comfy.Progress) {
id := req.Tags["job"]
if id == "" || r.store == nil || id != req.RequestID {
return nil
}
j, err := r.store.JobByID(ctx, id)
if err != nil || j.KeyID != req.KeyID {
return nil
}
return func(p comfy.Progress) {
b, err := json.Marshal(p)
if err != nil {
return
}
if err := r.store.SetJobProgress(context.WithoutCancel(ctx), id, string(b), p.PromptID); err != nil {
r.log.Warn("a job's progress could not be kept", "job", id, "err", err)
}
}
}
// registerMeshesPage mounts the page that makes 3D models, beside the
// pictures page and for the same reasons: a session, never a key, and the
// request sent down the path an API client's takes — here, a job.
func (r *Router) registerMeshesPage(mux *http.ServeMux, pages *web.AccountPages) {
pages.SetMeshes(func() bool {
for _, m := range r.modelMap() {
if m.MakesMeshes() {
return true
}
}
return false
})
mux.HandleFunc("GET /3d", pages.Guard(func(w http.ResponseWriter, req *http.Request, user store.User) {
contexts, _ := r.chatContexts(req.Context(), user)
current, err := r.pickContext(req.Context(), user, req.URL.Query().Get("context"))
if err != nil {
current = ""
}
if current == "" && !r.personalOff {
current = personalContext
}
web.Meshes(user, r.modelsOf(req, user, catalog.Model.MakesMeshes), contexts, current, version.Version)(w, req)
}))
mux.HandleFunc("POST /3d/generate", pages.Guard(r.meshesSend))
mux.HandleFunc("GET /3d/jobs/{id}", pages.Guard(func(w http.ResponseWriter, req *http.Request, user store.User) {
if next, ok := r.asPerson(w, req, user, req.URL.Query().Get("context")); ok {
r.handleGetJob(w, next)
}
}))
// The mesh itself, from where the page is, for its viewer to load.
mux.HandleFunc("GET /3d/jobs/{id}/file", pages.Guard(func(w http.ResponseWriter, req *http.Request, user store.User) {
next, ok := r.asPerson(w, req, user, req.URL.Query().Get("context"))
if !ok {
return
}
p, _ := auth.FromContext(next.Context())
j, ok := r.ownedJob(w, next, ownerOf(p))
if !ok {
return
}
if j.OutputFileID == "" {
http.Error(w, "the job has made nothing yet", http.StatusNotFound)
return
}
b, err := r.store.FileContent(req.Context(), j.OutputFileID)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", j.ContentType)
w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=%q", j.ID+".glb"))
_, _ = w.Write(b)
}))
}
// asPerson is a page's request as the signed-in person's, in the place they
// chose: what the API handlers read a key's principal from.
func (r *Router) asPerson(w http.ResponseWriter, req *http.Request, user store.User, context string) (*http.Request, bool) {
p, err := r.principalFor(req, user, context)
if err != nil {
http.Error(w, err.Error(), http.StatusForbidden)
return nil, false
}
return req.WithContext(auth.WithPrincipal(req.Context(), p)), true
}
// meshesSend asks for a 3D model on behalf of a signed-in person.
func (r *Router) meshesSend(w http.ResponseWriter, req *http.Request, user store.User) {
body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes))
if err != nil {
http.Error(w, "could not read the request", http.StatusBadRequest)
return
}
var in struct {
Model string `json:"model"`
Image string `json:"image"`
Seed *int64 `json:"seed"`
Resolution int `json:"resolution"`
Context string `json:"context"`
}
if err := json.Unmarshal(body, &in); err != nil || in.Model == "" {
http.Error(w, "a model and a picture are required", http.StatusBadRequest)
return
}
// Rebuilt from the fields the page has, so it can set nothing else.
out := map[string]any{"model": in.Model, "image": in.Image}
if in.Seed != nil {
out["seed"] = *in.Seed
}
if in.Resolution > 0 {
out["resolution"] = in.Resolution
}
upstream, err := json.Marshal(out)
if err != nil {
http.Error(w, "could not build the request", http.StatusInternalServerError)
return
}
next, ok := r.asPerson(w, req, user, in.Context)
if !ok {
return
}
next = next.Clone(next.Context())
next.Body = io.NopCloser(bytes.NewReader(upstream))
next.ContentLength = int64(len(upstream))
next.Header.Set("Content-Type", "application/json")
r.handleCreateJob(w, next)
}