flux and sf3d on one card handed it to each other, each starting ComfyUI and loading its weights from nothing (BACKLOG H3, placement side). An environment of two or more models now gets a hidden catalog entry, comfyui-env-<name>: on its first member's profiles with the most memory any member needs, on demand for the longest idle timeout, carrying every member's weights, repositories and the environment's custom nodes. The members get no replicas of their own; asking for one wants the entry. The agent fetches each member's weights; the gate links all of them, refusing two weights files of one name, links each member's Hugging Face repository, and answers /v1/models for every member. The control plane offers the replica under the members' names with each one's workflow and checkpoint, which the router sends by the model asked for. A member's sibling never evicts the replica serving it, and the members' load counts as the entry's. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
389 lines
13 KiB
Go
389 lines
13 KiB
Go
package router
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/base64"
|
|
"encoding/binary"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/comfy"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
|
|
)
|
|
|
|
// 3D models from a picture, made by ComfyUI. Clients ask for them as a job
|
|
// (jobs.go); the job worker replays the request through the ordinary path,
|
|
// and at a ComfyUI replica it arrives here: the picture is put in ComfyUI's
|
|
// input directory, the model's workflow is filled in with its name and run,
|
|
// and the mesh it saves comes back as the answer — the file the job keeps.
|
|
|
|
// meshRequest is what a request for a 3D model asks. The picture is the
|
|
// whole of the prompt: base64, or a data URL.
|
|
type meshRequest struct {
|
|
Model string `json:"model"`
|
|
Image string `json:"image"`
|
|
Seed *int64 `json:"seed"`
|
|
Steps int `json:"steps"`
|
|
// Resolution is the model's own grid, where it has a choice: TripoSR's
|
|
// marching cubes take 256 by default, TRELLIS.2 512 to 1536. Zero leaves
|
|
// it to the workflow.
|
|
Resolution int `json:"resolution"`
|
|
|
|
picture []byte
|
|
ext string
|
|
}
|
|
|
|
// maxPicture bounds the picture a mesh is made from.
|
|
const maxPicture = 16 << 20
|
|
|
|
func parseMeshRequest(body []byte) (meshRequest, error) {
|
|
var in meshRequest
|
|
if err := json.Unmarshal(body, &in); err != nil {
|
|
return in, err
|
|
}
|
|
s := strings.TrimSpace(in.Image)
|
|
if s == "" {
|
|
return in, errors.New("image is required: the picture to make a 3D model of, base64 or a data URL")
|
|
}
|
|
if strings.HasPrefix(s, "data:") {
|
|
_, after, ok := strings.Cut(s, ";base64,")
|
|
if !ok {
|
|
return in, errors.New("image is a data URL without base64 in it")
|
|
}
|
|
s = after
|
|
}
|
|
if base64.StdEncoding.DecodedLen(len(s)) > maxPicture+3 {
|
|
return in, fmt.Errorf("image is past %d MB", maxPicture>>20)
|
|
}
|
|
b, err := base64.StdEncoding.DecodeString(s)
|
|
if err != nil {
|
|
return in, errors.New("image is not base64: " + err.Error())
|
|
}
|
|
switch http.DetectContentType(b) {
|
|
case "image/png":
|
|
in.ext = ".png"
|
|
case "image/jpeg":
|
|
in.ext = ".jpg"
|
|
case "image/webp":
|
|
in.ext = ".webp"
|
|
default:
|
|
return in, errors.New("image is not a PNG, JPEG or WebP picture")
|
|
}
|
|
in.picture = b
|
|
if in.Steps < 0 || in.Steps > maxSteps {
|
|
return in, fmt.Errorf("steps is between 1 and %d", maxSteps)
|
|
}
|
|
if r := in.Resolution; r != 0 && (r < 64 || r > 2048 || r%32 != 0) {
|
|
return in, fmt.Errorf("resolution %d is not a multiple of 32 from 64 to 2048", r)
|
|
}
|
|
return in, nil
|
|
}
|
|
|
|
// generateMeshes answers a request for a 3D model at a ComfyUI replica. It
|
|
// writes the mesh itself, with its type, which is what the job keeps.
|
|
func (r *Router) generateMeshes(ctx context.Context, w http.ResponseWriter, req *parsedRequest, ep endpoint.Info, body []byte) (proxy.Outcome, error) {
|
|
in, err := parseMeshRequest(body)
|
|
if err != nil {
|
|
return proxy.Outcome{}, err
|
|
}
|
|
wf, _ := ep.ComfyFor(req.Model)
|
|
tpl, err := comfy.Load(wf)
|
|
if err != nil {
|
|
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
|
|
}
|
|
if t := r.cfg.Timeouts.Total; t > 0 {
|
|
var cancel context.CancelFunc
|
|
ctx, cancel = context.WithTimeout(ctx, t)
|
|
defer cancel()
|
|
}
|
|
start := time.Now()
|
|
fail := func(err error) (proxy.Outcome, error) {
|
|
var ce *comfy.Error
|
|
switch {
|
|
case errors.As(err, &ce):
|
|
return proxy.Outcome{Retryable: ce.Retryable}, err
|
|
case errors.Is(err, context.Canceled) && ctxCauseIsClient(ctx):
|
|
return proxy.Outcome{}, proxy.ErrClientGone
|
|
case errors.Is(err, context.DeadlineExceeded):
|
|
return proxy.Outcome{}, fmt.Errorf("%s made no 3D model within %s", ep.ID, r.cfg.Timeouts.Total)
|
|
}
|
|
return proxy.Outcome{}, err
|
|
}
|
|
// A name of its own: two requests' pictures must not overwrite each
|
|
// other in the replica's input directory.
|
|
rnd := make([]byte, 8)
|
|
_, _ = rand.Read(rnd)
|
|
name, err := r.comfy.Upload(ctx, ep.URL, ep.APIKey, "llmgw-"+hex.EncodeToString(rnd)+in.ext, in.picture)
|
|
if err != nil {
|
|
return fail(err)
|
|
}
|
|
seed := comfy.RandomSeed()
|
|
if in.Seed != nil {
|
|
seed = *in.Seed
|
|
}
|
|
var upstream struct {
|
|
Model string `json:"model"`
|
|
}
|
|
_ = json.Unmarshal(body, &upstream)
|
|
p := comfy.Params{Checkpoint: upstream.Model, Image: name, Seed: seed, Steps: in.Steps, Extra: map[string]any{}}
|
|
if in.Resolution > 0 {
|
|
p.Extra["resolution"] = in.Resolution
|
|
}
|
|
workflow, err := tpl.Fill(p)
|
|
if err != nil {
|
|
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
|
|
}
|
|
files, err := r.comfy.GenerateWatch(ctx, ep.URL, ep.APIKey, workflow, r.jobProgress(ctx, req))
|
|
if err != nil {
|
|
return fail(err)
|
|
}
|
|
var mesh *comfy.Image
|
|
for i := range files {
|
|
if isMeshType(files[i].ContentType) {
|
|
mesh = &files[i]
|
|
break
|
|
}
|
|
}
|
|
if mesh == nil {
|
|
return proxy.Outcome{}, fmt.Errorf("%s: the workflow saved no mesh (it saved %d other files): "+
|
|
"its last node should save a .glb", ep.ID, len(files))
|
|
}
|
|
data := mesh.Data
|
|
if mesh.ContentType == "model/obj" {
|
|
glb, err := objToGLB(data)
|
|
if err != nil {
|
|
return proxy.Outcome{}, fmt.Errorf("%s: the mesh it saved: %w", ep.ID, err)
|
|
}
|
|
data, mesh.ContentType = glb, "model/gltf-binary"
|
|
}
|
|
m := r.modelMap()[req.Model]
|
|
if m.License != nil && mesh.ContentType == "model/gltf-binary" {
|
|
if with, err := glbWithLicense(data, *m.License); err == nil {
|
|
data = with
|
|
} else {
|
|
// Refused rather than sent without it: a mesh travels without
|
|
// the API, and the notice is what the terms ask to travel too.
|
|
return proxy.Outcome{}, fmt.Errorf("%s: the licence could not be written into the mesh: %w", ep.ID, err)
|
|
}
|
|
}
|
|
w.Header().Set("x-llmgw-seed", strconv.FormatInt(seed, 10))
|
|
if m.License != nil {
|
|
w.Header().Set("x-llmgw-license", m.License.Name)
|
|
}
|
|
w.Header().Set("Content-Type", mesh.ContentType)
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write(data)
|
|
took := time.Since(start)
|
|
return proxy.Outcome{Wrote: true, Status: http.StatusOK, Bytes: int64(len(data)), TTFT: took, Decode: took,
|
|
Meshes: 1}, nil
|
|
}
|
|
|
|
func isMeshType(ct string) bool {
|
|
return strings.HasPrefix(ct, "model/") || ct == "application/ply"
|
|
}
|
|
|
|
// glbWithLicense is a GLB with the licence in asset.extras.license, where a
|
|
// file's terms travel with it: glTF's own place for what an application
|
|
// adds, kept by every tool that loads and saves the file.
|
|
func glbWithLicense(glb []byte, lic catalog.License) ([]byte, error) {
|
|
if len(glb) < 20 || string(glb[:4]) != "glTF" || binary.LittleEndian.Uint32(glb[4:]) != 2 {
|
|
return nil, errors.New("not a glTF 2 binary")
|
|
}
|
|
n := int(binary.LittleEndian.Uint32(glb[12:]))
|
|
if string(glb[16:20]) != "JSON" || 20+n > len(glb) {
|
|
return nil, errors.New("its first chunk is not its JSON")
|
|
}
|
|
var doc map[string]any
|
|
d := json.NewDecoder(bytes.NewReader(glb[20 : 20+n]))
|
|
d.UseNumber() // every number as it was written
|
|
if err := d.Decode(&doc); err != nil {
|
|
return nil, err
|
|
}
|
|
asset, _ := doc["asset"].(map[string]any)
|
|
if asset == nil {
|
|
asset = map[string]any{"version": "2.0"}
|
|
}
|
|
extras, _ := asset["extras"].(map[string]any)
|
|
if extras == nil {
|
|
extras = map[string]any{}
|
|
}
|
|
// The whole licence, as /v1/models gives it: its id, whether it allows
|
|
// commercial use and where it grants nothing are what a pipeline that
|
|
// sorts assets by licence reads, and the name alone is for people.
|
|
var l map[string]any
|
|
b, err := json.Marshal(lic)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := json.Unmarshal(b, &l); err != nil {
|
|
return nil, err
|
|
}
|
|
extras["license"] = l
|
|
asset["extras"] = extras
|
|
doc["asset"] = asset
|
|
js, err := json.Marshal(doc)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// A chunk is padded to four bytes, the JSON one with spaces.
|
|
for len(js)%4 != 0 {
|
|
js = append(js, ' ')
|
|
}
|
|
rest := glb[20+n:]
|
|
out := make([]byte, 0, 20+len(js)+len(rest))
|
|
out = append(out, "glTF"...)
|
|
out = binary.LittleEndian.AppendUint32(out, 2)
|
|
out = binary.LittleEndian.AppendUint32(out, uint32(20+len(js)+len(rest))) //nolint:gosec // bounded by the file read
|
|
out = binary.LittleEndian.AppendUint32(out, uint32(len(js))) //nolint:gosec // bounded by the file read
|
|
out = append(out, "JSON"...)
|
|
out = append(out, js...)
|
|
return append(out, rest...), nil
|
|
}
|
|
|
|
// jobProgress is where a job's request says how far along it is: into the
|
|
// job, for whoever asks after it. Nil for a request that is no job's — and
|
|
// for one that only says it is: the job must be the same key's, so nobody
|
|
// writes into another's progress by naming it in a tag.
|
|
func (r *Router) jobProgress(ctx context.Context, req *parsedRequest) func(comfy.Progress) {
|
|
id := req.Tags["job"]
|
|
if id == "" || r.store == nil || id != req.RequestID {
|
|
return nil
|
|
}
|
|
j, err := r.store.JobByID(ctx, id)
|
|
if err != nil || j.KeyID != req.KeyID {
|
|
return nil
|
|
}
|
|
return func(p comfy.Progress) {
|
|
b, err := json.Marshal(p)
|
|
if err != nil {
|
|
return
|
|
}
|
|
if err := r.store.SetJobProgress(context.WithoutCancel(ctx), id, string(b), p.PromptID); err != nil {
|
|
r.log.Warn("a job's progress could not be kept", "job", id, "err", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// registerMeshesPage mounts the page that makes 3D models, beside the
|
|
// pictures page and for the same reasons: a session, never a key, and the
|
|
// request sent down the path an API client's takes — here, a job.
|
|
func (r *Router) registerMeshesPage(mux *http.ServeMux, pages *web.AccountPages) {
|
|
pages.SetMeshes(func() bool {
|
|
for _, m := range r.modelMap() {
|
|
if m.MakesMeshes() {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
})
|
|
mux.HandleFunc("GET /3d", pages.Guard(func(w http.ResponseWriter, req *http.Request, user store.User) {
|
|
contexts, _ := r.chatContexts(req.Context(), user)
|
|
current, err := r.pickContext(req.Context(), user, req.URL.Query().Get("context"))
|
|
if err != nil {
|
|
current = ""
|
|
}
|
|
if current == "" && !r.personalOff {
|
|
current = personalContext
|
|
}
|
|
web.Meshes(user, r.modelsOf(req, user, catalog.Model.MakesMeshes), contexts, current, version.Version)(w, req)
|
|
}))
|
|
mux.HandleFunc("POST /3d/generate", pages.Guard(r.meshesSend))
|
|
mux.HandleFunc("GET /3d/jobs/{id}", pages.Guard(func(w http.ResponseWriter, req *http.Request, user store.User) {
|
|
if next, ok := r.asPerson(w, req, user, req.URL.Query().Get("context")); ok {
|
|
r.handleGetJob(w, next)
|
|
}
|
|
}))
|
|
// The mesh itself, from where the page is, for its viewer to load.
|
|
mux.HandleFunc("GET /3d/jobs/{id}/file", pages.Guard(func(w http.ResponseWriter, req *http.Request, user store.User) {
|
|
next, ok := r.asPerson(w, req, user, req.URL.Query().Get("context"))
|
|
if !ok {
|
|
return
|
|
}
|
|
p, _ := auth.FromContext(next.Context())
|
|
j, ok := r.ownedJob(w, next, ownerOf(p))
|
|
if !ok {
|
|
return
|
|
}
|
|
if j.OutputFileID == "" {
|
|
http.Error(w, "the job has made nothing yet", http.StatusNotFound)
|
|
return
|
|
}
|
|
b, err := r.store.FileContent(req.Context(), j.OutputFileID)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", j.ContentType)
|
|
w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=%q", j.ID+".glb"))
|
|
_, _ = w.Write(b)
|
|
}))
|
|
}
|
|
|
|
// asPerson is a page's request as the signed-in person's, in the place they
|
|
// chose: what the API handlers read a key's principal from.
|
|
func (r *Router) asPerson(w http.ResponseWriter, req *http.Request, user store.User, context string) (*http.Request, bool) {
|
|
p, err := r.principalFor(req, user, context)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusForbidden)
|
|
return nil, false
|
|
}
|
|
return req.WithContext(auth.WithPrincipal(req.Context(), p)), true
|
|
}
|
|
|
|
// meshesSend asks for a 3D model on behalf of a signed-in person.
|
|
func (r *Router) meshesSend(w http.ResponseWriter, req *http.Request, user store.User) {
|
|
body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes))
|
|
if err != nil {
|
|
http.Error(w, "could not read the request", http.StatusBadRequest)
|
|
return
|
|
}
|
|
var in struct {
|
|
Model string `json:"model"`
|
|
Image string `json:"image"`
|
|
Seed *int64 `json:"seed"`
|
|
Resolution int `json:"resolution"`
|
|
Context string `json:"context"`
|
|
}
|
|
if err := json.Unmarshal(body, &in); err != nil || in.Model == "" {
|
|
http.Error(w, "a model and a picture are required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
// Rebuilt from the fields the page has, so it can set nothing else.
|
|
out := map[string]any{"model": in.Model, "image": in.Image}
|
|
if in.Seed != nil {
|
|
out["seed"] = *in.Seed
|
|
}
|
|
if in.Resolution > 0 {
|
|
out["resolution"] = in.Resolution
|
|
}
|
|
upstream, err := json.Marshal(out)
|
|
if err != nil {
|
|
http.Error(w, "could not build the request", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
next, ok := r.asPerson(w, req, user, in.Context)
|
|
if !ok {
|
|
return
|
|
}
|
|
next = next.Clone(next.Context())
|
|
next.Body = io.NopCloser(bytes.NewReader(upstream))
|
|
next.ContentLength = int64(len(upstream))
|
|
next.Header.Set("Content-Type", "application/json")
|
|
r.handleCreateJob(w, next)
|
|
}
|