Files
llmgw/internal/router/meshes.go
T
Ilker ManapandClaude Opus 5.5 2eb2a6d8fe
CI / test (push) Failing after 6m25s
A mesh carries its whole licence, not only its name
GLBs carried the licence's name, URL and notice in asset.extras.license.
A pipeline that sorts assets by licence needs what a machine matches on:
the id, whether commercial use is allowed, the regions it excludes, the
revenue cap and the attribution. The whole licence is written, as
/v1/models gives it.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-14 01:12:47 +02:00

388 lines
13 KiB
Go

package router
import (
"bytes"
"context"
"crypto/rand"
"encoding/base64"
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/comfy"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
)
// 3D models from a picture, made by ComfyUI. Clients ask for them as a job
// (jobs.go); the job worker replays the request through the ordinary path,
// and at a ComfyUI replica it arrives here: the picture is put in ComfyUI's
// input directory, the model's workflow is filled in with its name and run,
// and the mesh it saves comes back as the answer — the file the job keeps.
// meshRequest is what a request for a 3D model asks. The picture is the
// whole of the prompt: base64, or a data URL.
type meshRequest struct {
Model string `json:"model"`
Image string `json:"image"`
Seed *int64 `json:"seed"`
Steps int `json:"steps"`
// Resolution is the model's own grid, where it has a choice: TripoSR's
// marching cubes take 256 by default, TRELLIS.2 512 to 1536. Zero leaves
// it to the workflow.
Resolution int `json:"resolution"`
picture []byte
ext string
}
// maxPicture bounds the picture a mesh is made from.
const maxPicture = 16 << 20
func parseMeshRequest(body []byte) (meshRequest, error) {
var in meshRequest
if err := json.Unmarshal(body, &in); err != nil {
return in, err
}
s := strings.TrimSpace(in.Image)
if s == "" {
return in, errors.New("image is required: the picture to make a 3D model of, base64 or a data URL")
}
if strings.HasPrefix(s, "data:") {
_, after, ok := strings.Cut(s, ";base64,")
if !ok {
return in, errors.New("image is a data URL without base64 in it")
}
s = after
}
if base64.StdEncoding.DecodedLen(len(s)) > maxPicture+3 {
return in, fmt.Errorf("image is past %d MB", maxPicture>>20)
}
b, err := base64.StdEncoding.DecodeString(s)
if err != nil {
return in, errors.New("image is not base64: " + err.Error())
}
switch http.DetectContentType(b) {
case "image/png":
in.ext = ".png"
case "image/jpeg":
in.ext = ".jpg"
case "image/webp":
in.ext = ".webp"
default:
return in, errors.New("image is not a PNG, JPEG or WebP picture")
}
in.picture = b
if in.Steps < 0 || in.Steps > maxSteps {
return in, fmt.Errorf("steps is between 1 and %d", maxSteps)
}
if r := in.Resolution; r != 0 && (r < 64 || r > 2048 || r%32 != 0) {
return in, fmt.Errorf("resolution %d is not a multiple of 32 from 64 to 2048", r)
}
return in, nil
}
// generateMeshes answers a request for a 3D model at a ComfyUI replica. It
// writes the mesh itself, with its type, which is what the job keeps.
func (r *Router) generateMeshes(ctx context.Context, w http.ResponseWriter, req *parsedRequest, ep endpoint.Info, body []byte) (proxy.Outcome, error) {
in, err := parseMeshRequest(body)
if err != nil {
return proxy.Outcome{}, err
}
tpl, err := comfy.Load(ep.Workflow)
if err != nil {
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
}
if t := r.cfg.Timeouts.Total; t > 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, t)
defer cancel()
}
start := time.Now()
fail := func(err error) (proxy.Outcome, error) {
var ce *comfy.Error
switch {
case errors.As(err, &ce):
return proxy.Outcome{Retryable: ce.Retryable}, err
case errors.Is(err, context.Canceled) && ctxCauseIsClient(ctx):
return proxy.Outcome{}, proxy.ErrClientGone
case errors.Is(err, context.DeadlineExceeded):
return proxy.Outcome{}, fmt.Errorf("%s made no 3D model within %s", ep.ID, r.cfg.Timeouts.Total)
}
return proxy.Outcome{}, err
}
// A name of its own: two requests' pictures must not overwrite each
// other in the replica's input directory.
rnd := make([]byte, 8)
_, _ = rand.Read(rnd)
name, err := r.comfy.Upload(ctx, ep.URL, ep.APIKey, "llmgw-"+hex.EncodeToString(rnd)+in.ext, in.picture)
if err != nil {
return fail(err)
}
seed := comfy.RandomSeed()
if in.Seed != nil {
seed = *in.Seed
}
var upstream struct {
Model string `json:"model"`
}
_ = json.Unmarshal(body, &upstream)
p := comfy.Params{Checkpoint: upstream.Model, Image: name, Seed: seed, Steps: in.Steps, Extra: map[string]any{}}
if in.Resolution > 0 {
p.Extra["resolution"] = in.Resolution
}
workflow, err := tpl.Fill(p)
if err != nil {
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
}
files, err := r.comfy.GenerateWatch(ctx, ep.URL, ep.APIKey, workflow, r.jobProgress(ctx, req))
if err != nil {
return fail(err)
}
var mesh *comfy.Image
for i := range files {
if isMeshType(files[i].ContentType) {
mesh = &files[i]
break
}
}
if mesh == nil {
return proxy.Outcome{}, fmt.Errorf("%s: the workflow saved no mesh (it saved %d other files): "+
"its last node should save a .glb", ep.ID, len(files))
}
data := mesh.Data
if mesh.ContentType == "model/obj" {
glb, err := objToGLB(data)
if err != nil {
return proxy.Outcome{}, fmt.Errorf("%s: the mesh it saved: %w", ep.ID, err)
}
data, mesh.ContentType = glb, "model/gltf-binary"
}
m := r.modelMap()[req.Model]
if m.License != nil && mesh.ContentType == "model/gltf-binary" {
if with, err := glbWithLicense(data, *m.License); err == nil {
data = with
} else {
// Refused rather than sent without it: a mesh travels without
// the API, and the notice is what the terms ask to travel too.
return proxy.Outcome{}, fmt.Errorf("%s: the licence could not be written into the mesh: %w", ep.ID, err)
}
}
w.Header().Set("x-llmgw-seed", strconv.FormatInt(seed, 10))
if m.License != nil {
w.Header().Set("x-llmgw-license", m.License.Name)
}
w.Header().Set("Content-Type", mesh.ContentType)
w.WriteHeader(http.StatusOK)
_, _ = w.Write(data)
took := time.Since(start)
return proxy.Outcome{Wrote: true, Status: http.StatusOK, Bytes: int64(len(data)), TTFT: took, Decode: took,
Meshes: 1}, nil
}
func isMeshType(ct string) bool {
return strings.HasPrefix(ct, "model/") || ct == "application/ply"
}
// glbWithLicense is a GLB with the licence in asset.extras.license, where a
// file's terms travel with it: glTF's own place for what an application
// adds, kept by every tool that loads and saves the file.
func glbWithLicense(glb []byte, lic catalog.License) ([]byte, error) {
if len(glb) < 20 || string(glb[:4]) != "glTF" || binary.LittleEndian.Uint32(glb[4:]) != 2 {
return nil, errors.New("not a glTF 2 binary")
}
n := int(binary.LittleEndian.Uint32(glb[12:]))
if string(glb[16:20]) != "JSON" || 20+n > len(glb) {
return nil, errors.New("its first chunk is not its JSON")
}
var doc map[string]any
d := json.NewDecoder(bytes.NewReader(glb[20 : 20+n]))
d.UseNumber() // every number as it was written
if err := d.Decode(&doc); err != nil {
return nil, err
}
asset, _ := doc["asset"].(map[string]any)
if asset == nil {
asset = map[string]any{"version": "2.0"}
}
extras, _ := asset["extras"].(map[string]any)
if extras == nil {
extras = map[string]any{}
}
// The whole licence, as /v1/models gives it: its id, whether it allows
// commercial use and where it grants nothing are what a pipeline that
// sorts assets by licence reads, and the name alone is for people.
var l map[string]any
b, err := json.Marshal(lic)
if err != nil {
return nil, err
}
if err := json.Unmarshal(b, &l); err != nil {
return nil, err
}
extras["license"] = l
asset["extras"] = extras
doc["asset"] = asset
js, err := json.Marshal(doc)
if err != nil {
return nil, err
}
// A chunk is padded to four bytes, the JSON one with spaces.
for len(js)%4 != 0 {
js = append(js, ' ')
}
rest := glb[20+n:]
out := make([]byte, 0, 20+len(js)+len(rest))
out = append(out, "glTF"...)
out = binary.LittleEndian.AppendUint32(out, 2)
out = binary.LittleEndian.AppendUint32(out, uint32(20+len(js)+len(rest))) //nolint:gosec // bounded by the file read
out = binary.LittleEndian.AppendUint32(out, uint32(len(js))) //nolint:gosec // bounded by the file read
out = append(out, "JSON"...)
out = append(out, js...)
return append(out, rest...), nil
}
// jobProgress is where a job's request says how far along it is: into the
// job, for whoever asks after it. Nil for a request that is no job's — and
// for one that only says it is: the job must be the same key's, so nobody
// writes into another's progress by naming it in a tag.
func (r *Router) jobProgress(ctx context.Context, req *parsedRequest) func(comfy.Progress) {
id := req.Tags["job"]
if id == "" || r.store == nil || id != req.RequestID {
return nil
}
j, err := r.store.JobByID(ctx, id)
if err != nil || j.KeyID != req.KeyID {
return nil
}
return func(p comfy.Progress) {
b, err := json.Marshal(p)
if err != nil {
return
}
if err := r.store.SetJobProgress(context.WithoutCancel(ctx), id, string(b), p.PromptID); err != nil {
r.log.Warn("a job's progress could not be kept", "job", id, "err", err)
}
}
}
// registerMeshesPage mounts the page that makes 3D models, beside the
// pictures page and for the same reasons: a session, never a key, and the
// request sent down the path an API client's takes — here, a job.
func (r *Router) registerMeshesPage(mux *http.ServeMux, pages *web.AccountPages) {
pages.SetMeshes(func() bool {
for _, m := range r.modelMap() {
if m.MakesMeshes() {
return true
}
}
return false
})
mux.HandleFunc("GET /3d", pages.Guard(func(w http.ResponseWriter, req *http.Request, user store.User) {
contexts, _ := r.chatContexts(req.Context(), user)
current, err := r.pickContext(req.Context(), user, req.URL.Query().Get("context"))
if err != nil {
current = ""
}
if current == "" && !r.personalOff {
current = personalContext
}
web.Meshes(user, r.modelsOf(req, user, catalog.Model.MakesMeshes), contexts, current, version.Version)(w, req)
}))
mux.HandleFunc("POST /3d/generate", pages.Guard(r.meshesSend))
mux.HandleFunc("GET /3d/jobs/{id}", pages.Guard(func(w http.ResponseWriter, req *http.Request, user store.User) {
if next, ok := r.asPerson(w, req, user, req.URL.Query().Get("context")); ok {
r.handleGetJob(w, next)
}
}))
// The mesh itself, from where the page is, for its viewer to load.
mux.HandleFunc("GET /3d/jobs/{id}/file", pages.Guard(func(w http.ResponseWriter, req *http.Request, user store.User) {
next, ok := r.asPerson(w, req, user, req.URL.Query().Get("context"))
if !ok {
return
}
p, _ := auth.FromContext(next.Context())
j, ok := r.ownedJob(w, next, ownerOf(p))
if !ok {
return
}
if j.OutputFileID == "" {
http.Error(w, "the job has made nothing yet", http.StatusNotFound)
return
}
b, err := r.store.FileContent(req.Context(), j.OutputFileID)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", j.ContentType)
w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=%q", j.ID+".glb"))
_, _ = w.Write(b)
}))
}
// asPerson is a page's request as the signed-in person's, in the place they
// chose: what the API handlers read a key's principal from.
func (r *Router) asPerson(w http.ResponseWriter, req *http.Request, user store.User, context string) (*http.Request, bool) {
p, err := r.principalFor(req, user, context)
if err != nil {
http.Error(w, err.Error(), http.StatusForbidden)
return nil, false
}
return req.WithContext(auth.WithPrincipal(req.Context(), p)), true
}
// meshesSend asks for a 3D model on behalf of a signed-in person.
func (r *Router) meshesSend(w http.ResponseWriter, req *http.Request, user store.User) {
body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes))
if err != nil {
http.Error(w, "could not read the request", http.StatusBadRequest)
return
}
var in struct {
Model string `json:"model"`
Image string `json:"image"`
Seed *int64 `json:"seed"`
Resolution int `json:"resolution"`
Context string `json:"context"`
}
if err := json.Unmarshal(body, &in); err != nil || in.Model == "" {
http.Error(w, "a model and a picture are required", http.StatusBadRequest)
return
}
// Rebuilt from the fields the page has, so it can set nothing else.
out := map[string]any{"model": in.Model, "image": in.Image}
if in.Seed != nil {
out["seed"] = *in.Seed
}
if in.Resolution > 0 {
out["resolution"] = in.Resolution
}
upstream, err := json.Marshal(out)
if err != nil {
http.Error(w, "could not build the request", http.StatusInternalServerError)
return
}
next, ok := r.asPerson(w, req, user, in.Context)
if !ok {
return
}
next = next.Clone(next.Context())
next.Body = io.NopCloser(bytes.NewReader(upstream))
next.ContentLength = int64(len(upstream))
next.Header.Set("Content-Type", "application/json")
r.handleCreateJob(w, next)
}