CI / test (push) Failing after 6m25s
GLBs carried the licence's name, URL and notice in asset.extras.license. A pipeline that sorts assets by licence needs what a machine matches on: the id, whether commercial use is allowed, the regions it excludes, the revenue cap and the attribution. The whole licence is written, as /v1/models gives it. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
388 lines
13 KiB
Go
388 lines
13 KiB
Go
package router
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"crypto/rand"
|
|
"encoding/base64"
|
|
"encoding/binary"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/comfy"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
|
|
)
|
|
|
|
// 3D models from a picture, made by ComfyUI. Clients ask for them as a job
|
|
// (jobs.go); the job worker replays the request through the ordinary path,
|
|
// and at a ComfyUI replica it arrives here: the picture is put in ComfyUI's
|
|
// input directory, the model's workflow is filled in with its name and run,
|
|
// and the mesh it saves comes back as the answer — the file the job keeps.
|
|
|
|
// meshRequest is what a request for a 3D model asks. The picture is the
|
|
// whole of the prompt: base64, or a data URL.
|
|
type meshRequest struct {
|
|
Model string `json:"model"`
|
|
Image string `json:"image"`
|
|
Seed *int64 `json:"seed"`
|
|
Steps int `json:"steps"`
|
|
// Resolution is the model's own grid, where it has a choice: TripoSR's
|
|
// marching cubes take 256 by default, TRELLIS.2 512 to 1536. Zero leaves
|
|
// it to the workflow.
|
|
Resolution int `json:"resolution"`
|
|
|
|
picture []byte
|
|
ext string
|
|
}
|
|
|
|
// maxPicture bounds the picture a mesh is made from.
|
|
const maxPicture = 16 << 20
|
|
|
|
func parseMeshRequest(body []byte) (meshRequest, error) {
|
|
var in meshRequest
|
|
if err := json.Unmarshal(body, &in); err != nil {
|
|
return in, err
|
|
}
|
|
s := strings.TrimSpace(in.Image)
|
|
if s == "" {
|
|
return in, errors.New("image is required: the picture to make a 3D model of, base64 or a data URL")
|
|
}
|
|
if strings.HasPrefix(s, "data:") {
|
|
_, after, ok := strings.Cut(s, ";base64,")
|
|
if !ok {
|
|
return in, errors.New("image is a data URL without base64 in it")
|
|
}
|
|
s = after
|
|
}
|
|
if base64.StdEncoding.DecodedLen(len(s)) > maxPicture+3 {
|
|
return in, fmt.Errorf("image is past %d MB", maxPicture>>20)
|
|
}
|
|
b, err := base64.StdEncoding.DecodeString(s)
|
|
if err != nil {
|
|
return in, errors.New("image is not base64: " + err.Error())
|
|
}
|
|
switch http.DetectContentType(b) {
|
|
case "image/png":
|
|
in.ext = ".png"
|
|
case "image/jpeg":
|
|
in.ext = ".jpg"
|
|
case "image/webp":
|
|
in.ext = ".webp"
|
|
default:
|
|
return in, errors.New("image is not a PNG, JPEG or WebP picture")
|
|
}
|
|
in.picture = b
|
|
if in.Steps < 0 || in.Steps > maxSteps {
|
|
return in, fmt.Errorf("steps is between 1 and %d", maxSteps)
|
|
}
|
|
if r := in.Resolution; r != 0 && (r < 64 || r > 2048 || r%32 != 0) {
|
|
return in, fmt.Errorf("resolution %d is not a multiple of 32 from 64 to 2048", r)
|
|
}
|
|
return in, nil
|
|
}
|
|
|
|
// generateMeshes answers a request for a 3D model at a ComfyUI replica. It
|
|
// writes the mesh itself, with its type, which is what the job keeps.
|
|
func (r *Router) generateMeshes(ctx context.Context, w http.ResponseWriter, req *parsedRequest, ep endpoint.Info, body []byte) (proxy.Outcome, error) {
|
|
in, err := parseMeshRequest(body)
|
|
if err != nil {
|
|
return proxy.Outcome{}, err
|
|
}
|
|
tpl, err := comfy.Load(ep.Workflow)
|
|
if err != nil {
|
|
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
|
|
}
|
|
if t := r.cfg.Timeouts.Total; t > 0 {
|
|
var cancel context.CancelFunc
|
|
ctx, cancel = context.WithTimeout(ctx, t)
|
|
defer cancel()
|
|
}
|
|
start := time.Now()
|
|
fail := func(err error) (proxy.Outcome, error) {
|
|
var ce *comfy.Error
|
|
switch {
|
|
case errors.As(err, &ce):
|
|
return proxy.Outcome{Retryable: ce.Retryable}, err
|
|
case errors.Is(err, context.Canceled) && ctxCauseIsClient(ctx):
|
|
return proxy.Outcome{}, proxy.ErrClientGone
|
|
case errors.Is(err, context.DeadlineExceeded):
|
|
return proxy.Outcome{}, fmt.Errorf("%s made no 3D model within %s", ep.ID, r.cfg.Timeouts.Total)
|
|
}
|
|
return proxy.Outcome{}, err
|
|
}
|
|
// A name of its own: two requests' pictures must not overwrite each
|
|
// other in the replica's input directory.
|
|
rnd := make([]byte, 8)
|
|
_, _ = rand.Read(rnd)
|
|
name, err := r.comfy.Upload(ctx, ep.URL, ep.APIKey, "llmgw-"+hex.EncodeToString(rnd)+in.ext, in.picture)
|
|
if err != nil {
|
|
return fail(err)
|
|
}
|
|
seed := comfy.RandomSeed()
|
|
if in.Seed != nil {
|
|
seed = *in.Seed
|
|
}
|
|
var upstream struct {
|
|
Model string `json:"model"`
|
|
}
|
|
_ = json.Unmarshal(body, &upstream)
|
|
p := comfy.Params{Checkpoint: upstream.Model, Image: name, Seed: seed, Steps: in.Steps, Extra: map[string]any{}}
|
|
if in.Resolution > 0 {
|
|
p.Extra["resolution"] = in.Resolution
|
|
}
|
|
workflow, err := tpl.Fill(p)
|
|
if err != nil {
|
|
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
|
|
}
|
|
files, err := r.comfy.GenerateWatch(ctx, ep.URL, ep.APIKey, workflow, r.jobProgress(ctx, req))
|
|
if err != nil {
|
|
return fail(err)
|
|
}
|
|
var mesh *comfy.Image
|
|
for i := range files {
|
|
if isMeshType(files[i].ContentType) {
|
|
mesh = &files[i]
|
|
break
|
|
}
|
|
}
|
|
if mesh == nil {
|
|
return proxy.Outcome{}, fmt.Errorf("%s: the workflow saved no mesh (it saved %d other files): "+
|
|
"its last node should save a .glb", ep.ID, len(files))
|
|
}
|
|
data := mesh.Data
|
|
if mesh.ContentType == "model/obj" {
|
|
glb, err := objToGLB(data)
|
|
if err != nil {
|
|
return proxy.Outcome{}, fmt.Errorf("%s: the mesh it saved: %w", ep.ID, err)
|
|
}
|
|
data, mesh.ContentType = glb, "model/gltf-binary"
|
|
}
|
|
m := r.modelMap()[req.Model]
|
|
if m.License != nil && mesh.ContentType == "model/gltf-binary" {
|
|
if with, err := glbWithLicense(data, *m.License); err == nil {
|
|
data = with
|
|
} else {
|
|
// Refused rather than sent without it: a mesh travels without
|
|
// the API, and the notice is what the terms ask to travel too.
|
|
return proxy.Outcome{}, fmt.Errorf("%s: the licence could not be written into the mesh: %w", ep.ID, err)
|
|
}
|
|
}
|
|
w.Header().Set("x-llmgw-seed", strconv.FormatInt(seed, 10))
|
|
if m.License != nil {
|
|
w.Header().Set("x-llmgw-license", m.License.Name)
|
|
}
|
|
w.Header().Set("Content-Type", mesh.ContentType)
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write(data)
|
|
took := time.Since(start)
|
|
return proxy.Outcome{Wrote: true, Status: http.StatusOK, Bytes: int64(len(data)), TTFT: took, Decode: took,
|
|
Meshes: 1}, nil
|
|
}
|
|
|
|
func isMeshType(ct string) bool {
|
|
return strings.HasPrefix(ct, "model/") || ct == "application/ply"
|
|
}
|
|
|
|
// glbWithLicense is a GLB with the licence in asset.extras.license, where a
|
|
// file's terms travel with it: glTF's own place for what an application
|
|
// adds, kept by every tool that loads and saves the file.
|
|
func glbWithLicense(glb []byte, lic catalog.License) ([]byte, error) {
|
|
if len(glb) < 20 || string(glb[:4]) != "glTF" || binary.LittleEndian.Uint32(glb[4:]) != 2 {
|
|
return nil, errors.New("not a glTF 2 binary")
|
|
}
|
|
n := int(binary.LittleEndian.Uint32(glb[12:]))
|
|
if string(glb[16:20]) != "JSON" || 20+n > len(glb) {
|
|
return nil, errors.New("its first chunk is not its JSON")
|
|
}
|
|
var doc map[string]any
|
|
d := json.NewDecoder(bytes.NewReader(glb[20 : 20+n]))
|
|
d.UseNumber() // every number as it was written
|
|
if err := d.Decode(&doc); err != nil {
|
|
return nil, err
|
|
}
|
|
asset, _ := doc["asset"].(map[string]any)
|
|
if asset == nil {
|
|
asset = map[string]any{"version": "2.0"}
|
|
}
|
|
extras, _ := asset["extras"].(map[string]any)
|
|
if extras == nil {
|
|
extras = map[string]any{}
|
|
}
|
|
// The whole licence, as /v1/models gives it: its id, whether it allows
|
|
// commercial use and where it grants nothing are what a pipeline that
|
|
// sorts assets by licence reads, and the name alone is for people.
|
|
var l map[string]any
|
|
b, err := json.Marshal(lic)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if err := json.Unmarshal(b, &l); err != nil {
|
|
return nil, err
|
|
}
|
|
extras["license"] = l
|
|
asset["extras"] = extras
|
|
doc["asset"] = asset
|
|
js, err := json.Marshal(doc)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// A chunk is padded to four bytes, the JSON one with spaces.
|
|
for len(js)%4 != 0 {
|
|
js = append(js, ' ')
|
|
}
|
|
rest := glb[20+n:]
|
|
out := make([]byte, 0, 20+len(js)+len(rest))
|
|
out = append(out, "glTF"...)
|
|
out = binary.LittleEndian.AppendUint32(out, 2)
|
|
out = binary.LittleEndian.AppendUint32(out, uint32(20+len(js)+len(rest))) //nolint:gosec // bounded by the file read
|
|
out = binary.LittleEndian.AppendUint32(out, uint32(len(js))) //nolint:gosec // bounded by the file read
|
|
out = append(out, "JSON"...)
|
|
out = append(out, js...)
|
|
return append(out, rest...), nil
|
|
}
|
|
|
|
// jobProgress is where a job's request says how far along it is: into the
|
|
// job, for whoever asks after it. Nil for a request that is no job's — and
|
|
// for one that only says it is: the job must be the same key's, so nobody
|
|
// writes into another's progress by naming it in a tag.
|
|
func (r *Router) jobProgress(ctx context.Context, req *parsedRequest) func(comfy.Progress) {
|
|
id := req.Tags["job"]
|
|
if id == "" || r.store == nil || id != req.RequestID {
|
|
return nil
|
|
}
|
|
j, err := r.store.JobByID(ctx, id)
|
|
if err != nil || j.KeyID != req.KeyID {
|
|
return nil
|
|
}
|
|
return func(p comfy.Progress) {
|
|
b, err := json.Marshal(p)
|
|
if err != nil {
|
|
return
|
|
}
|
|
if err := r.store.SetJobProgress(context.WithoutCancel(ctx), id, string(b), p.PromptID); err != nil {
|
|
r.log.Warn("a job's progress could not be kept", "job", id, "err", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// registerMeshesPage mounts the page that makes 3D models, beside the
|
|
// pictures page and for the same reasons: a session, never a key, and the
|
|
// request sent down the path an API client's takes — here, a job.
|
|
func (r *Router) registerMeshesPage(mux *http.ServeMux, pages *web.AccountPages) {
|
|
pages.SetMeshes(func() bool {
|
|
for _, m := range r.modelMap() {
|
|
if m.MakesMeshes() {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
})
|
|
mux.HandleFunc("GET /3d", pages.Guard(func(w http.ResponseWriter, req *http.Request, user store.User) {
|
|
contexts, _ := r.chatContexts(req.Context(), user)
|
|
current, err := r.pickContext(req.Context(), user, req.URL.Query().Get("context"))
|
|
if err != nil {
|
|
current = ""
|
|
}
|
|
if current == "" && !r.personalOff {
|
|
current = personalContext
|
|
}
|
|
web.Meshes(user, r.modelsOf(req, user, catalog.Model.MakesMeshes), contexts, current, version.Version)(w, req)
|
|
}))
|
|
mux.HandleFunc("POST /3d/generate", pages.Guard(r.meshesSend))
|
|
mux.HandleFunc("GET /3d/jobs/{id}", pages.Guard(func(w http.ResponseWriter, req *http.Request, user store.User) {
|
|
if next, ok := r.asPerson(w, req, user, req.URL.Query().Get("context")); ok {
|
|
r.handleGetJob(w, next)
|
|
}
|
|
}))
|
|
// The mesh itself, from where the page is, for its viewer to load.
|
|
mux.HandleFunc("GET /3d/jobs/{id}/file", pages.Guard(func(w http.ResponseWriter, req *http.Request, user store.User) {
|
|
next, ok := r.asPerson(w, req, user, req.URL.Query().Get("context"))
|
|
if !ok {
|
|
return
|
|
}
|
|
p, _ := auth.FromContext(next.Context())
|
|
j, ok := r.ownedJob(w, next, ownerOf(p))
|
|
if !ok {
|
|
return
|
|
}
|
|
if j.OutputFileID == "" {
|
|
http.Error(w, "the job has made nothing yet", http.StatusNotFound)
|
|
return
|
|
}
|
|
b, err := r.store.FileContent(req.Context(), j.OutputFileID)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", j.ContentType)
|
|
w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=%q", j.ID+".glb"))
|
|
_, _ = w.Write(b)
|
|
}))
|
|
}
|
|
|
|
// asPerson is a page's request as the signed-in person's, in the place they
|
|
// chose: what the API handlers read a key's principal from.
|
|
func (r *Router) asPerson(w http.ResponseWriter, req *http.Request, user store.User, context string) (*http.Request, bool) {
|
|
p, err := r.principalFor(req, user, context)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusForbidden)
|
|
return nil, false
|
|
}
|
|
return req.WithContext(auth.WithPrincipal(req.Context(), p)), true
|
|
}
|
|
|
|
// meshesSend asks for a 3D model on behalf of a signed-in person.
|
|
func (r *Router) meshesSend(w http.ResponseWriter, req *http.Request, user store.User) {
|
|
body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes))
|
|
if err != nil {
|
|
http.Error(w, "could not read the request", http.StatusBadRequest)
|
|
return
|
|
}
|
|
var in struct {
|
|
Model string `json:"model"`
|
|
Image string `json:"image"`
|
|
Seed *int64 `json:"seed"`
|
|
Resolution int `json:"resolution"`
|
|
Context string `json:"context"`
|
|
}
|
|
if err := json.Unmarshal(body, &in); err != nil || in.Model == "" {
|
|
http.Error(w, "a model and a picture are required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
// Rebuilt from the fields the page has, so it can set nothing else.
|
|
out := map[string]any{"model": in.Model, "image": in.Image}
|
|
if in.Seed != nil {
|
|
out["seed"] = *in.Seed
|
|
}
|
|
if in.Resolution > 0 {
|
|
out["resolution"] = in.Resolution
|
|
}
|
|
upstream, err := json.Marshal(out)
|
|
if err != nil {
|
|
http.Error(w, "could not build the request", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
next, ok := r.asPerson(w, req, user, in.Context)
|
|
if !ok {
|
|
return
|
|
}
|
|
next = next.Clone(next.Context())
|
|
next.Body = io.NopCloser(bytes.NewReader(upstream))
|
|
next.ContentLength = int64(len(upstream))
|
|
next.Header.Set("Content-Type", "application/json")
|
|
r.handleCreateJob(w, next)
|
|
}
|