Files
llmgw/internal/router/meshes.go
T
Ilker ManapandClaude Opus 5.5 c87e413360 A ComfyUI model may make meshes: from a picture, with its licence inside
G3, the part that needs no model: a ComfyUI model's comfyui.output (or
external.output) says pictures or meshes. A request for a 3D model — a
job at /v1/3d/generations — carries the picture as base64; at the replica
it is uploaded to ComfyUI's input directory (the gate lets /upload/image
through), the workflow is filled in with its name, the seed and the
resolution, and the mesh the workflow saves comes back with its type,
which is the file the job keeps. comfy.Client reads every list of files an
output names, not "images" only, and types a file by its extension where
ComfyUI sends octet-stream.

A model's license (name, url, notice) is said in /v1/models and in the
job, and written into a GLB's asset.extras, since the file travels without
the API; a mesh it cannot be written into is refused. per_mesh in
auth.pricing, usage.meshes in the rows, the sums and the export
(migration 0045). A picture asked of a mesh model, or a mesh of a picture
model, is refused saying which it makes.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-10 18:30:52 +02:00

241 lines
7.5 KiB
Go

package router
import (
"bytes"
"context"
"crypto/rand"
"encoding/base64"
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net/http"
"strconv"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/comfy"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
)
// 3D models from a picture, made by ComfyUI. Clients ask for them as a job
// (jobs.go); the job worker replays the request through the ordinary path,
// and at a ComfyUI replica it arrives here: the picture is put in ComfyUI's
// input directory, the model's workflow is filled in with its name and run,
// and the mesh it saves comes back as the answer — the file the job keeps.
// meshRequest is what a request for a 3D model asks. The picture is the
// whole of the prompt: base64, or a data URL.
type meshRequest struct {
Model string `json:"model"`
Image string `json:"image"`
Seed *int64 `json:"seed"`
Steps int `json:"steps"`
// Resolution is the model's own grid, where it has a choice: TRELLIS.2
// makes 512, 1024 or 1536. Zero leaves it to the workflow.
Resolution int `json:"resolution"`
picture []byte
ext string
}
// maxPicture bounds the picture a mesh is made from.
const maxPicture = 16 << 20
func parseMeshRequest(body []byte) (meshRequest, error) {
var in meshRequest
if err := json.Unmarshal(body, &in); err != nil {
return in, err
}
s := strings.TrimSpace(in.Image)
if s == "" {
return in, errors.New("image is required: the picture to make a 3D model of, base64 or a data URL")
}
if strings.HasPrefix(s, "data:") {
_, after, ok := strings.Cut(s, ";base64,")
if !ok {
return in, errors.New("image is a data URL without base64 in it")
}
s = after
}
if base64.StdEncoding.DecodedLen(len(s)) > maxPicture+3 {
return in, fmt.Errorf("image is past %d MB", maxPicture>>20)
}
b, err := base64.StdEncoding.DecodeString(s)
if err != nil {
return in, errors.New("image is not base64: " + err.Error())
}
switch http.DetectContentType(b) {
case "image/png":
in.ext = ".png"
case "image/jpeg":
in.ext = ".jpg"
case "image/webp":
in.ext = ".webp"
default:
return in, errors.New("image is not a PNG, JPEG or WebP picture")
}
in.picture = b
if in.Steps < 0 || in.Steps > maxSteps {
return in, fmt.Errorf("steps is between 1 and %d", maxSteps)
}
switch in.Resolution {
case 0, 512, 1024, 1536:
default:
return in, fmt.Errorf("resolution %d is not 512, 1024 or 1536", in.Resolution)
}
return in, nil
}
// generateMeshes answers a request for a 3D model at a ComfyUI replica. It
// writes the mesh itself, with its type, which is what the job keeps.
func (r *Router) generateMeshes(ctx context.Context, w http.ResponseWriter, req *parsedRequest, ep endpoint.Info, body []byte) (proxy.Outcome, error) {
in, err := parseMeshRequest(body)
if err != nil {
return proxy.Outcome{}, err
}
tpl, err := comfy.Load(ep.Workflow)
if err != nil {
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
}
if t := r.cfg.Timeouts.Total; t > 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, t)
defer cancel()
}
start := time.Now()
fail := func(err error) (proxy.Outcome, error) {
var ce *comfy.Error
switch {
case errors.As(err, &ce):
return proxy.Outcome{Retryable: ce.Retryable}, err
case errors.Is(err, context.Canceled) && ctxCauseIsClient(ctx):
return proxy.Outcome{}, proxy.ErrClientGone
case errors.Is(err, context.DeadlineExceeded):
return proxy.Outcome{}, fmt.Errorf("%s made no 3D model within %s", ep.ID, r.cfg.Timeouts.Total)
}
return proxy.Outcome{}, err
}
// A name of its own: two requests' pictures must not overwrite each
// other in the replica's input directory.
rnd := make([]byte, 8)
_, _ = rand.Read(rnd)
name, err := r.comfy.Upload(ctx, ep.URL, ep.APIKey, "llmgw-"+hex.EncodeToString(rnd)+in.ext, in.picture)
if err != nil {
return fail(err)
}
seed := comfy.RandomSeed()
if in.Seed != nil {
seed = *in.Seed
}
var upstream struct {
Model string `json:"model"`
}
_ = json.Unmarshal(body, &upstream)
p := comfy.Params{Checkpoint: upstream.Model, Image: name, Seed: seed, Steps: in.Steps, Extra: map[string]any{}}
if in.Resolution > 0 {
p.Extra["resolution"] = in.Resolution
}
workflow, err := tpl.Fill(p)
if err != nil {
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
}
files, err := r.comfy.Generate(ctx, ep.URL, ep.APIKey, workflow)
if err != nil {
return fail(err)
}
var mesh *comfy.Image
for i := range files {
if isMeshType(files[i].ContentType) {
mesh = &files[i]
break
}
}
if mesh == nil {
return proxy.Outcome{}, fmt.Errorf("%s: the workflow saved no mesh (it saved %d other files): "+
"its last node should save a .glb", ep.ID, len(files))
}
data := mesh.Data
m := r.modelMap()[req.Model]
if m.License != nil && mesh.ContentType == "model/gltf-binary" {
if with, err := glbWithLicense(data, *m.License); err == nil {
data = with
} else {
// Refused rather than sent without it: a mesh travels without
// the API, and the notice is what the terms ask to travel too.
return proxy.Outcome{}, fmt.Errorf("%s: the licence could not be written into the mesh: %w", ep.ID, err)
}
}
w.Header().Set("x-llmgw-seed", strconv.FormatInt(seed, 10))
if m.License != nil {
w.Header().Set("x-llmgw-license", m.License.Name)
}
w.Header().Set("Content-Type", mesh.ContentType)
w.WriteHeader(http.StatusOK)
_, _ = w.Write(data)
took := time.Since(start)
return proxy.Outcome{Wrote: true, Status: http.StatusOK, Bytes: int64(len(data)), TTFT: took, Decode: took,
Meshes: 1}, nil
}
func isMeshType(ct string) bool {
return strings.HasPrefix(ct, "model/") || ct == "application/ply"
}
// glbWithLicense is a GLB with the licence in asset.extras.license, where a
// file's terms travel with it: glTF's own place for what an application
// adds, kept by every tool that loads and saves the file.
func glbWithLicense(glb []byte, lic catalog.License) ([]byte, error) {
if len(glb) < 20 || string(glb[:4]) != "glTF" || binary.LittleEndian.Uint32(glb[4:]) != 2 {
return nil, errors.New("not a glTF 2 binary")
}
n := int(binary.LittleEndian.Uint32(glb[12:]))
if string(glb[16:20]) != "JSON" || 20+n > len(glb) {
return nil, errors.New("its first chunk is not its JSON")
}
var doc map[string]any
d := json.NewDecoder(bytes.NewReader(glb[20 : 20+n]))
d.UseNumber() // every number as it was written
if err := d.Decode(&doc); err != nil {
return nil, err
}
asset, _ := doc["asset"].(map[string]any)
if asset == nil {
asset = map[string]any{"version": "2.0"}
}
extras, _ := asset["extras"].(map[string]any)
if extras == nil {
extras = map[string]any{}
}
l := map[string]any{"name": lic.Name}
if lic.URL != "" {
l["url"] = lic.URL
}
if lic.Notice != "" {
l["notice"] = lic.Notice
}
extras["license"] = l
asset["extras"] = extras
doc["asset"] = asset
js, err := json.Marshal(doc)
if err != nil {
return nil, err
}
// A chunk is padded to four bytes, the JSON one with spaces.
for len(js)%4 != 0 {
js = append(js, ' ')
}
rest := glb[20+n:]
out := make([]byte, 0, 20+len(js)+len(rest))
out = append(out, "glTF"...)
out = binary.LittleEndian.AppendUint32(out, 2)
out = binary.LittleEndian.AppendUint32(out, uint32(20+len(js)+len(rest))) //nolint:gosec // bounded by the file read
out = binary.LittleEndian.AppendUint32(out, uint32(len(js))) //nolint:gosec // bounded by the file read
out = append(out, "JSON"...)
out = append(out, js...)
return append(out, rest...), nil
}