Files
llmgw/internal/router/images.go
T
Ilker ManapandClaude Opus 5.5 c87e413360 A ComfyUI model may make meshes: from a picture, with its licence inside
G3, the part that needs no model: a ComfyUI model's comfyui.output (or
external.output) says pictures or meshes. A request for a 3D model — a
job at /v1/3d/generations — carries the picture as base64; at the replica
it is uploaded to ComfyUI's input directory (the gate lets /upload/image
through), the workflow is filled in with its name, the seed and the
resolution, and the mesh the workflow saves comes back with its type,
which is the file the job keeps. comfy.Client reads every list of files an
output names, not "images" only, and types a file by its extension where
ComfyUI sends octet-stream.

A model's license (name, url, notice) is said in /v1/models and in the
job, and written into a GLB's asset.extras, since the file travels without
the API; a mesh it cannot be written into is refused. per_mesh in
auth.pricing, usage.meshes in the rows, the sums and the export
(migration 0045). A picture asked of a mesh model, or a mesh of a picture
model, is refused saying which it makes.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-10 18:30:52 +02:00

307 lines
10 KiB
Go

package router
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"strconv"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/comfy"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
)
// /v1/images/generations, OpenAI's shape, made by ComfyUI: the request goes
// through the same checks as any other — key, allowlist, limits, admission,
// guardrails on its prompt — and only at the replica is it turned into a
// workflow, queued and waited on, and its pictures sent back as base64.
const imagesPath = "/v1/images/generations"
// imageRequest is what a request for pictures asks. Beyond OpenAI's fields,
// those a diffusion model has and DALL·E does not: seed, steps,
// negative_prompt and guidance.
type imageRequest struct {
Prompt string `json:"prompt"`
N int `json:"n"`
Size string `json:"size"`
ResponseFormat string `json:"response_format"`
Seed *int64 `json:"seed"`
Steps int `json:"steps"`
NegativePrompt string `json:"negative_prompt"`
Guidance *float64 `json:"guidance"`
width, height int
}
// The bounds of a request. A FLUX latent is 1/16 of the picture, so a side
// that is not a multiple of 16 is not one it can make; the rest keep one
// request from holding a card for the length of a film.
const (
maxImages = 4
minSide = 256
maxSide = 2048
maxSteps = 100
defaultSide = 1024
)
func parseImageRequest(body []byte) (imageRequest, error) {
var in imageRequest
if err := json.Unmarshal(body, &in); err != nil {
return in, err
}
if strings.TrimSpace(in.Prompt) == "" {
return in, errors.New("prompt is required")
}
switch {
case in.N == 0:
in.N = 1
case in.N < 0 || in.N > maxImages:
return in, fmt.Errorf("n is between 1 and %d", maxImages)
}
switch in.ResponseFormat {
case "", "b64_json":
case "url":
// A URL would have to be served by somebody and kept for some time,
// and a picture made from a prompt is the prompt's owner's: it is
// not kept here.
return in, errors.New(`response_format "url" is not offered: the picture comes back as b64_json and is kept nowhere`)
default:
return in, fmt.Errorf("response_format %q is not b64_json", in.ResponseFormat)
}
in.width, in.height = defaultSide, defaultSide
if s := strings.TrimSpace(in.Size); s != "" && s != "auto" {
w, h, ok := strings.Cut(strings.ToLower(s), "x")
wi, err1 := strconv.Atoi(w)
hi, err2 := strconv.Atoi(h)
if !ok || err1 != nil || err2 != nil {
return in, fmt.Errorf("size %q is not WIDTHxHEIGHT", s)
}
in.width, in.height = wi, hi
}
for _, side := range []int{in.width, in.height} {
if side < minSide || side > maxSide || side%16 != 0 {
return in, fmt.Errorf("size %dx%d: each side is a multiple of 16 from %d to %d", in.width, in.height, minSide, maxSide)
}
}
if in.Steps < 0 || in.Steps > maxSteps {
return in, fmt.Errorf("steps is between 1 and %d", maxSteps)
}
if in.Guidance != nil && (*in.Guidance <= 0 || *in.Guidance > 30) {
return in, errors.New("guidance is above 0 and at most 30")
}
return in, nil
}
// imageRouteRefusal says why a request cannot be answered at the path it was
// sent to, before it takes a place in a queue: a picture asked of a model
// that writes text, or text of one that makes pictures. Empty when it can,
// and for a model the catalog does not know, which a hand-configured
// endpoint may serve as it likes.
func (r *Router) imageRouteRefusal(path, model string, body []byte) string {
m, known := r.modelMap()[model]
if path == meshesPath {
if known && !m.MakesMeshes() {
if m.MakesImages() {
return fmt.Sprintf("%s makes pictures, not 3D models: ask for them at %s", m.Name, imagesPath)
}
return fmt.Sprintf("%s does not make 3D models", m.Name)
}
if _, err := parseMeshRequest(body); err != nil {
return err.Error()
}
return ""
}
if path == imagesPath {
if known && m.MakesMeshes() && !m.MakesImages() {
return fmt.Sprintf("%s makes 3D models, not pictures: ask for one at %s, as a job", m.Name, meshesPath)
}
if known && !m.MakesImages() {
return fmt.Sprintf("%s does not make pictures", m.Name)
}
if _, err := parseImageRequest(body); err != nil {
return err.Error()
}
return ""
}
if known && m.ServesComfyUI() {
if m.MakesMeshes() {
return fmt.Sprintf("%s makes 3D models: ask for one at %s, as a job", m.Name, meshesPath)
}
return fmt.Sprintf("%s makes pictures: ask for them at %s", m.Name, imagesPath)
}
return ""
}
// generateImages answers a request at a ComfyUI replica. It returns what the
// proxy would: a failure of this replica that another may not have is
// retryable, and nothing is written until the pictures are all in hand.
func (r *Router) generateImages(ctx context.Context, w http.ResponseWriter, req *parsedRequest, ep endpoint.Info, body []byte) (proxy.Outcome, error) {
if req.Path == meshesPath {
return r.generateMeshes(ctx, w, req, ep, body)
}
if req.Path != imagesPath {
return proxy.Outcome{}, fmt.Errorf("%s is a ComfyUI server: it answers %s and %s only", ep.ID, imagesPath, meshesPath)
}
in, err := parseImageRequest(body)
if err != nil {
return proxy.Outcome{}, err
}
var upstream struct {
Model string `json:"model"`
}
_ = json.Unmarshal(body, &upstream)
tpl, err := comfy.Load(ep.Workflow)
if err != nil {
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
}
seed := comfy.RandomSeed()
if in.Seed != nil {
seed = *in.Seed
}
p := comfy.Params{
Checkpoint: upstream.Model, Prompt: in.Prompt, NegativePrompt: in.NegativePrompt,
Width: in.width, Height: in.height, N: in.N, Seed: seed, Steps: in.Steps,
}
if in.Guidance != nil {
p.Guidance = *in.Guidance
}
workflow, err := tpl.Fill(p)
if err != nil {
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
}
if t := r.cfg.Timeouts.Total; t > 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, t)
defer cancel()
}
start := time.Now()
images, err := r.comfy.Generate(ctx, ep.URL, ep.APIKey, workflow)
if err != nil {
var ce *comfy.Error
switch {
case errors.As(err, &ce):
return proxy.Outcome{Retryable: ce.Retryable}, err
case errors.Is(err, context.Canceled) && ctxCauseIsClient(ctx):
return proxy.Outcome{}, proxy.ErrClientGone
case errors.Is(err, context.DeadlineExceeded):
return proxy.Outcome{}, fmt.Errorf("%s made no picture within %s", ep.ID, r.cfg.Timeouts.Total)
}
return proxy.Outcome{}, err
}
data := make([]map[string]any, 0, len(images))
for _, im := range images {
data = append(data, map[string]any{"b64_json": base64.StdEncoding.EncodeToString(im.Data)})
}
out, err := json.Marshal(map[string]any{"created": time.Now().Unix(), "data": data})
if err != nil {
return proxy.Outcome{}, err
}
// The seed that made them, so a picture liked can be made again.
w.Header().Set("x-llmgw-seed", strconv.FormatInt(seed, 10))
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, _ = w.Write(out)
took := time.Since(start)
return proxy.Outcome{Wrote: true, Status: http.StatusOK, Bytes: int64(len(out)), TTFT: took, Decode: took,
Images: len(images)}, nil
}
// ctxCauseIsClient reports whether a context ended because the client went
// away, rather than because a deadline set here passed.
func ctxCauseIsClient(ctx context.Context) bool {
return !errors.Is(context.Cause(ctx), context.DeadlineExceeded)
}
// registerImagesPage mounts the page that makes pictures, beside the chat
// and for the same reasons: a model tried without a client, with the
// browser holding a session and never a key, and the request sent down the
// one path every API request takes.
func (r *Router) registerImagesPage(mux *http.ServeMux, pages *web.AccountPages) {
pages.SetImages(func() bool {
for _, m := range r.modelMap() {
if m.MakesImages() {
return true
}
}
return false
})
mux.HandleFunc("GET /images", pages.Guard(func(w http.ResponseWriter, req *http.Request, user store.User) {
contexts, _ := r.chatContexts(req.Context(), user)
current, err := r.pickContext(req.Context(), user, req.URL.Query().Get("context"))
if err != nil {
current = ""
}
if current == "" && !r.personalOff {
current = personalContext
}
web.Images(user, r.modelsOf(req, user, true), contexts, current, version.Version)(w, req)
}))
mux.HandleFunc("POST /images/generate", pages.Guard(r.imagesSend))
}
// imagesSend makes pictures on behalf of a signed-in person.
func (r *Router) imagesSend(w http.ResponseWriter, req *http.Request, user store.User) {
body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes))
if err != nil {
http.Error(w, "could not read the request", http.StatusBadRequest)
return
}
var in struct {
Model string `json:"model"`
Prompt string `json:"prompt"`
Size string `json:"size"`
N int `json:"n"`
Seed *int64 `json:"seed"`
Steps int `json:"steps"`
NegativePrompt string `json:"negative_prompt"`
// Context is what the pictures are charged to, checked as the
// chat's is.
Context string `json:"context"`
}
if err := json.Unmarshal(body, &in); err != nil || in.Model == "" {
http.Error(w, "a model and a prompt are required", http.StatusBadRequest)
return
}
principal, err := r.principalFor(req, user, in.Context)
if err != nil {
http.Error(w, err.Error(), http.StatusForbidden)
return
}
// Rebuilt from the fields the page has, so it can set nothing else.
out := map[string]any{"model": in.Model, "prompt": in.Prompt, "size": in.Size, "n": in.N}
if in.Seed != nil {
out["seed"] = *in.Seed
}
if in.Steps > 0 {
out["steps"] = in.Steps
}
if s := strings.TrimSpace(in.NegativePrompt); s != "" {
out["negative_prompt"] = s
}
upstream, err := json.Marshal(out)
if err != nil {
http.Error(w, "could not build the request", http.StatusInternalServerError)
return
}
next := req.Clone(auth.WithPrincipal(req.Context(), principal))
next.Body = io.NopCloser(bytes.NewReader(upstream))
next.ContentLength = int64(len(upstream))
next.Header.Set("Content-Type", "application/json")
next.Header.Del(knowledgeHeader)
r.handleInference(imagesPath)(w, next)
}