G3, the part that needs no model: a ComfyUI model's comfyui.output (or external.output) says pictures or meshes. A request for a 3D model — a job at /v1/3d/generations — carries the picture as base64; at the replica it is uploaded to ComfyUI's input directory (the gate lets /upload/image through), the workflow is filled in with its name, the seed and the resolution, and the mesh the workflow saves comes back with its type, which is the file the job keeps. comfy.Client reads every list of files an output names, not "images" only, and types a file by its extension where ComfyUI sends octet-stream. A model's license (name, url, notice) is said in /v1/models and in the job, and written into a GLB's asset.extras, since the file travels without the API; a mesh it cannot be written into is refused. per_mesh in auth.pricing, usage.meshes in the rows, the sums and the export (migration 0045). A picture asked of a mesh model, or a mesh of a picture model, is refused saying which it makes. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
307 lines
10 KiB
Go
307 lines
10 KiB
Go
package router
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/comfy"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/web"
|
|
)
|
|
|
|
// /v1/images/generations, OpenAI's shape, made by ComfyUI: the request goes
|
|
// through the same checks as any other — key, allowlist, limits, admission,
|
|
// guardrails on its prompt — and only at the replica is it turned into a
|
|
// workflow, queued and waited on, and its pictures sent back as base64.
|
|
|
|
const imagesPath = "/v1/images/generations"
|
|
|
|
// imageRequest is what a request for pictures asks. Beyond OpenAI's fields,
|
|
// those a diffusion model has and DALL·E does not: seed, steps,
|
|
// negative_prompt and guidance.
|
|
type imageRequest struct {
|
|
Prompt string `json:"prompt"`
|
|
N int `json:"n"`
|
|
Size string `json:"size"`
|
|
ResponseFormat string `json:"response_format"`
|
|
Seed *int64 `json:"seed"`
|
|
Steps int `json:"steps"`
|
|
NegativePrompt string `json:"negative_prompt"`
|
|
Guidance *float64 `json:"guidance"`
|
|
|
|
width, height int
|
|
}
|
|
|
|
// The bounds of a request. A FLUX latent is 1/16 of the picture, so a side
|
|
// that is not a multiple of 16 is not one it can make; the rest keep one
|
|
// request from holding a card for the length of a film.
|
|
const (
|
|
maxImages = 4
|
|
minSide = 256
|
|
maxSide = 2048
|
|
maxSteps = 100
|
|
defaultSide = 1024
|
|
)
|
|
|
|
func parseImageRequest(body []byte) (imageRequest, error) {
|
|
var in imageRequest
|
|
if err := json.Unmarshal(body, &in); err != nil {
|
|
return in, err
|
|
}
|
|
if strings.TrimSpace(in.Prompt) == "" {
|
|
return in, errors.New("prompt is required")
|
|
}
|
|
switch {
|
|
case in.N == 0:
|
|
in.N = 1
|
|
case in.N < 0 || in.N > maxImages:
|
|
return in, fmt.Errorf("n is between 1 and %d", maxImages)
|
|
}
|
|
switch in.ResponseFormat {
|
|
case "", "b64_json":
|
|
case "url":
|
|
// A URL would have to be served by somebody and kept for some time,
|
|
// and a picture made from a prompt is the prompt's owner's: it is
|
|
// not kept here.
|
|
return in, errors.New(`response_format "url" is not offered: the picture comes back as b64_json and is kept nowhere`)
|
|
default:
|
|
return in, fmt.Errorf("response_format %q is not b64_json", in.ResponseFormat)
|
|
}
|
|
in.width, in.height = defaultSide, defaultSide
|
|
if s := strings.TrimSpace(in.Size); s != "" && s != "auto" {
|
|
w, h, ok := strings.Cut(strings.ToLower(s), "x")
|
|
wi, err1 := strconv.Atoi(w)
|
|
hi, err2 := strconv.Atoi(h)
|
|
if !ok || err1 != nil || err2 != nil {
|
|
return in, fmt.Errorf("size %q is not WIDTHxHEIGHT", s)
|
|
}
|
|
in.width, in.height = wi, hi
|
|
}
|
|
for _, side := range []int{in.width, in.height} {
|
|
if side < minSide || side > maxSide || side%16 != 0 {
|
|
return in, fmt.Errorf("size %dx%d: each side is a multiple of 16 from %d to %d", in.width, in.height, minSide, maxSide)
|
|
}
|
|
}
|
|
if in.Steps < 0 || in.Steps > maxSteps {
|
|
return in, fmt.Errorf("steps is between 1 and %d", maxSteps)
|
|
}
|
|
if in.Guidance != nil && (*in.Guidance <= 0 || *in.Guidance > 30) {
|
|
return in, errors.New("guidance is above 0 and at most 30")
|
|
}
|
|
return in, nil
|
|
}
|
|
|
|
// imageRouteRefusal says why a request cannot be answered at the path it was
|
|
// sent to, before it takes a place in a queue: a picture asked of a model
|
|
// that writes text, or text of one that makes pictures. Empty when it can,
|
|
// and for a model the catalog does not know, which a hand-configured
|
|
// endpoint may serve as it likes.
|
|
func (r *Router) imageRouteRefusal(path, model string, body []byte) string {
|
|
m, known := r.modelMap()[model]
|
|
if path == meshesPath {
|
|
if known && !m.MakesMeshes() {
|
|
if m.MakesImages() {
|
|
return fmt.Sprintf("%s makes pictures, not 3D models: ask for them at %s", m.Name, imagesPath)
|
|
}
|
|
return fmt.Sprintf("%s does not make 3D models", m.Name)
|
|
}
|
|
if _, err := parseMeshRequest(body); err != nil {
|
|
return err.Error()
|
|
}
|
|
return ""
|
|
}
|
|
if path == imagesPath {
|
|
if known && m.MakesMeshes() && !m.MakesImages() {
|
|
return fmt.Sprintf("%s makes 3D models, not pictures: ask for one at %s, as a job", m.Name, meshesPath)
|
|
}
|
|
if known && !m.MakesImages() {
|
|
return fmt.Sprintf("%s does not make pictures", m.Name)
|
|
}
|
|
if _, err := parseImageRequest(body); err != nil {
|
|
return err.Error()
|
|
}
|
|
return ""
|
|
}
|
|
if known && m.ServesComfyUI() {
|
|
if m.MakesMeshes() {
|
|
return fmt.Sprintf("%s makes 3D models: ask for one at %s, as a job", m.Name, meshesPath)
|
|
}
|
|
return fmt.Sprintf("%s makes pictures: ask for them at %s", m.Name, imagesPath)
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// generateImages answers a request at a ComfyUI replica. It returns what the
|
|
// proxy would: a failure of this replica that another may not have is
|
|
// retryable, and nothing is written until the pictures are all in hand.
|
|
func (r *Router) generateImages(ctx context.Context, w http.ResponseWriter, req *parsedRequest, ep endpoint.Info, body []byte) (proxy.Outcome, error) {
|
|
if req.Path == meshesPath {
|
|
return r.generateMeshes(ctx, w, req, ep, body)
|
|
}
|
|
if req.Path != imagesPath {
|
|
return proxy.Outcome{}, fmt.Errorf("%s is a ComfyUI server: it answers %s and %s only", ep.ID, imagesPath, meshesPath)
|
|
}
|
|
in, err := parseImageRequest(body)
|
|
if err != nil {
|
|
return proxy.Outcome{}, err
|
|
}
|
|
var upstream struct {
|
|
Model string `json:"model"`
|
|
}
|
|
_ = json.Unmarshal(body, &upstream)
|
|
tpl, err := comfy.Load(ep.Workflow)
|
|
if err != nil {
|
|
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
|
|
}
|
|
seed := comfy.RandomSeed()
|
|
if in.Seed != nil {
|
|
seed = *in.Seed
|
|
}
|
|
p := comfy.Params{
|
|
Checkpoint: upstream.Model, Prompt: in.Prompt, NegativePrompt: in.NegativePrompt,
|
|
Width: in.width, Height: in.height, N: in.N, Seed: seed, Steps: in.Steps,
|
|
}
|
|
if in.Guidance != nil {
|
|
p.Guidance = *in.Guidance
|
|
}
|
|
workflow, err := tpl.Fill(p)
|
|
if err != nil {
|
|
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
|
|
}
|
|
|
|
if t := r.cfg.Timeouts.Total; t > 0 {
|
|
var cancel context.CancelFunc
|
|
ctx, cancel = context.WithTimeout(ctx, t)
|
|
defer cancel()
|
|
}
|
|
start := time.Now()
|
|
images, err := r.comfy.Generate(ctx, ep.URL, ep.APIKey, workflow)
|
|
if err != nil {
|
|
var ce *comfy.Error
|
|
switch {
|
|
case errors.As(err, &ce):
|
|
return proxy.Outcome{Retryable: ce.Retryable}, err
|
|
case errors.Is(err, context.Canceled) && ctxCauseIsClient(ctx):
|
|
return proxy.Outcome{}, proxy.ErrClientGone
|
|
case errors.Is(err, context.DeadlineExceeded):
|
|
return proxy.Outcome{}, fmt.Errorf("%s made no picture within %s", ep.ID, r.cfg.Timeouts.Total)
|
|
}
|
|
return proxy.Outcome{}, err
|
|
}
|
|
|
|
data := make([]map[string]any, 0, len(images))
|
|
for _, im := range images {
|
|
data = append(data, map[string]any{"b64_json": base64.StdEncoding.EncodeToString(im.Data)})
|
|
}
|
|
out, err := json.Marshal(map[string]any{"created": time.Now().Unix(), "data": data})
|
|
if err != nil {
|
|
return proxy.Outcome{}, err
|
|
}
|
|
// The seed that made them, so a picture liked can be made again.
|
|
w.Header().Set("x-llmgw-seed", strconv.FormatInt(seed, 10))
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write(out)
|
|
took := time.Since(start)
|
|
return proxy.Outcome{Wrote: true, Status: http.StatusOK, Bytes: int64(len(out)), TTFT: took, Decode: took,
|
|
Images: len(images)}, nil
|
|
}
|
|
|
|
// ctxCauseIsClient reports whether a context ended because the client went
|
|
// away, rather than because a deadline set here passed.
|
|
func ctxCauseIsClient(ctx context.Context) bool {
|
|
return !errors.Is(context.Cause(ctx), context.DeadlineExceeded)
|
|
}
|
|
|
|
// registerImagesPage mounts the page that makes pictures, beside the chat
|
|
// and for the same reasons: a model tried without a client, with the
|
|
// browser holding a session and never a key, and the request sent down the
|
|
// one path every API request takes.
|
|
func (r *Router) registerImagesPage(mux *http.ServeMux, pages *web.AccountPages) {
|
|
pages.SetImages(func() bool {
|
|
for _, m := range r.modelMap() {
|
|
if m.MakesImages() {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
})
|
|
mux.HandleFunc("GET /images", pages.Guard(func(w http.ResponseWriter, req *http.Request, user store.User) {
|
|
contexts, _ := r.chatContexts(req.Context(), user)
|
|
current, err := r.pickContext(req.Context(), user, req.URL.Query().Get("context"))
|
|
if err != nil {
|
|
current = ""
|
|
}
|
|
if current == "" && !r.personalOff {
|
|
current = personalContext
|
|
}
|
|
web.Images(user, r.modelsOf(req, user, true), contexts, current, version.Version)(w, req)
|
|
}))
|
|
mux.HandleFunc("POST /images/generate", pages.Guard(r.imagesSend))
|
|
}
|
|
|
|
// imagesSend makes pictures on behalf of a signed-in person.
|
|
func (r *Router) imagesSend(w http.ResponseWriter, req *http.Request, user store.User) {
|
|
body, err := io.ReadAll(io.LimitReader(req.Body, r.cfg.MaxBodyBytes))
|
|
if err != nil {
|
|
http.Error(w, "could not read the request", http.StatusBadRequest)
|
|
return
|
|
}
|
|
var in struct {
|
|
Model string `json:"model"`
|
|
Prompt string `json:"prompt"`
|
|
Size string `json:"size"`
|
|
N int `json:"n"`
|
|
Seed *int64 `json:"seed"`
|
|
Steps int `json:"steps"`
|
|
NegativePrompt string `json:"negative_prompt"`
|
|
// Context is what the pictures are charged to, checked as the
|
|
// chat's is.
|
|
Context string `json:"context"`
|
|
}
|
|
if err := json.Unmarshal(body, &in); err != nil || in.Model == "" {
|
|
http.Error(w, "a model and a prompt are required", http.StatusBadRequest)
|
|
return
|
|
}
|
|
principal, err := r.principalFor(req, user, in.Context)
|
|
if err != nil {
|
|
http.Error(w, err.Error(), http.StatusForbidden)
|
|
return
|
|
}
|
|
// Rebuilt from the fields the page has, so it can set nothing else.
|
|
out := map[string]any{"model": in.Model, "prompt": in.Prompt, "size": in.Size, "n": in.N}
|
|
if in.Seed != nil {
|
|
out["seed"] = *in.Seed
|
|
}
|
|
if in.Steps > 0 {
|
|
out["steps"] = in.Steps
|
|
}
|
|
if s := strings.TrimSpace(in.NegativePrompt); s != "" {
|
|
out["negative_prompt"] = s
|
|
}
|
|
upstream, err := json.Marshal(out)
|
|
if err != nil {
|
|
http.Error(w, "could not build the request", http.StatusInternalServerError)
|
|
return
|
|
}
|
|
next := req.Clone(auth.WithPrincipal(req.Context(), principal))
|
|
next.Body = io.NopCloser(bytes.NewReader(upstream))
|
|
next.ContentLength = int64(len(upstream))
|
|
next.Header.Set("Content-Type", "application/json")
|
|
next.Header.Del(knowledgeHeader)
|
|
r.handleInference(imagesPath)(w, next)
|
|
}
|