An external model with api: comfyui is a ComfyUI server. A request for
pictures goes through the same key, allowlist, limit, admission and guard
checks as any other; at the replica it becomes a workflow — a built-in
(flux-schnell, flux-dev) or a JSON file in API format, with {{prompt}},
{{width}}, {{seed}} and the like filled in — queued, waited on, and its
pictures sent back as b64_json. A client that leaves takes its workflow
off the queue or interrupts it. ComfyUI is never scraped for metrics.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
201 lines
6.7 KiB
Go
201 lines
6.7 KiB
Go
package router
|
|
|
|
import (
|
|
"context"
|
|
"encoding/base64"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/comfy"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
|
|
)
|
|
|
|
// /v1/images/generations, OpenAI's shape, made by ComfyUI: the request goes
|
|
// through the same checks as any other — key, allowlist, limits, admission,
|
|
// guardrails on its prompt — and only at the replica is it turned into a
|
|
// workflow, queued and waited on, and its pictures sent back as base64.
|
|
|
|
const imagesPath = "/v1/images/generations"
|
|
|
|
// imageRequest is what a request for pictures asks. Beyond OpenAI's fields,
|
|
// those a diffusion model has and DALL·E does not: seed, steps,
|
|
// negative_prompt and guidance.
|
|
type imageRequest struct {
|
|
Prompt string `json:"prompt"`
|
|
N int `json:"n"`
|
|
Size string `json:"size"`
|
|
ResponseFormat string `json:"response_format"`
|
|
Seed *int64 `json:"seed"`
|
|
Steps int `json:"steps"`
|
|
NegativePrompt string `json:"negative_prompt"`
|
|
Guidance *float64 `json:"guidance"`
|
|
|
|
width, height int
|
|
}
|
|
|
|
// The bounds of a request. A FLUX latent is 1/16 of the picture, so a side
|
|
// that is not a multiple of 16 is not one it can make; the rest keep one
|
|
// request from holding a card for the length of a film.
|
|
const (
|
|
maxImages = 4
|
|
minSide = 256
|
|
maxSide = 2048
|
|
maxSteps = 100
|
|
defaultSide = 1024
|
|
)
|
|
|
|
func parseImageRequest(body []byte) (imageRequest, error) {
|
|
var in imageRequest
|
|
if err := json.Unmarshal(body, &in); err != nil {
|
|
return in, err
|
|
}
|
|
if strings.TrimSpace(in.Prompt) == "" {
|
|
return in, errors.New("prompt is required")
|
|
}
|
|
switch {
|
|
case in.N == 0:
|
|
in.N = 1
|
|
case in.N < 0 || in.N > maxImages:
|
|
return in, fmt.Errorf("n is between 1 and %d", maxImages)
|
|
}
|
|
switch in.ResponseFormat {
|
|
case "", "b64_json":
|
|
case "url":
|
|
// A URL would have to be served by somebody and kept for some time,
|
|
// and a picture made from a prompt is the prompt's owner's: it is
|
|
// not kept here.
|
|
return in, errors.New(`response_format "url" is not offered: the picture comes back as b64_json and is kept nowhere`)
|
|
default:
|
|
return in, fmt.Errorf("response_format %q is not b64_json", in.ResponseFormat)
|
|
}
|
|
in.width, in.height = defaultSide, defaultSide
|
|
if s := strings.TrimSpace(in.Size); s != "" && s != "auto" {
|
|
w, h, ok := strings.Cut(strings.ToLower(s), "x")
|
|
wi, err1 := strconv.Atoi(w)
|
|
hi, err2 := strconv.Atoi(h)
|
|
if !ok || err1 != nil || err2 != nil {
|
|
return in, fmt.Errorf("size %q is not WIDTHxHEIGHT", s)
|
|
}
|
|
in.width, in.height = wi, hi
|
|
}
|
|
for _, side := range []int{in.width, in.height} {
|
|
if side < minSide || side > maxSide || side%16 != 0 {
|
|
return in, fmt.Errorf("size %dx%d: each side is a multiple of 16 from %d to %d", in.width, in.height, minSide, maxSide)
|
|
}
|
|
}
|
|
if in.Steps < 0 || in.Steps > maxSteps {
|
|
return in, fmt.Errorf("steps is between 1 and %d", maxSteps)
|
|
}
|
|
if in.Guidance != nil && (*in.Guidance <= 0 || *in.Guidance > 30) {
|
|
return in, errors.New("guidance is above 0 and at most 30")
|
|
}
|
|
return in, nil
|
|
}
|
|
|
|
// imageRouteRefusal says why a request cannot be answered at the path it was
|
|
// sent to, before it takes a place in a queue: a picture asked of a model
|
|
// that writes text, or text of one that makes pictures. Empty when it can,
|
|
// and for a model the catalog does not know, which a hand-configured
|
|
// endpoint may serve as it likes.
|
|
func (r *Router) imageRouteRefusal(path, model string, body []byte) string {
|
|
m, known := r.modelMap()[model]
|
|
if path == imagesPath {
|
|
if known && !m.MakesImages() {
|
|
return fmt.Sprintf("%s does not make pictures", m.Name)
|
|
}
|
|
if _, err := parseImageRequest(body); err != nil {
|
|
return err.Error()
|
|
}
|
|
return ""
|
|
}
|
|
if known && m.External != nil && m.External.API == catalog.APIComfyUI {
|
|
return fmt.Sprintf("%s makes pictures: ask for them at %s", m.Name, imagesPath)
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// generateImages answers a request at a ComfyUI replica. It returns what the
|
|
// proxy would: a failure of this replica that another may not have is
|
|
// retryable, and nothing is written until the pictures are all in hand.
|
|
func (r *Router) generateImages(ctx context.Context, w http.ResponseWriter, req *parsedRequest, ep endpoint.Info, body []byte) (proxy.Outcome, error) {
|
|
if req.Path != imagesPath {
|
|
return proxy.Outcome{}, fmt.Errorf("%s is a ComfyUI server: it answers %s only", ep.ID, imagesPath)
|
|
}
|
|
in, err := parseImageRequest(body)
|
|
if err != nil {
|
|
return proxy.Outcome{}, err
|
|
}
|
|
var upstream struct {
|
|
Model string `json:"model"`
|
|
}
|
|
_ = json.Unmarshal(body, &upstream)
|
|
tpl, err := comfy.Load(ep.Workflow)
|
|
if err != nil {
|
|
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
|
|
}
|
|
seed := comfy.RandomSeed()
|
|
if in.Seed != nil {
|
|
seed = *in.Seed
|
|
}
|
|
p := comfy.Params{
|
|
Checkpoint: upstream.Model, Prompt: in.Prompt, NegativePrompt: in.NegativePrompt,
|
|
Width: in.width, Height: in.height, N: in.N, Seed: seed, Steps: in.Steps,
|
|
}
|
|
if in.Guidance != nil {
|
|
p.Guidance = *in.Guidance
|
|
}
|
|
workflow, err := tpl.Fill(p)
|
|
if err != nil {
|
|
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
|
|
}
|
|
|
|
if t := r.cfg.Timeouts.Total; t > 0 {
|
|
var cancel context.CancelFunc
|
|
ctx, cancel = context.WithTimeout(ctx, t)
|
|
defer cancel()
|
|
}
|
|
start := time.Now()
|
|
images, err := r.comfy.Generate(ctx, ep.URL, ep.APIKey, workflow)
|
|
if err != nil {
|
|
var ce *comfy.Error
|
|
switch {
|
|
case errors.As(err, &ce):
|
|
return proxy.Outcome{Retryable: ce.Retryable}, err
|
|
case errors.Is(err, context.Canceled) && ctxCauseIsClient(ctx):
|
|
return proxy.Outcome{}, proxy.ErrClientGone
|
|
case errors.Is(err, context.DeadlineExceeded):
|
|
return proxy.Outcome{}, fmt.Errorf("%s made no picture within %s", ep.ID, r.cfg.Timeouts.Total)
|
|
}
|
|
return proxy.Outcome{}, err
|
|
}
|
|
|
|
data := make([]map[string]any, 0, len(images))
|
|
for _, im := range images {
|
|
data = append(data, map[string]any{"b64_json": base64.StdEncoding.EncodeToString(im.Data)})
|
|
}
|
|
out, err := json.Marshal(map[string]any{"created": time.Now().Unix(), "data": data})
|
|
if err != nil {
|
|
return proxy.Outcome{}, err
|
|
}
|
|
// The seed that made them, so a picture liked can be made again.
|
|
w.Header().Set("x-llmgw-seed", strconv.FormatInt(seed, 10))
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(http.StatusOK)
|
|
_, _ = w.Write(out)
|
|
took := time.Since(start)
|
|
return proxy.Outcome{Wrote: true, Status: http.StatusOK, Bytes: int64(len(out)), TTFT: took, Decode: took}, nil
|
|
}
|
|
|
|
// ctxCauseIsClient reports whether a context ended because the client went
|
|
// away, rather than because a deadline set here passed.
|
|
func ctxCauseIsClient(ctx context.Context) bool {
|
|
return !errors.Is(context.Cause(ctx), context.DeadlineExceeded)
|
|
}
|