Any ComfyUI workflow runs as a job, from the inputs its catalog entry declares
CI / test (push) Successful in 9m7s

A model's catalog entry now says what its workflow takes and makes:
comfyui.inputs, each a placeholder of its name with a type (text, int,
float, bool, choice, image, mask, audio, video, file), bounds, choices
and a default; and comfyui.output, which gains video, audio and files
beside pictures and meshes. POST /v1/workflows/runs with {model, inputs}
checks a request against them before a job is written — a required input
left out, an unknown one, a number out of bounds, a song sent as a
picture, another owner's file id are each refused with what was wrong —
and the job worker replays it to a replica, where the files go into
ComfyUI's input directory, the workflow is filled and run, and the first
file of the model's output kind is the job's result. /v1/models lists
each ComfyUI model's workflow, so a client can build its form.

A PNG result carries the licence in an iTXt chunk, as a GLB does in
asset.extras. A file whose server named no type is typed by its name
before its bytes, so a WAV is audio/wav, not audio/wave.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-11 14:28:15 +02:00
1 parent b43e66c95e
commit b6bfc33ef7
20 files changed
+1660 -44

No files matched your search

+16
View File
@@ -143,6 +143,12 @@ func (r *Router) Handler() http.Handler {
mux.HandleFunc("GET "+meshesPath, r.handleListJobs)
mux.HandleFunc("GET "+meshesPath+"/{id}", r.handleGetJob)
mux.HandleFunc("POST "+meshesPath+"/{id}/cancel", r.handleCancelJob)
// Workflow runs: whatever a ComfyUI model makes, from the inputs its
// catalog entry declares, as a job like a 3D model.
mux.HandleFunc("POST "+workflowsPath, r.createJobAt(workflowsPath))
mux.HandleFunc("GET "+workflowsPath, r.handleListJobs)
mux.HandleFunc("GET "+workflowsPath+"/{id}", r.handleGetJob)
mux.HandleFunc("POST "+workflowsPath+"/{id}/cancel", r.handleCancelJob)
// Knowledge bases: documents a person's llmgw-kb read and embedded.
r.registerKB(mux)
// Liveness: this process is up. It says nothing about whether it can
@@ -1198,6 +1204,16 @@ func (r *Router) handleModels(w http.ResponseWriter, req *http.Request) {
entry["capabilities"] = map[string]bool{
"tools": m.Capabilities.Tools, "vision": m.Capabilities.Vision,
"embeddings": m.Capabilities.Embeddings, "images": m.MakesImages(), "meshes": m.MakesMeshes(),
"workflows": m.RunsWorkflows(),
}
// What a run of it takes and makes, so a client can build the form
// without the catalog in hand.
if m.RunsWorkflows() {
var inputs any = m.ComfyInputs()
if len(m.ComfyInputs()) == 0 {
inputs = []any{}
}
entry["workflow"] = map[string]any{"output": m.ComfyOutput(), "inputs": inputs}
}
data = append(data, entry)
}
+23 -4
View File
@@ -112,6 +112,18 @@ func parseImageRequest(body []byte) (imageRequest, error) {
// endpoint may serve as it likes.
func (r *Router) imageRouteRefusal(path, model string, body []byte) string {
m, known := r.modelMap()[model]
if path == workflowsPath {
if !known {
return ""
}
if !m.RunsWorkflows() {
return fmt.Sprintf("%s runs no ComfyUI workflow", m.Name)
}
if _, err := parseRunRequest(m, body); err != nil {
return err.Error()
}
return ""
}
if path == meshesPath {
if known && !m.MakesMeshes() {
if m.MakesImages() {
@@ -137,6 +149,9 @@ func (r *Router) imageRouteRefusal(path, model string, body []byte) string {
return ""
}
if known && m.ServesComfyUI() {
if !m.MakesMeshes() && !m.MakesImages() {
return fmt.Sprintf("%s makes %s: ask for it at %s, as a job", m.Name, m.ComfyOutput(), workflowsPath)
}
if m.MakesMeshes() {
return fmt.Sprintf("%s makes 3D models: ask for one at %s, as a job", m.Name, meshesPath)
}
@@ -149,11 +164,15 @@ func (r *Router) imageRouteRefusal(path, model string, body []byte) string {
// proxy would: a failure of this replica that another may not have is
// retryable, and nothing is written until the pictures are all in hand.
func (r *Router) generateImages(ctx context.Context, w http.ResponseWriter, req *parsedRequest, ep endpoint.Info, body []byte) (proxy.Outcome, error) {
if req.Path == meshesPath {
switch req.Path {
case meshesPath:
return r.generateMeshes(ctx, w, req, ep, body)
}
if req.Path != imagesPath {
return proxy.Outcome{}, fmt.Errorf("%s is a ComfyUI server: it answers %s and %s only", ep.ID, imagesPath, meshesPath)
case workflowsPath:
return r.generateRun(ctx, w, req, ep, body)
case imagesPath:
default:
return proxy.Outcome{}, fmt.Errorf("%s is a ComfyUI server: it answers %s, %s and %s only", ep.ID,
imagesPath, meshesPath, workflowsPath)
}
in, err := parseImageRequest(body)
if err != nil {
+25 -6
View File
@@ -66,11 +66,21 @@ func (r *Router) jobJSON(ctx context.Context, j store.Job) map[string]any {
return out
}
// handleCreateJob queues a request for a 3D model. Refused now: no key, a
// model that is not there or not allowed, a request the model cannot serve.
// Refused when it is served, and said as the job's error: limits, the
// guardrails, a model that will not come up.
// handleCreateJob queues a request for a 3D model.
func (r *Router) handleCreateJob(w http.ResponseWriter, req *http.Request) {
r.createJobAt(meshesPath)(w, req)
}
// createJobAt queues a request at path: a 3D model, a workflow run. Refused
// now: no key, a model that is not there or not allowed, a request the model
// cannot serve, a file it names that is not the caller's. Refused when it is
// served, and said as the job's error: limits, the guardrails, a model that
// will not come up.
func (r *Router) createJobAt(path string) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) { r.createJob(w, req, path) }
}
func (r *Router) createJob(w http.ResponseWriter, req *http.Request, path string) {
p, ok := r.batchPrincipal(w, req)
if !ok {
return
@@ -100,12 +110,21 @@ func (r *Router) handleCreateJob(w http.ResponseWriter, req *http.Request) {
auth.WriteForbiddenModel(w, in.Model)
return
}
if why := r.imageRouteRefusal(meshesPath, in.Model, body); why != "" {
if why := r.imageRouteRefusal(path, in.Model, body); why != "" {
writeError(w, http.StatusBadRequest, "invalid_request_error", why)
return
}
if path == workflowsPath {
if m, known := r.modelMap()[in.Model]; known {
run, _ := parseRunRequest(m, body)
if err := r.checkRunFiles(req.Context(), ownerOf(p), run); err != nil {
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
return
}
}
}
now := time.Now()
j := store.Job{ID: newID("job_"), Owner: ownerOf(p), KeyID: p.KeyID, Endpoint: meshesPath, Model: in.Model,
j := store.Job{ID: newID("job_"), Owner: ownerOf(p), KeyID: p.KeyID, Endpoint: path, Model: in.Model,
Input: body, Status: store.JobQueued, CreatedAt: now, ExpiresAt: now.Add(jobWindow)}
if err := r.store.PutJob(req.Context(), j); err != nil {
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
+17 -1
View File
@@ -25,7 +25,7 @@ const (
jobPoll = 5 * time.Second
// jobCancelCheck is how often a running job looks for a cancel.
jobCancelCheck = 2 * time.Second
// jobMaxResult bounds what one job may answer: a mesh, not a film.
// jobMaxResult bounds what one job may answer: a mesh, a short video.
jobMaxResult = 256 << 20
)
@@ -264,6 +264,22 @@ func extensionFor(contentType string) string {
return ".jpg"
case "image/webp":
return ".webp"
case "image/gif":
return ".gif"
case "video/mp4":
return ".mp4"
case "video/webm":
return ".webm"
case "video/quicktime":
return ".mov"
case "audio/wav", "audio/wave", "audio/x-wav":
return ".wav"
case "audio/mpeg":
return ".mp3"
case "audio/flac":
return ".flac"
case "audio/ogg":
return ".ogg"
}
return ".bin"
}
+501
View File
@@ -0,0 +1,501 @@
package router
import (
"bytes"
"context"
"crypto/rand"
"encoding/base64"
"encoding/binary"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"hash/crc32"
"math"
"net/http"
"path"
"slices"
"sort"
"strconv"
"strings"
"time"
"unicode/utf8"
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
"gitea.mynodes.xyz/ilker/llmgw/internal/comfy"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/proxy"
)
// Workflow runs: whatever a ComfyUI model makes, from the inputs its catalog
// entry declares. Pictures and meshes have fixed shapes of their own
// (images.go, meshes.go); this is the one shape for everything, so a model
// is added to the catalog as data — its workflow, its inputs, its output
// kind — rather than as code. A run is a job like a 3D model (jobs.go): it
// is queued, asked after, and its result is a file.
// workflowsPath is where workflow runs are asked for. The synchronous path
// behind it is served to the job worker only; clients get a job.
const workflowsPath = "/v1/workflows/runs"
// runRequest is a request to run a model's workflow.
type runRequest struct {
Model string `json:"model"`
Inputs map[string]json.RawMessage `json:"inputs"`
Seed *int64 `json:"seed"`
// values are the inputs that are not files, as the workflow gets them.
values map[string]any
// files are the file inputs, by input name.
files map[string]runFile
}
// runFile is a file input: its bytes, or the id of a file in /v1/files to be
// read when the run is served.
type runFile struct {
data []byte
fileID string
ext string
}
// maxTextInput bounds a text input that names no max of its own.
const maxTextInput = 32 << 10
// parseRunRequest checks a request against what the model's workflow takes.
// It refuses an input the workflow does not take, a required one left out, a
// value of the wrong type or out of its bounds, and a file that is not what
// the input says.
func parseRunRequest(m catalog.Model, body []byte) (runRequest, error) {
var in runRequest
if err := json.Unmarshal(body, &in); err != nil {
return in, err
}
in.values, in.files = map[string]any{}, map[string]runFile{}
declared := m.ComfyInputs()
for name := range in.Inputs {
if !slices.ContainsFunc(declared, func(d catalog.Input) bool { return d.Name == name }) {
names := make([]string, 0, len(declared))
for _, d := range declared {
names = append(names, d.Name)
}
if len(names) == 0 {
return in, fmt.Errorf("%s takes no inputs, and was given %q", m.Name, name)
}
return in, fmt.Errorf("%s takes no input %q (it takes %s)", m.Name, name, strings.Join(names, ", "))
}
}
for _, d := range declared {
raw, given := in.Inputs[d.Name]
if !given || string(raw) == "null" {
if d.Required {
return in, fmt.Errorf("input %q is required: %s", d.Name, describeInput(d))
}
if d.Default != nil {
in.values[d.Name] = d.Default
}
continue
}
if d.IsFile() {
f, err := parseFileInput(d, raw)
if err != nil {
return in, err
}
in.files[d.Name] = f
continue
}
v, err := parseValueInput(d, raw)
if err != nil {
return in, err
}
in.values[d.Name] = v
}
return in, nil
}
func describeInput(d catalog.Input) string {
s := d.Type
if d.Type == catalog.InputChoice {
s += " of " + strings.Join(d.Choices, ", ")
}
if d.Doc != "" {
s += " — " + d.Doc
}
return s
}
func parseValueInput(d catalog.Input, raw json.RawMessage) (any, error) {
bounds := func(f float64) error {
if d.Min != nil && f < *d.Min || d.Max != nil && f > *d.Max {
return fmt.Errorf("input %q is %v, outside %s", d.Name, f, boundsText(d))
}
return nil
}
switch d.Type {
case catalog.InputText:
var s string
if json.Unmarshal(raw, &s) != nil {
return nil, fmt.Errorf("input %q is text", d.Name)
}
limit := maxTextInput
if d.Max != nil {
limit = int(*d.Max)
}
if n := utf8.RuneCountInString(s); n > limit {
return nil, fmt.Errorf("input %q is %d characters, past %d", d.Name, n, limit)
}
return s, nil
case catalog.InputInt:
var f float64
if json.Unmarshal(raw, &f) != nil || f != math.Trunc(f) || math.Abs(f) > 1<<53 {
return nil, fmt.Errorf("input %q is a whole number", d.Name)
}
return int64(f), bounds(f)
case catalog.InputFloat:
var f float64
if json.Unmarshal(raw, &f) != nil || math.IsNaN(f) || math.IsInf(f, 0) {
return nil, fmt.Errorf("input %q is a number", d.Name)
}
return f, bounds(f)
case catalog.InputBool:
var b bool
if json.Unmarshal(raw, &b) != nil {
return nil, fmt.Errorf("input %q is true or false", d.Name)
}
return b, nil
case catalog.InputChoice:
var s string
if json.Unmarshal(raw, &s) != nil || !slices.Contains(d.Choices, s) {
return nil, fmt.Errorf("input %q is one of %s", d.Name, strings.Join(d.Choices, ", "))
}
return s, nil
}
return nil, fmt.Errorf("input %q has type %q, which takes no value", d.Name, d.Type)
}
func boundsText(d catalog.Input) string {
switch {
case d.Min != nil && d.Max != nil:
return fmt.Sprintf("%v to %v", *d.Min, *d.Max)
case d.Min != nil:
return fmt.Sprintf("at least %v", *d.Min)
case d.Max != nil:
return fmt.Sprintf("at most %v", *d.Max)
}
return "its bounds"
}
// parseFileInput reads a file input: base64, a data URL, or a file id.
func parseFileInput(d catalog.Input, raw json.RawMessage) (runFile, error) {
var s string
if json.Unmarshal(raw, &s) != nil || strings.TrimSpace(s) == "" {
return runFile{}, fmt.Errorf("input %q is a file: base64, a data URL, or the id of a file uploaded to /v1/files", d.Name)
}
s = strings.TrimSpace(s)
if strings.HasPrefix(s, "file-") && len(s) < 80 && !strings.ContainsAny(s, " /") {
return runFile{fileID: s}, nil
}
if strings.HasPrefix(s, "data:") {
_, after, ok := strings.Cut(s, ";base64,")
if !ok {
return runFile{}, fmt.Errorf("input %q is a data URL without base64 in it", d.Name)
}
s = after
}
if int64(base64.StdEncoding.DecodedLen(len(s))) > d.MaxBytes()+3 {
return runFile{}, fmt.Errorf("input %q is past %d MB: upload it to /v1/files and give its id", d.Name, d.MaxBytes()>>20)
}
b, err := base64.StdEncoding.DecodeString(s)
if err != nil {
return runFile{}, fmt.Errorf("input %q is not base64: %v", d.Name, err)
}
return checkFileInput(d, b)
}
// checkFileInput is a file input's bytes, if they are what the input says.
func checkFileInput(d catalog.Input, b []byte) (runFile, error) {
if int64(len(b)) > d.MaxBytes() {
return runFile{}, fmt.Errorf("input %q is past %d MB", d.Name, d.MaxBytes()>>20)
}
ext, kind := sniffFile(b)
want := map[string]string{catalog.InputImage: "image", catalog.InputMask: "image",
catalog.InputAudio: "audio", catalog.InputVideo: "video"}[d.Type]
if want != "" && kind != want {
return runFile{}, fmt.Errorf("input %q is %s, and this is not one it can read", d.Name, map[string]string{
"image": "a PNG, JPEG, WebP or GIF picture", "audio": "WAV, MP3, FLAC or Ogg audio",
"video": "an MP4, WebM or QuickTime video"}[want])
}
if ext == "" {
ext = ".bin"
}
return runFile{data: b, ext: ext}, nil
}
// sniffFile is a file's extension and kind (image, audio, video) by its
// first bytes, or empty when it is none of the kinds a workflow loads.
func sniffFile(b []byte) (ext, kind string) {
has := func(off int, sig string) bool { return len(b) >= off+len(sig) && string(b[off:off+len(sig)]) == sig }
switch {
case has(0, "\x89PNG\r\n\x1a\n"):
return ".png", "image"
case has(0, "\xff\xd8\xff"):
return ".jpg", "image"
case has(0, "RIFF") && has(8, "WEBP"):
return ".webp", "image"
case has(0, "GIF87a"), has(0, "GIF89a"):
return ".gif", "image"
case has(0, "RIFF") && has(8, "WAVE"):
return ".wav", "audio"
case has(0, "fLaC"):
return ".flac", "audio"
case has(0, "OggS"):
return ".ogg", "audio"
case has(0, "ID3"), len(b) > 2 && b[0] == 0xff && b[1]&0xe0 == 0xe0:
return ".mp3", "audio"
case has(4, "ftypqt"):
return ".mov", "video"
case has(4, "ftyp"):
return ".mp4", "video"
case has(0, "\x1a\x45\xdf\xa3"):
return ".webm", "video"
}
return "", ""
}
// runFileIDs are the file ids a run's inputs name.
func (in runRequest) runFileIDs() []string {
var ids []string
for _, f := range in.files {
if f.fileID != "" {
ids = append(ids, f.fileID)
}
}
sort.Strings(ids)
return ids
}
// checkRunFiles refuses a run naming a file its owner does not have.
func (r *Router) checkRunFiles(ctx context.Context, owner string, in runRequest) error {
for _, id := range in.runFileIDs() {
f, err := r.store.FileByID(ctx, id)
if err != nil || f.Owner != owner {
return fmt.Errorf("no file %q", id)
}
}
return nil
}
// generateRun answers a workflow run at a ComfyUI replica: the file inputs
// put in its input directory, the workflow filled in and run, and the file
// it made written back with its type — what the job keeps.
func (r *Router) generateRun(ctx context.Context, w http.ResponseWriter, req *parsedRequest, ep endpoint.Info, body []byte) (proxy.Outcome, error) {
m, known := r.modelMap()[req.Model]
if !known {
return proxy.Outcome{}, fmt.Errorf("%s: no catalog entry for %q says what its workflow takes", ep.ID, req.Model)
}
in, err := parseRunRequest(m, body)
if err != nil {
return proxy.Outcome{}, err
}
tpl, err := comfy.Load(ep.Workflow)
if err != nil {
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
}
if err := checkPlaceholders(m, tpl); err != nil {
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
}
if t := r.cfg.Timeouts.Total; t > 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, t)
defer cancel()
}
start := time.Now()
fail := func(err error) (proxy.Outcome, error) {
var ce *comfy.Error
switch {
case errors.As(err, &ce):
return proxy.Outcome{Retryable: ce.Retryable}, err
case errors.Is(err, context.Canceled) && ctxCauseIsClient(ctx):
return proxy.Outcome{}, proxy.ErrClientGone
case errors.Is(err, context.DeadlineExceeded):
return proxy.Outcome{}, fmt.Errorf("%s finished nothing within %s", ep.ID, r.cfg.Timeouts.Total)
}
return proxy.Outcome{}, err
}
var upstream struct {
Model string `json:"model"`
}
_ = json.Unmarshal(body, &upstream)
p := comfy.Params{Checkpoint: upstream.Model, Seed: comfy.RandomSeed(), Extra: in.values}
if in.Seed != nil {
p.Seed = *in.Seed
}
if s, ok := in.values["seed"].(int64); ok {
p.Seed = s
}
// Each file under a name of its own: two runs' files must not overwrite
// each other in the replica's input directory.
names := make([]string, 0, len(in.files))
for name := range in.files {
names = append(names, name)
}
sort.Strings(names)
for _, name := range names {
f := in.files[name]
if f.fileID != "" {
pr, _ := auth.FromContext(ctx)
d := m.ComfyInputs()[slices.IndexFunc(m.ComfyInputs(), func(x catalog.Input) bool { return x.Name == name })]
meta, err := r.store.FileByID(ctx, f.fileID)
if err != nil || meta.Owner != ownerOf(pr) {
return proxy.Outcome{}, fmt.Errorf("input %q: no file %q", name, f.fileID)
}
b, err := r.store.FileContent(ctx, f.fileID)
if err != nil {
return proxy.Outcome{}, fmt.Errorf("input %q: reading file %q: %w", name, f.fileID, err)
}
if f, err = checkFileInput(d, b); err != nil {
return proxy.Outcome{}, err
}
in.files[name] = f
}
rnd := make([]byte, 8)
_, _ = rand.Read(rnd)
f = in.files[name]
up, err := r.comfy.Upload(ctx, ep.URL, ep.APIKey, "llmgw-"+hex.EncodeToString(rnd)+f.ext, f.data)
if err != nil {
return fail(err)
}
p.Extra[name] = up
}
workflow, err := tpl.Fill(p)
if err != nil {
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
}
files, err := r.comfy.GenerateWatch(ctx, ep.URL, ep.APIKey, workflow, r.jobProgress(ctx, req))
if err != nil {
return fail(err)
}
out, err := pickOutput(m.ComfyOutput(), files)
if err != nil {
return proxy.Outcome{}, fmt.Errorf("%s: %w", ep.ID, err)
}
data, ct := out.Data, out.ContentType
if ct == "model/obj" {
if data, err = objToGLB(data); err != nil {
return proxy.Outcome{}, fmt.Errorf("%s: the mesh it saved: %w", ep.ID, err)
}
ct = "model/gltf-binary"
}
if m.License != nil {
if data, err = withLicense(data, ct, *m.License); err != nil {
// Refused rather than sent without it: the file travels without
// the API, and the notice is what the terms ask to travel too.
return proxy.Outcome{}, fmt.Errorf("%s: the licence could not be written into the file: %w", ep.ID, err)
}
w.Header().Set("x-llmgw-license", m.License.Name)
}
if _, ok := tpl.Placeholders()["seed"]; ok {
w.Header().Set("x-llmgw-seed", strconv.FormatInt(p.Seed, 10))
}
w.Header().Set("Content-Type", ct)
w.WriteHeader(http.StatusOK)
_, _ = w.Write(data)
took := time.Since(start)
o := proxy.Outcome{Wrote: true, Status: http.StatusOK, Bytes: int64(len(data)), TTFT: took, Decode: took}
switch m.ComfyOutput() {
case catalog.OutputPictures:
o.Images = 1
case catalog.OutputMeshes:
o.Meshes = 1
}
return o, nil
}
// checkPlaceholders refuses a workflow that wants what no request can give:
// a placeholder that is neither an input, nor one llmgw fills itself, nor
// has a default of its own. The catalog's to fix, said as such.
func checkPlaceholders(m catalog.Model, tpl *comfy.Template) error {
var missing []string
for name, hasDefault := range tpl.Placeholders() {
if hasDefault || slices.Contains(comfy.Builtin, name) ||
slices.ContainsFunc(m.ComfyInputs(), func(d catalog.Input) bool { return d.Name == name }) {
continue
}
missing = append(missing, name)
}
if len(missing) > 0 {
sort.Strings(missing)
return fmt.Errorf("the workflow has placeholders %s that the catalog declares no input for and that have no default",
strings.Join(missing, ", "))
}
return nil
}
// pickOutput is the file a run answers with: the first the workflow saved of
// the model's output kind, or the first of any kind for files.
func pickOutput(kind string, files []comfy.Image) (comfy.Image, error) {
is := func(ct string) bool {
switch kind {
case catalog.OutputPictures:
return strings.HasPrefix(ct, "image/") && ct != "image/gif"
case catalog.OutputMeshes:
return isMeshType(ct)
case catalog.OutputVideo:
return strings.HasPrefix(ct, "video/") || ct == "image/gif" || ct == "image/webp"
case catalog.OutputAudio:
return strings.HasPrefix(ct, "audio/")
}
return true
}
for _, f := range files {
if is(f.ContentType) {
return f, nil
}
}
saved := make([]string, 0, len(files))
for _, f := range files {
saved = append(saved, path.Base(f.Name)+" ("+f.ContentType+")")
}
return comfy.Image{}, fmt.Errorf("the workflow saved no %s: it saved %s", kind, strings.Join(saved, ", "))
}
// withLicense is a file with its licence written in, where its format has a
// place for one: a GLB's asset.extras, a PNG's text. Any other is returned as
// it is, and the licence goes beside it in the job's answer only.
func withLicense(data []byte, ct string, lic catalog.License) ([]byte, error) {
switch ct {
case "model/gltf-binary":
return glbWithLicense(data, lic)
case "image/png":
return pngWithLicense(data, lic)
}
return data, nil
}
// pngWithLicense is a PNG with an iTXt chunk "License" after its header,
// holding the licence as JSON. The workflow's own text chunks were stripped
// when the file was read; this one is the operator's to say.
func pngWithLicense(png []byte, lic catalog.License) ([]byte, error) {
const sig = "\x89PNG\r\n\x1a\n"
if len(png) < len(sig)+25 || string(png[:len(sig)]) != sig || string(png[12:16]) != "IHDR" {
return nil, errors.New("not a PNG")
}
text, err := json.Marshal(lic)
if err != nil {
return nil, err
}
// keyword, NUL, compression flag 0, method 0, empty language, NUL, empty
// translated keyword, NUL, then the UTF-8 text.
var chunk bytes.Buffer
chunk.WriteString("iTXt")
chunk.WriteString("License\x00\x00\x00\x00\x00")
chunk.Write(text)
body := chunk.Bytes()
out := make([]byte, 0, len(png)+len(body)+8)
ihdrEnd := len(sig) + 8 + 13 + 4
out = append(out, png[:ihdrEnd]...)
out = binary.BigEndian.AppendUint32(out, uint32(len(body)-4)) //nolint:gosec // a licence's JSON
out = append(out, body...)
out = binary.BigEndian.AppendUint32(out, crc32.ChecksumIEEE(body))
return append(out, png[ihdrEnd:]...), nil
}
+296
View File
@@ -0,0 +1,296 @@
package router
import (
"bytes"
"crypto/sha256"
"encoding/base64"
"encoding/binary"
"encoding/json"
"fmt"
"image"
"image/png"
"io"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"sync"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/comfy"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// testWAV is the smallest WAV header and a few samples.
func testWAV() []byte {
var b bytes.Buffer
data := []byte{0, 0, 1, 0, 2, 0, 3, 0}
b.WriteString("RIFF")
_ = binary.Write(&b, binary.LittleEndian, uint32(36+len(data)))
b.WriteString("WAVEfmt ")
_ = binary.Write(&b, binary.LittleEndian, []any{uint32(16), uint16(1), uint16(1), uint32(8000), uint32(16000), uint16(2), uint16(16)})
b.WriteString("data")
_ = binary.Write(&b, binary.LittleEndian, uint32(len(data)))
b.Write(data)
return b.Bytes()
}
// runComfy is a ComfyUI whose workflow saves what it is told to: a song, a
// picture, under whatever key its save node uses.
type runComfy struct {
srv *httptest.Server
mu sync.Mutex
uploaded []string
workflow map[string]any
out []byte
outName string
}
func newRunComfy(t *testing.T, out []byte, outName string) *runComfy {
c := &runComfy{out: out, outName: outName}
c.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/upload/image":
f, h, err := r.FormFile("image")
if err != nil {
http.Error(w, err.Error(), 400)
return
}
b, _ := io.ReadAll(f)
ext, _ := sniffFile(b)
c.mu.Lock()
c.uploaded = append(c.uploaded, h.Filename+":"+ext)
c.mu.Unlock()
fmt.Fprintf(w, `{"name":%q,"subfolder":"","type":"input"}`, h.Filename)
case "/prompt":
var b struct {
Prompt map[string]any `json:"prompt"`
}
_ = json.NewDecoder(r.Body).Decode(&b)
c.mu.Lock()
c.workflow = b.Prompt
c.mu.Unlock()
fmt.Fprint(w, `{"prompt_id":"r1"}`)
case "/history/r1":
fmt.Fprintf(w, `{"r1":{"status":{"status_str":"success","completed":true},"outputs":{
"9":{"audio":[{"filename":%q,"subfolder":"","type":"output"}]}}}}`, c.outName)
case "/view":
w.Header().Set("Content-Type", "application/octet-stream")
_, _ = w.Write(c.out)
default:
http.NotFound(w, r)
}
}))
t.Cleanup(c.srv.Close)
return c
}
const songWorkflow = `{
"1": {"class_type": "LoadImage", "inputs": {"image": "{{cover}}"}},
"2": {"class_type": "TextEncodeAceStepAudio", "inputs": {"tags": "{{tags}}", "lyrics": "{{lyrics|}}", "strength": "{{strength}}"}},
"3": {"class_type": "EmptyAceStepLatentAudio", "inputs": {"seconds": "{{seconds}}", "style": "{{style}}"}},
"4": {"class_type": "KSampler", "inputs": {"seed": "{{seed}}", "model": "{{checkpoint}}"}},
"9": {"class_type": "SaveAudio", "inputs": {"filename_prefix": "audio/song"}}
}`
func f64(v float64) *float64 { return &v }
func runRouter(t *testing.T, c *runComfy, workflow string, output string) (*Router, string, string) {
t.Helper()
cfg := baseConfig()
cfg.Models = []catalog.Model{
{Name: "ace", Source: "external", License: &catalog.License{ID: "Apache-2.0", Name: "Apache 2.0"},
External: &catalog.External{URL: c.srv.URL, Model: "ace_step.safetensors", API: catalog.APIComfyUI,
Output: output, Workflow: "w.json"},
ComfyUI: &catalog.ComfyUI{Inputs: []catalog.Input{
{Name: "tags", Type: catalog.InputText, Required: true, Max: f64(200), Doc: "genres and moods"},
{Name: "lyrics", Type: catalog.InputText},
{Name: "strength", Type: catalog.InputFloat, Min: f64(0), Max: f64(1), Default: 0.7},
{Name: "seconds", Type: catalog.InputInt, Min: f64(5), Max: f64(240), Default: 30},
{Name: "style", Type: catalog.InputChoice, Choices: []string{"pop", "metal"}, Default: "pop"},
{Name: "cover", Type: catalog.InputImage, Required: true},
}}},
{Name: "flux", Source: "external", External: &catalog.External{URL: c.srv.URL,
Model: "flux1-schnell-fp8.safetensors", API: catalog.APIComfyUI}},
}
cfg.Auth.Enabled = true
cfg.Auth.Store = filepath.Join(t.TempDir(), "store.db")
cfg.Auth.FlushInterval = 20 * time.Millisecond
cfg.Auth.BatchSize = 1
rt := newTestRouter(t, cfg)
rt.comfy = &comfy.Client{Poll: time.Millisecond}
rt.SetEndpoints(t.Context(), []endpoint.Info{
{ID: "external:ace", URL: c.srv.URL, Models: []string{"ace"}, Engine: endpoint.EngineComfyUI,
Pool: "external", Weight: 1, UpstreamModel: "ace_step.safetensors", Workflow: workflow},
{ID: "external:flux", URL: c.srv.URL, Models: []string{"flux"}, Engine: endpoint.EngineComfyUI,
Pool: "external", Weight: 1, UpstreamModel: "flux1-schnell-fp8.safetensors"},
})
org, _ := rt.store.CreateOrg(t.Context(), "studio")
team, _ := rt.store.CreateTeam(t.Context(), org, "sound")
project, err := rt.store.CreateProject(t.Context(), team, "score")
if err != nil {
t.Fatal(err)
}
secret := store.KeyPrefix + strings.Repeat("w", 43)
sum := sha256.Sum256([]byte(secret))
if err := rt.store.CreateKey(t.Context(), store.Key{ID: "key-w", Hash: sum[:], Prefix: "wwww", Name: "sound",
ProjectID: project, CreatedAt: time.Now()}); err != nil {
t.Fatal(err)
}
return rt, secret, project
}
func TestAWorkflowRunsFromTheInputsItsCatalogEntryDeclares(t *testing.T) {
wav := testWAV()
c := newRunComfy(t, wav, "song_00001_.wav")
rt, secret, _ := runRouter(t, c, songWorkflow, catalog.OutputAudio)
h := rt.Handler()
code, j, raw := call(t, h, secret, http.MethodPost, workflowsPath, map[string]any{"model": "ace", "seed": 5,
"inputs": map[string]any{"tags": "synthwave, driving", "seconds": 60, "cover": "data:image/png;base64," + testPicture(t)}})
if code != http.StatusAccepted {
t.Fatalf("%d %s", code, raw)
}
got := waitJob(t, h, secret, j["id"].(string), jobDone)
if got["status"] != "completed" || got["content_type"] != "audio/wav" {
t.Fatalf("job %v", got)
}
code, _, file := call(t, h, secret, http.MethodGet, "/v1/files/"+got["output_file_id"].(string)+"/content", nil)
if code != http.StatusOK || file != string(wav) {
t.Fatalf("the result is not the song: %d", code)
}
c.mu.Lock()
wf, _ := json.Marshal(c.workflow)
uploaded := strings.Join(c.uploaded, ",")
c.mu.Unlock()
if !strings.HasPrefix(uploaded, "llmgw-") || !strings.HasSuffix(uploaded, ".png:.png") {
t.Errorf("uploaded %q", uploaded)
}
name := strings.TrimSuffix(uploaded, ":.png")
for _, want := range []string{`"image":"` + name + `"`, `"tags":"synthwave, driving"`, `"seconds":60`,
`"strength":0.7`, `"style":"pop"`, `"lyrics":""`, `"seed":5`, `"model":"ace_step.safetensors"`} {
if !strings.Contains(string(wf), want) {
t.Errorf("%s not in the workflow sent: %s", want, wf)
}
}
}
func TestARunIsRefusedWhatItsWorkflowDoesNotTake(t *testing.T) {
c := newRunComfy(t, testWAV(), "song.wav")
rt, secret, _ := runRouter(t, c, songWorkflow, catalog.OutputAudio)
h := rt.Handler()
cover := testPicture(t)
in := func(extra map[string]any) map[string]any {
m := map[string]any{"tags": "lofi", "cover": cover}
for k, v := range extra {
m[k] = v
}
return map[string]any{"model": "ace", "inputs": m}
}
for name, c := range map[string]struct {
body map[string]any
want string
}{
"no tags": {map[string]any{"model": "ace", "inputs": map[string]any{"cover": cover}}, `input "tags" is required: text — genres and moods`},
"an unknown input": {in(map[string]any{"tempo": 120}), `takes no input "tempo"`},
"a number as text": {in(map[string]any{"tags": 5}), `input "tags" is text`},
"too long": {in(map[string]any{"tags": strings.Repeat("x", 201)}), "past 200"},
"out of bounds": {in(map[string]any{"strength": 2}), "outside 0 to 1"},
"not whole": {in(map[string]any{"seconds": 10.5}), "whole number"},
"not a choice": {in(map[string]any{"style": "jazz"}), "one of pop, metal"},
"a song as a picture": {in(map[string]any{"cover": base64.StdEncoding.EncodeToString(testWAV())}), "PNG, JPEG"},
"someone else's file": {in(map[string]any{"cover": "file-nothere"}), `no file "file-nothere"`},
"a run of flux's": {map[string]any{"model": "flux", "inputs": map[string]any{"tags": "x"}}, "takes no inputs"},
"a chat with ace": {map[string]any{"model": "ace", "messages": []any{}}, "makes audio: ask for it at " + workflowsPath},
} {
path := workflowsPath
if name == "a chat with ace" {
path = "/v1/chat/completions"
}
code, _, raw := call(t, h, secret, http.MethodPost, path, c.body)
if code != http.StatusBadRequest || !strings.Contains(errorMessage([]byte(raw)), c.want) {
t.Errorf("%s: %d %s", name, code, raw)
}
}
}
func TestARunTakesAFileUploadedBeforeAndNamedByItsID(t *testing.T) {
c := newRunComfy(t, testWAV(), "song.wav")
rt, secret, project := runRouter(t, c, songWorkflow, catalog.OutputAudio)
h := rt.Handler()
pic, _ := base64.StdEncoding.DecodeString(testPicture(t))
now := time.Now()
if err := rt.store.PutFile(t.Context(), store.File{ID: "file-cover", Owner: "project:" + project, Purpose: "user_data",
Filename: "cover.png", Bytes: int64(len(pic)), CreatedAt: now, ExpiresAt: now.Add(time.Hour)}, pic); err != nil {
t.Fatal(err)
}
if err := rt.store.PutFile(t.Context(), store.File{ID: "file-theirs", Owner: "project:other", Purpose: "user_data",
Filename: "x.png", Bytes: int64(len(pic)), CreatedAt: now, ExpiresAt: now.Add(time.Hour)}, pic); err != nil {
t.Fatal(err)
}
code, _, raw := call(t, h, secret, http.MethodPost, workflowsPath,
map[string]any{"model": "ace", "inputs": map[string]any{"tags": "x", "cover": "file-theirs"}})
if code != http.StatusBadRequest || !strings.Contains(errorMessage([]byte(raw)), `no file "file-theirs"`) {
t.Errorf("another project's file: %d %s", code, raw)
}
code, j, raw := call(t, h, secret, http.MethodPost, workflowsPath,
map[string]any{"model": "ace", "inputs": map[string]any{"tags": "x", "cover": "file-cover"}})
if code != http.StatusAccepted {
t.Fatalf("%d %s", code, raw)
}
if got := waitJob(t, h, secret, j["id"].(string), jobDone); got["status"] != "completed" {
t.Fatalf("job %v", got)
}
c.mu.Lock()
defer c.mu.Unlock()
if len(c.uploaded) != 1 || !strings.HasSuffix(c.uploaded[0], ".png:.png") {
t.Errorf("uploaded %v", c.uploaded)
}
}
func TestAWorkflowWantingWhatNoInputGivesFailsSayingSo(t *testing.T) {
c := newRunComfy(t, testWAV(), "song.wav")
wf := strings.Replace(songWorkflow, `"{{style}}"`, `"{{mood}}"`, 1)
rt, secret, _ := runRouter(t, c, wf, catalog.OutputAudio)
h := rt.Handler()
code, j, raw := call(t, h, secret, http.MethodPost, workflowsPath,
map[string]any{"model": "ace", "inputs": map[string]any{"tags": "x", "cover": testPicture(t)}})
if code != http.StatusAccepted {
t.Fatalf("%d %s", code, raw)
}
got := waitJob(t, h, secret, j["id"].(string), jobDone)
if e, _ := got["error"].(map[string]any); got["status"] != "failed" || !strings.Contains(fmt.Sprint(e["message"]), "placeholders mood") {
t.Errorf("job %v", got)
}
}
func TestARunThatSavedTheWrongKindFailsAndAPictureCarriesItsLicence(t *testing.T) {
c := newRunComfy(t, testWAV(), "song.wav")
rt, secret, _ := runRouter(t, c, songWorkflow, catalog.OutputVideo)
h := rt.Handler()
_, j, _ := call(t, h, secret, http.MethodPost, workflowsPath,
map[string]any{"model": "ace", "inputs": map[string]any{"tags": "x", "cover": testPicture(t)}})
got := waitJob(t, h, secret, j["id"].(string), jobDone)
if e, _ := got["error"].(map[string]any); !strings.Contains(fmt.Sprint(e["message"]), "saved no video: it saved song.wav (audio/wav)") {
t.Errorf("job %v", got)
}
var b bytes.Buffer
_ = png.Encode(&b, image.NewRGBA(image.Rect(0, 0, 4, 4)))
with, err := pngWithLicense(b.Bytes(), catalog.License{ID: "Apache-2.0", Name: "Apache 2.0", Attribution: "Made with ACE"})
if err != nil {
t.Fatal(err)
}
if _, err := png.Decode(bytes.NewReader(with)); err != nil {
t.Fatalf("not a PNG any more: %v", err)
}
if !bytes.Contains(with, []byte("iTXtLicense\x00\x00\x00\x00\x00{\"id\":\"Apache-2.0\"")) {
t.Error("no licence chunk in the picture")
}
if _, err := pngWithLicense([]byte("GIF89a"), catalog.License{Name: "x"}); err == nil {
t.Error("a GIF taken for a PNG")
}
}