POST /v1/3d/generations answers 202 with a job; GET says its status, progress and result file; cancel stops it in the engine's queue too. The request is kept as sent and replayed by whichever gateway holds the lease through the path a client's request takes, so it is charged and judged like any request (tagged job=<id>), and a model still loading is waited out for as long as the job has. Migration 0044. Nothing makes 3D models yet; the door refuses every model until the ComfyUI adapter learns meshes. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
286 lines
10 KiB
Go
286 lines
10 KiB
Go
package router
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/comfy"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// jobComfy is a ComfyUI that makes one picture a workflow, slowly when
|
|
// asked: stall holds every workflow until it is cleared, and the paths the
|
|
// router took to stop one are counted.
|
|
type jobComfy struct {
|
|
srv *httptest.Server
|
|
stall atomic.Bool
|
|
prompts atomic.Int32
|
|
cancelled atomic.Int32
|
|
}
|
|
|
|
func newJobComfy(t *testing.T) *jobComfy {
|
|
t.Helper()
|
|
c := &jobComfy{}
|
|
c.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch {
|
|
case r.URL.Path == "/prompt":
|
|
c.prompts.Add(1)
|
|
fmt.Fprint(w, `{"prompt_id":"p1"}`)
|
|
case r.URL.Path == "/history/p1":
|
|
if c.stall.Load() {
|
|
fmt.Fprint(w, `{}`)
|
|
return
|
|
}
|
|
fmt.Fprint(w, `{"p1":{"status":{"status_str":"success","completed":true},"outputs":{"7":{"images":[{"filename":"0.png","type":"temp"}]}}}}`)
|
|
case r.URL.Path == "/view":
|
|
w.Header().Set("Content-Type", "image/png")
|
|
fmt.Fprint(w, "png-bytes")
|
|
case r.URL.Path == "/queue" && r.Method == http.MethodPost, r.URL.Path == "/interrupt":
|
|
c.cancelled.Add(1)
|
|
fmt.Fprint(w, `{}`)
|
|
case r.URL.Path == "/queue":
|
|
fmt.Fprint(w, `{"queue_running":[[0,"p1"]],"queue_pending":[]}`)
|
|
default:
|
|
http.NotFound(w, r)
|
|
}
|
|
}))
|
|
t.Cleanup(c.srv.Close)
|
|
return c
|
|
}
|
|
|
|
// jobRouter is a gateway with the store, a key, a text model and a ComfyUI
|
|
// model priced by the picture.
|
|
func jobRouter(t *testing.T, c *jobComfy) (*Router, string) {
|
|
t.Helper()
|
|
cfg := baseConfig()
|
|
cfg.Models = []catalog.Model{catModel("chat", nil), {Name: "flux", Source: "external",
|
|
External: &catalog.External{URL: c.srv.URL, Model: "flux1-schnell-fp8.safetensors", API: catalog.APIComfyUI}}}
|
|
cfg.Auth.Enabled = true
|
|
cfg.Auth.Store = filepath.Join(t.TempDir(), "store.db")
|
|
cfg.Auth.Pricing = map[string]config.Price{"flux": {PerImage: 5000}}
|
|
cfg.Auth.FlushInterval = 20 * time.Millisecond
|
|
cfg.Auth.BatchSize = 1
|
|
rt := newTestRouter(t, cfg)
|
|
rt.comfy = &comfy.Client{Poll: time.Millisecond}
|
|
rt.SetEndpoints(t.Context(), []endpoint.Info{
|
|
{ID: "external:flux", URL: c.srv.URL, Models: []string{"flux"}, Engine: endpoint.EngineComfyUI,
|
|
Pool: "external", Weight: 1, UpstreamModel: "flux1-schnell-fp8.safetensors"},
|
|
})
|
|
org, _ := rt.store.CreateOrg(t.Context(), "acme")
|
|
team, _ := rt.store.CreateTeam(t.Context(), org, "platform")
|
|
project, err := rt.store.CreateProject(t.Context(), team, "gateway")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
secret := store.KeyPrefix + strings.Repeat("a", 43)
|
|
sum := sha256.Sum256([]byte(secret))
|
|
if err := rt.store.CreateKey(t.Context(), store.Key{ID: "key-1", Hash: sum[:], Prefix: "aaaa", Name: "test",
|
|
ProjectID: project, CreatedAt: time.Now()}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return rt, secret
|
|
}
|
|
|
|
// queueJob puts a job in the store as the create endpoint would, for an
|
|
// endpoint that serves today: until a model makes 3D models, the worker is
|
|
// tried with pictures.
|
|
func queueJob(t *testing.T, rt *Router, id, status string, body string) store.Job {
|
|
t.Helper()
|
|
now := time.Now()
|
|
j := store.Job{ID: id, Owner: "project:" + mustProjectOf(t, rt, "key-1"), KeyID: "key-1", Endpoint: imagesPath,
|
|
Model: "flux", Input: []byte(body), Status: status, CreatedAt: now, ExpiresAt: now.Add(jobWindow)}
|
|
if err := rt.store.PutJob(t.Context(), j); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return j
|
|
}
|
|
|
|
func waitJob(t *testing.T, h http.Handler, secret, id string, until func(map[string]any) bool) map[string]any {
|
|
t.Helper()
|
|
deadline := time.Now().Add(10 * time.Second)
|
|
for {
|
|
_, j, raw := call(t, h, secret, http.MethodGet, meshesPath+"/"+id, nil)
|
|
if until(j) {
|
|
return j
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("the job did not get there: %s", raw)
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
func jobDone(j map[string]any) bool { return store.JobDone(fmt.Sprint(j["status"])) }
|
|
|
|
func TestAJobIsServedItsResultKeptAndItsWorkCharged(t *testing.T) {
|
|
c := newJobComfy(t)
|
|
rt, secret := jobRouter(t, c)
|
|
h := rt.Handler()
|
|
j := queueJob(t, rt, "job_1", store.JobQueued, `{"model":"flux","prompt":"a cube"}`)
|
|
rt.wakeJobs()
|
|
|
|
got := waitJob(t, h, secret, j.ID, jobDone)
|
|
if got["status"] != "completed" || got["content_type"] != "application/json" || got["error"] != nil {
|
|
t.Fatalf("%v", got)
|
|
}
|
|
if got["started_at"] == nil || got["finished_at"] == nil {
|
|
t.Errorf("times: %v", got)
|
|
}
|
|
code, _, out := call(t, h, secret, http.MethodGet, "/v1/files/"+got["output_file_id"].(string)+"/content", nil)
|
|
if code != 200 || !strings.Contains(out, `"b64_json":"cG5nLWJ5dGVz"`) {
|
|
t.Errorf("result %d: %s", code, out)
|
|
}
|
|
_, f, _ := call(t, h, secret, http.MethodGet, "/v1/files/"+got["output_file_id"].(string), nil)
|
|
if f["purpose"] != purposeJobOutput || f["filename"] != "job_1.json" || f["expires_at"] == nil {
|
|
t.Errorf("file %v", f)
|
|
}
|
|
|
|
// Charged like any request, by the picture, and findable by the job.
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for {
|
|
u, ok, _ := rt.store.UsageByRequest(t.Context(), j.ID)
|
|
if ok {
|
|
if u.Tags["job"] != j.ID || u.Images != 1 || u.CostMicros != 5000 {
|
|
t.Errorf("usage %+v", u)
|
|
}
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("the job left no usage")
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
got = waitJob(t, h, secret, j.ID, func(j map[string]any) bool { return j["usage"] != nil })
|
|
if u := got["usage"].(map[string]any); u["images"] != 1.0 {
|
|
t.Errorf("usage shown %v", u)
|
|
}
|
|
_, list, _ := call(t, h, secret, http.MethodGet, meshesPath, nil)
|
|
if d := list["data"].([]any); len(d) != 1 {
|
|
t.Errorf("listed %v", list)
|
|
}
|
|
}
|
|
|
|
func TestAJobIsRefusedBeforeItIsQueuedWhenItCannotBeServed(t *testing.T) {
|
|
c := newJobComfy(t)
|
|
rt, secret := jobRouter(t, c)
|
|
h := rt.Handler()
|
|
code, _, raw := call(t, h, secret, http.MethodPost, meshesPath, map[string]any{"model": "chat", "prompt": "a cube"})
|
|
if code != 400 || !strings.Contains(raw, "does not make 3D models") {
|
|
t.Errorf("text model: %d %s", code, raw)
|
|
}
|
|
code, _, raw = call(t, h, secret, http.MethodPost, meshesPath, map[string]any{"model": "nope", "prompt": "a cube"})
|
|
if code != 404 || !strings.Contains(raw, "model_not_found") {
|
|
t.Errorf("unknown model: %d %s", code, raw)
|
|
}
|
|
code, _, raw = call(t, h, secret, http.MethodPost, meshesPath, map[string]any{"prompt": "a cube"})
|
|
if code != 400 || !strings.Contains(raw, "model is required") {
|
|
t.Errorf("no model: %d %s", code, raw)
|
|
}
|
|
if code, _, _ := call(t, h, "", http.MethodPost, meshesPath, map[string]any{"model": "chat"}); code != 401 {
|
|
t.Errorf("no key: %d", code)
|
|
}
|
|
if _, list, _ := call(t, h, secret, http.MethodGet, meshesPath, nil); len(list["data"].([]any)) != 0 {
|
|
t.Errorf("a refused job was queued: %v", list)
|
|
}
|
|
}
|
|
|
|
func TestARunningJobIsCancelledAndTheEngineToldToStop(t *testing.T) {
|
|
c := newJobComfy(t)
|
|
c.stall.Store(true)
|
|
rt, secret := jobRouter(t, c)
|
|
h := rt.Handler()
|
|
j := queueJob(t, rt, "job_slow", store.JobQueued, `{"model":"flux","prompt":"a cube"}`)
|
|
rt.wakeJobs()
|
|
waitJob(t, h, secret, j.ID, func(j map[string]any) bool { return j["status"] == "running" })
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for c.prompts.Load() == 0 && time.Now().Before(deadline) {
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
|
|
code, got, raw := call(t, h, secret, http.MethodPost, meshesPath+"/"+j.ID+"/cancel", nil)
|
|
if code != 200 || got["status"] != "cancelling" {
|
|
t.Fatalf("cancel: %d %s", code, raw)
|
|
}
|
|
got = waitJob(t, h, secret, j.ID, jobDone)
|
|
if got["status"] != "cancelled" || got["output_file_id"] != nil {
|
|
t.Errorf("%v", got)
|
|
}
|
|
if c.cancelled.Load() == 0 {
|
|
t.Error("ComfyUI was not told to stop")
|
|
}
|
|
|
|
// One not yet taken ends at once, and is never sent.
|
|
before := c.prompts.Load()
|
|
j2 := queueJob(t, rt, "job_never", store.JobQueued, `{"model":"flux","prompt":"a cube"}`)
|
|
_, got, _ = call(t, h, secret, http.MethodPost, meshesPath+"/"+j2.ID+"/cancel", nil)
|
|
if got["status"] != "cancelled" {
|
|
t.Errorf("%v", got)
|
|
}
|
|
rt.wakeJobs()
|
|
time.Sleep(100 * time.Millisecond)
|
|
if c.prompts.Load() != before {
|
|
t.Error("a cancelled job was sent")
|
|
}
|
|
}
|
|
|
|
func TestAJobWhoseGatewayDiedIsTakenByAnother(t *testing.T) {
|
|
c := newJobComfy(t)
|
|
rt, secret := jobRouter(t, c)
|
|
h := rt.Handler()
|
|
// As a gateway that died mid-job left it: running, its lease run out.
|
|
j := queueJob(t, rt, "job_orphan", store.JobRunning, `{"model":"flux","prompt":"a cube"}`)
|
|
j.StartedAt = time.Now().Add(-time.Minute)
|
|
_ = rt.store.UpdateJob(t.Context(), j)
|
|
rt.wakeJobs()
|
|
if got := waitJob(t, h, secret, j.ID, jobDone); got["status"] != "completed" {
|
|
t.Errorf("%v", got)
|
|
}
|
|
|
|
// A refusal of the request is the job's end, said.
|
|
bad := queueJob(t, rt, "job_bad", store.JobQueued, `{"model":"flux","prompt":"a cube","n":99}`)
|
|
rt.wakeJobs()
|
|
got := waitJob(t, h, secret, bad.ID, jobDone)
|
|
if got["status"] != "failed" || !strings.Contains(fmt.Sprint(got["error"]), "n is between 1 and 4") {
|
|
t.Errorf("%v", got)
|
|
}
|
|
}
|
|
|
|
func TestJobsAreTheirOwners(t *testing.T) {
|
|
c := newJobComfy(t)
|
|
rt, secret := jobRouter(t, c)
|
|
h := rt.Handler()
|
|
j := queueJob(t, rt, "job_mine", store.JobQueued, `{"model":"flux","prompt":"a cube"}`)
|
|
rt.wakeJobs()
|
|
waitJob(t, h, secret, j.ID, jobDone)
|
|
|
|
org, _ := rt.store.CreateOrg(t.Context(), "other")
|
|
team, _ := rt.store.CreateTeam(t.Context(), org, "t")
|
|
project, _ := rt.store.CreateProject(t.Context(), team, "p")
|
|
other := store.KeyPrefix + strings.Repeat("b", 43)
|
|
sum := sha256.Sum256([]byte(other))
|
|
if err := rt.store.CreateKey(t.Context(), store.Key{ID: "key-2", Hash: sum[:], Prefix: "bbbb", Name: "other",
|
|
ProjectID: project, CreatedAt: time.Now()}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if code, _, _ := call(t, h, other, http.MethodGet, meshesPath+"/"+j.ID, nil); code != 404 {
|
|
t.Errorf("another project read a job: %d", code)
|
|
}
|
|
if code, _, _ := call(t, h, other, http.MethodPost, meshesPath+"/"+j.ID+"/cancel", nil); code != 404 {
|
|
t.Errorf("another project cancelled a job: %d", code)
|
|
}
|
|
_, mine, _ := call(t, h, secret, http.MethodGet, meshesPath+"/"+j.ID, nil)
|
|
if code, _, _ := call(t, h, other, http.MethodGet, "/v1/files/"+mine["output_file_id"].(string)+"/content", nil); code != 404 {
|
|
t.Errorf("another project read a job's result: %d", code)
|
|
}
|
|
}
|