Files
llmgw/internal/router/jobs_test.go
T
Ilker ManapandClaude Fable 5.1 b51cc3cf92 A 3D model is a job: queued with an id, asked after, its result a file
POST /v1/3d/generations answers 202 with a job; GET says its status,
progress and result file; cancel stops it in the engine's queue too. The
request is kept as sent and replayed by whichever gateway holds the lease
through the path a client's request takes, so it is charged and judged
like any request (tagged job=<id>), and a model still loading is waited
out for as long as the job has. Migration 0044. Nothing makes 3D models
yet; the door refuses every model until the ComfyUI adapter learns meshes.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-10 16:04:11 +02:00

286 lines
10 KiB
Go

package router
import (
"crypto/sha256"
"fmt"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"sync/atomic"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/comfy"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// jobComfy is a ComfyUI that makes one picture a workflow, slowly when
// asked: stall holds every workflow until it is cleared, and the paths the
// router took to stop one are counted.
type jobComfy struct {
srv *httptest.Server
stall atomic.Bool
prompts atomic.Int32
cancelled atomic.Int32
}
func newJobComfy(t *testing.T) *jobComfy {
t.Helper()
c := &jobComfy{}
c.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch {
case r.URL.Path == "/prompt":
c.prompts.Add(1)
fmt.Fprint(w, `{"prompt_id":"p1"}`)
case r.URL.Path == "/history/p1":
if c.stall.Load() {
fmt.Fprint(w, `{}`)
return
}
fmt.Fprint(w, `{"p1":{"status":{"status_str":"success","completed":true},"outputs":{"7":{"images":[{"filename":"0.png","type":"temp"}]}}}}`)
case r.URL.Path == "/view":
w.Header().Set("Content-Type", "image/png")
fmt.Fprint(w, "png-bytes")
case r.URL.Path == "/queue" && r.Method == http.MethodPost, r.URL.Path == "/interrupt":
c.cancelled.Add(1)
fmt.Fprint(w, `{}`)
case r.URL.Path == "/queue":
fmt.Fprint(w, `{"queue_running":[[0,"p1"]],"queue_pending":[]}`)
default:
http.NotFound(w, r)
}
}))
t.Cleanup(c.srv.Close)
return c
}
// jobRouter is a gateway with the store, a key, a text model and a ComfyUI
// model priced by the picture.
func jobRouter(t *testing.T, c *jobComfy) (*Router, string) {
t.Helper()
cfg := baseConfig()
cfg.Models = []catalog.Model{catModel("chat", nil), {Name: "flux", Source: "external",
External: &catalog.External{URL: c.srv.URL, Model: "flux1-schnell-fp8.safetensors", API: catalog.APIComfyUI}}}
cfg.Auth.Enabled = true
cfg.Auth.Store = filepath.Join(t.TempDir(), "store.db")
cfg.Auth.Pricing = map[string]config.Price{"flux": {PerImage: 5000}}
cfg.Auth.FlushInterval = 20 * time.Millisecond
cfg.Auth.BatchSize = 1
rt := newTestRouter(t, cfg)
rt.comfy = &comfy.Client{Poll: time.Millisecond}
rt.SetEndpoints(t.Context(), []endpoint.Info{
{ID: "external:flux", URL: c.srv.URL, Models: []string{"flux"}, Engine: endpoint.EngineComfyUI,
Pool: "external", Weight: 1, UpstreamModel: "flux1-schnell-fp8.safetensors"},
})
org, _ := rt.store.CreateOrg(t.Context(), "acme")
team, _ := rt.store.CreateTeam(t.Context(), org, "platform")
project, err := rt.store.CreateProject(t.Context(), team, "gateway")
if err != nil {
t.Fatal(err)
}
secret := store.KeyPrefix + strings.Repeat("a", 43)
sum := sha256.Sum256([]byte(secret))
if err := rt.store.CreateKey(t.Context(), store.Key{ID: "key-1", Hash: sum[:], Prefix: "aaaa", Name: "test",
ProjectID: project, CreatedAt: time.Now()}); err != nil {
t.Fatal(err)
}
return rt, secret
}
// queueJob puts a job in the store as the create endpoint would, for an
// endpoint that serves today: until a model makes 3D models, the worker is
// tried with pictures.
func queueJob(t *testing.T, rt *Router, id, status string, body string) store.Job {
t.Helper()
now := time.Now()
j := store.Job{ID: id, Owner: "project:" + mustProjectOf(t, rt, "key-1"), KeyID: "key-1", Endpoint: imagesPath,
Model: "flux", Input: []byte(body), Status: status, CreatedAt: now, ExpiresAt: now.Add(jobWindow)}
if err := rt.store.PutJob(t.Context(), j); err != nil {
t.Fatal(err)
}
return j
}
func waitJob(t *testing.T, h http.Handler, secret, id string, until func(map[string]any) bool) map[string]any {
t.Helper()
deadline := time.Now().Add(10 * time.Second)
for {
_, j, raw := call(t, h, secret, http.MethodGet, meshesPath+"/"+id, nil)
if until(j) {
return j
}
if time.Now().After(deadline) {
t.Fatalf("the job did not get there: %s", raw)
}
time.Sleep(20 * time.Millisecond)
}
}
func jobDone(j map[string]any) bool { return store.JobDone(fmt.Sprint(j["status"])) }
func TestAJobIsServedItsResultKeptAndItsWorkCharged(t *testing.T) {
c := newJobComfy(t)
rt, secret := jobRouter(t, c)
h := rt.Handler()
j := queueJob(t, rt, "job_1", store.JobQueued, `{"model":"flux","prompt":"a cube"}`)
rt.wakeJobs()
got := waitJob(t, h, secret, j.ID, jobDone)
if got["status"] != "completed" || got["content_type"] != "application/json" || got["error"] != nil {
t.Fatalf("%v", got)
}
if got["started_at"] == nil || got["finished_at"] == nil {
t.Errorf("times: %v", got)
}
code, _, out := call(t, h, secret, http.MethodGet, "/v1/files/"+got["output_file_id"].(string)+"/content", nil)
if code != 200 || !strings.Contains(out, `"b64_json":"cG5nLWJ5dGVz"`) {
t.Errorf("result %d: %s", code, out)
}
_, f, _ := call(t, h, secret, http.MethodGet, "/v1/files/"+got["output_file_id"].(string), nil)
if f["purpose"] != purposeJobOutput || f["filename"] != "job_1.json" || f["expires_at"] == nil {
t.Errorf("file %v", f)
}
// Charged like any request, by the picture, and findable by the job.
deadline := time.Now().Add(5 * time.Second)
for {
u, ok, _ := rt.store.UsageByRequest(t.Context(), j.ID)
if ok {
if u.Tags["job"] != j.ID || u.Images != 1 || u.CostMicros != 5000 {
t.Errorf("usage %+v", u)
}
break
}
if time.Now().After(deadline) {
t.Fatal("the job left no usage")
}
time.Sleep(20 * time.Millisecond)
}
got = waitJob(t, h, secret, j.ID, func(j map[string]any) bool { return j["usage"] != nil })
if u := got["usage"].(map[string]any); u["images"] != 1.0 {
t.Errorf("usage shown %v", u)
}
_, list, _ := call(t, h, secret, http.MethodGet, meshesPath, nil)
if d := list["data"].([]any); len(d) != 1 {
t.Errorf("listed %v", list)
}
}
func TestAJobIsRefusedBeforeItIsQueuedWhenItCannotBeServed(t *testing.T) {
c := newJobComfy(t)
rt, secret := jobRouter(t, c)
h := rt.Handler()
code, _, raw := call(t, h, secret, http.MethodPost, meshesPath, map[string]any{"model": "chat", "prompt": "a cube"})
if code != 400 || !strings.Contains(raw, "does not make 3D models") {
t.Errorf("text model: %d %s", code, raw)
}
code, _, raw = call(t, h, secret, http.MethodPost, meshesPath, map[string]any{"model": "nope", "prompt": "a cube"})
if code != 404 || !strings.Contains(raw, "model_not_found") {
t.Errorf("unknown model: %d %s", code, raw)
}
code, _, raw = call(t, h, secret, http.MethodPost, meshesPath, map[string]any{"prompt": "a cube"})
if code != 400 || !strings.Contains(raw, "model is required") {
t.Errorf("no model: %d %s", code, raw)
}
if code, _, _ := call(t, h, "", http.MethodPost, meshesPath, map[string]any{"model": "chat"}); code != 401 {
t.Errorf("no key: %d", code)
}
if _, list, _ := call(t, h, secret, http.MethodGet, meshesPath, nil); len(list["data"].([]any)) != 0 {
t.Errorf("a refused job was queued: %v", list)
}
}
func TestARunningJobIsCancelledAndTheEngineToldToStop(t *testing.T) {
c := newJobComfy(t)
c.stall.Store(true)
rt, secret := jobRouter(t, c)
h := rt.Handler()
j := queueJob(t, rt, "job_slow", store.JobQueued, `{"model":"flux","prompt":"a cube"}`)
rt.wakeJobs()
waitJob(t, h, secret, j.ID, func(j map[string]any) bool { return j["status"] == "running" })
deadline := time.Now().Add(5 * time.Second)
for c.prompts.Load() == 0 && time.Now().Before(deadline) {
time.Sleep(10 * time.Millisecond)
}
code, got, raw := call(t, h, secret, http.MethodPost, meshesPath+"/"+j.ID+"/cancel", nil)
if code != 200 || got["status"] != "cancelling" {
t.Fatalf("cancel: %d %s", code, raw)
}
got = waitJob(t, h, secret, j.ID, jobDone)
if got["status"] != "cancelled" || got["output_file_id"] != nil {
t.Errorf("%v", got)
}
if c.cancelled.Load() == 0 {
t.Error("ComfyUI was not told to stop")
}
// One not yet taken ends at once, and is never sent.
before := c.prompts.Load()
j2 := queueJob(t, rt, "job_never", store.JobQueued, `{"model":"flux","prompt":"a cube"}`)
_, got, _ = call(t, h, secret, http.MethodPost, meshesPath+"/"+j2.ID+"/cancel", nil)
if got["status"] != "cancelled" {
t.Errorf("%v", got)
}
rt.wakeJobs()
time.Sleep(100 * time.Millisecond)
if c.prompts.Load() != before {
t.Error("a cancelled job was sent")
}
}
func TestAJobWhoseGatewayDiedIsTakenByAnother(t *testing.T) {
c := newJobComfy(t)
rt, secret := jobRouter(t, c)
h := rt.Handler()
// As a gateway that died mid-job left it: running, its lease run out.
j := queueJob(t, rt, "job_orphan", store.JobRunning, `{"model":"flux","prompt":"a cube"}`)
j.StartedAt = time.Now().Add(-time.Minute)
_ = rt.store.UpdateJob(t.Context(), j)
rt.wakeJobs()
if got := waitJob(t, h, secret, j.ID, jobDone); got["status"] != "completed" {
t.Errorf("%v", got)
}
// A refusal of the request is the job's end, said.
bad := queueJob(t, rt, "job_bad", store.JobQueued, `{"model":"flux","prompt":"a cube","n":99}`)
rt.wakeJobs()
got := waitJob(t, h, secret, bad.ID, jobDone)
if got["status"] != "failed" || !strings.Contains(fmt.Sprint(got["error"]), "n is between 1 and 4") {
t.Errorf("%v", got)
}
}
func TestJobsAreTheirOwners(t *testing.T) {
c := newJobComfy(t)
rt, secret := jobRouter(t, c)
h := rt.Handler()
j := queueJob(t, rt, "job_mine", store.JobQueued, `{"model":"flux","prompt":"a cube"}`)
rt.wakeJobs()
waitJob(t, h, secret, j.ID, jobDone)
org, _ := rt.store.CreateOrg(t.Context(), "other")
team, _ := rt.store.CreateTeam(t.Context(), org, "t")
project, _ := rt.store.CreateProject(t.Context(), team, "p")
other := store.KeyPrefix + strings.Repeat("b", 43)
sum := sha256.Sum256([]byte(other))
if err := rt.store.CreateKey(t.Context(), store.Key{ID: "key-2", Hash: sum[:], Prefix: "bbbb", Name: "other",
ProjectID: project, CreatedAt: time.Now()}); err != nil {
t.Fatal(err)
}
if code, _, _ := call(t, h, other, http.MethodGet, meshesPath+"/"+j.ID, nil); code != 404 {
t.Errorf("another project read a job: %d", code)
}
if code, _, _ := call(t, h, other, http.MethodPost, meshesPath+"/"+j.ID+"/cancel", nil); code != 404 {
t.Errorf("another project cancelled a job: %d", code)
}
_, mine, _ := call(t, h, secret, http.MethodGet, meshesPath+"/"+j.ID, nil)
if code, _, _ := call(t, h, other, http.MethodGet, "/v1/files/"+mine["output_file_id"].(string)+"/content", nil); code != 404 {
t.Errorf("another project read a job's result: %d", code)
}
}