One job at a time, a 3D job waiting for the card held up another model's job that could have run (seen on the PoC, BACKLOG H7). The leader's worker claims jobs while fewer than eight run, passing over the models that have one running, and starts each in its own goroutine; one ending wakes it for the next. Lease and leader-only rules are as they were. BACKLOG: H5–H7 marked, H8 (presets) and H9 (what the PoC taught) written. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
392 lines
14 KiB
Go
392 lines
14 KiB
Go
package router
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/comfy"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// jobComfy is a ComfyUI that makes one picture a workflow, slowly when
|
|
// asked: stall holds every workflow until it is cleared, and the paths the
|
|
// router took to stop one are counted.
|
|
type jobComfy struct {
|
|
srv *httptest.Server
|
|
stall atomic.Bool
|
|
prompts atomic.Int32
|
|
cancelled atomic.Int32
|
|
}
|
|
|
|
func newJobComfy(t *testing.T) *jobComfy {
|
|
t.Helper()
|
|
c := &jobComfy{}
|
|
c.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
switch {
|
|
case r.URL.Path == "/prompt":
|
|
c.prompts.Add(1)
|
|
fmt.Fprint(w, `{"prompt_id":"p1"}`)
|
|
case r.URL.Path == "/history/p1":
|
|
if c.stall.Load() {
|
|
fmt.Fprint(w, `{}`)
|
|
return
|
|
}
|
|
fmt.Fprint(w, `{"p1":{"status":{"status_str":"success","completed":true},"outputs":{"7":{"images":[{"filename":"0.png","type":"temp"}]}}}}`)
|
|
case r.URL.Path == "/view":
|
|
w.Header().Set("Content-Type", "image/png")
|
|
fmt.Fprint(w, "png-bytes")
|
|
case r.URL.Path == "/queue" && r.Method == http.MethodPost, r.URL.Path == "/interrupt":
|
|
c.cancelled.Add(1)
|
|
fmt.Fprint(w, `{}`)
|
|
case r.URL.Path == "/queue":
|
|
fmt.Fprint(w, `{"queue_running":[[0,"p1"]],"queue_pending":[]}`)
|
|
default:
|
|
http.NotFound(w, r)
|
|
}
|
|
}))
|
|
t.Cleanup(c.srv.Close)
|
|
return c
|
|
}
|
|
|
|
// jobRouter is a gateway with the store, a key, a text model and a ComfyUI
|
|
// model priced by the picture.
|
|
func jobRouter(t *testing.T, c *jobComfy) (*Router, string) {
|
|
t.Helper()
|
|
cfg := baseConfig()
|
|
cfg.Models = []catalog.Model{catModel("chat", nil), {Name: "flux", Source: "external",
|
|
External: &catalog.External{URL: c.srv.URL, Model: "flux1-schnell-fp8.safetensors", API: catalog.APIComfyUI}}}
|
|
cfg.Auth.Enabled = true
|
|
cfg.Auth.Store = filepath.Join(t.TempDir(), "store.db")
|
|
cfg.Auth.Pricing = map[string]config.Price{"flux": {PerImage: 5000}}
|
|
cfg.Auth.FlushInterval = 20 * time.Millisecond
|
|
cfg.Auth.BatchSize = 1
|
|
rt := newTestRouter(t, cfg)
|
|
rt.comfy = &comfy.Client{Poll: time.Millisecond}
|
|
rt.SetEndpoints(t.Context(), []endpoint.Info{
|
|
{ID: "external:flux", URL: c.srv.URL, Models: []string{"flux"}, Engine: endpoint.EngineComfyUI,
|
|
Pool: "external", Weight: 1, UpstreamModel: "flux1-schnell-fp8.safetensors"},
|
|
})
|
|
org, _ := rt.store.CreateOrg(t.Context(), "acme")
|
|
team, _ := rt.store.CreateTeam(t.Context(), org, "platform")
|
|
project, err := rt.store.CreateProject(t.Context(), team, "gateway")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
secret := store.KeyPrefix + strings.Repeat("a", 43)
|
|
sum := sha256.Sum256([]byte(secret))
|
|
if err := rt.store.CreateKey(t.Context(), store.Key{ID: "key-1", Hash: sum[:], Prefix: "aaaa", Name: "test",
|
|
ProjectID: project, CreatedAt: time.Now()}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return rt, secret
|
|
}
|
|
|
|
// queueJob puts a job in the store as the create endpoint would, for an
|
|
// endpoint that serves today: until a model makes 3D models, the worker is
|
|
// tried with pictures.
|
|
func queueJob(t *testing.T, rt *Router, id, status string, body string) store.Job {
|
|
t.Helper()
|
|
now := time.Now()
|
|
j := store.Job{ID: id, Owner: "project:" + mustProjectOf(t, rt, "key-1"), KeyID: "key-1", Endpoint: imagesPath,
|
|
Model: "flux", Input: []byte(body), Status: status, CreatedAt: now, ExpiresAt: now.Add(jobWindow)}
|
|
if err := rt.store.PutJob(t.Context(), j); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return j
|
|
}
|
|
|
|
func waitJob(t *testing.T, h http.Handler, secret, id string, until func(map[string]any) bool) map[string]any {
|
|
t.Helper()
|
|
deadline := time.Now().Add(10 * time.Second)
|
|
for {
|
|
_, j, raw := call(t, h, secret, http.MethodGet, meshesPath+"/"+id, nil)
|
|
if until(j) {
|
|
return j
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatalf("the job did not get there: %s", raw)
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
}
|
|
|
|
func jobDone(j map[string]any) bool { return store.JobDone(fmt.Sprint(j["status"])) }
|
|
|
|
func TestAJobIsServedItsResultKeptAndItsWorkCharged(t *testing.T) {
|
|
c := newJobComfy(t)
|
|
rt, secret := jobRouter(t, c)
|
|
h := rt.Handler()
|
|
j := queueJob(t, rt, "job_1", store.JobQueued, `{"model":"flux","prompt":"a cube"}`)
|
|
rt.wakeJobs()
|
|
|
|
got := waitJob(t, h, secret, j.ID, jobDone)
|
|
if got["status"] != "completed" || got["content_type"] != "application/json" || got["error"] != nil {
|
|
t.Fatalf("%v", got)
|
|
}
|
|
if got["started_at"] == nil || got["finished_at"] == nil {
|
|
t.Errorf("times: %v", got)
|
|
}
|
|
code, _, out := call(t, h, secret, http.MethodGet, "/v1/files/"+got["output_file_id"].(string)+"/content", nil)
|
|
if code != 200 || !strings.Contains(out, `"b64_json":"cG5nLWJ5dGVz"`) {
|
|
t.Errorf("result %d: %s", code, out)
|
|
}
|
|
_, f, _ := call(t, h, secret, http.MethodGet, "/v1/files/"+got["output_file_id"].(string), nil)
|
|
if f["purpose"] != purposeJobOutput || f["filename"] != "job_1.json" || f["expires_at"] == nil {
|
|
t.Errorf("file %v", f)
|
|
}
|
|
|
|
// Charged like any request, by the picture, and findable by the job.
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for {
|
|
u, ok, _ := rt.store.UsageByRequest(t.Context(), j.ID)
|
|
if ok {
|
|
if u.Tags["job"] != j.ID || u.Images != 1 || u.CostMicros != 5000 {
|
|
t.Errorf("usage %+v", u)
|
|
}
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
t.Fatal("the job left no usage")
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
got = waitJob(t, h, secret, j.ID, func(j map[string]any) bool { return j["usage"] != nil })
|
|
if u := got["usage"].(map[string]any); u["images"] != 1.0 {
|
|
t.Errorf("usage shown %v", u)
|
|
}
|
|
_, list, _ := call(t, h, secret, http.MethodGet, meshesPath, nil)
|
|
if d := list["data"].([]any); len(d) != 1 {
|
|
t.Errorf("listed %v", list)
|
|
}
|
|
}
|
|
|
|
func TestAJobIsRefusedBeforeItIsQueuedWhenItCannotBeServed(t *testing.T) {
|
|
c := newJobComfy(t)
|
|
rt, secret := jobRouter(t, c)
|
|
h := rt.Handler()
|
|
code, _, raw := call(t, h, secret, http.MethodPost, meshesPath, map[string]any{"model": "chat", "prompt": "a cube"})
|
|
if code != 400 || !strings.Contains(raw, "does not make 3D models") {
|
|
t.Errorf("text model: %d %s", code, raw)
|
|
}
|
|
code, _, raw = call(t, h, secret, http.MethodPost, meshesPath, map[string]any{"model": "nope", "prompt": "a cube"})
|
|
if code != 404 || !strings.Contains(raw, "model_not_found") {
|
|
t.Errorf("unknown model: %d %s", code, raw)
|
|
}
|
|
code, _, raw = call(t, h, secret, http.MethodPost, meshesPath, map[string]any{"prompt": "a cube"})
|
|
if code != 400 || !strings.Contains(raw, "model is required") {
|
|
t.Errorf("no model: %d %s", code, raw)
|
|
}
|
|
if code, _, _ := call(t, h, "", http.MethodPost, meshesPath, map[string]any{"model": "chat"}); code != 401 {
|
|
t.Errorf("no key: %d", code)
|
|
}
|
|
if _, list, _ := call(t, h, secret, http.MethodGet, meshesPath, nil); len(list["data"].([]any)) != 0 {
|
|
t.Errorf("a refused job was queued: %v", list)
|
|
}
|
|
}
|
|
|
|
func TestARunningJobIsCancelledAndTheEngineToldToStop(t *testing.T) {
|
|
c := newJobComfy(t)
|
|
c.stall.Store(true)
|
|
rt, secret := jobRouter(t, c)
|
|
h := rt.Handler()
|
|
j := queueJob(t, rt, "job_slow", store.JobQueued, `{"model":"flux","prompt":"a cube"}`)
|
|
rt.wakeJobs()
|
|
waitJob(t, h, secret, j.ID, func(j map[string]any) bool { return j["status"] == "running" })
|
|
deadline := time.Now().Add(5 * time.Second)
|
|
for c.prompts.Load() == 0 && time.Now().Before(deadline) {
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
|
|
code, got, raw := call(t, h, secret, http.MethodPost, meshesPath+"/"+j.ID+"/cancel", nil)
|
|
if code != 200 || got["status"] != "cancelling" {
|
|
t.Fatalf("cancel: %d %s", code, raw)
|
|
}
|
|
got = waitJob(t, h, secret, j.ID, jobDone)
|
|
if got["status"] != "cancelled" || got["output_file_id"] != nil {
|
|
t.Errorf("%v", got)
|
|
}
|
|
if c.cancelled.Load() == 0 {
|
|
t.Error("ComfyUI was not told to stop")
|
|
}
|
|
|
|
// One not yet taken ends at once, and is never sent.
|
|
before := c.prompts.Load()
|
|
j2 := queueJob(t, rt, "job_never", store.JobQueued, `{"model":"flux","prompt":"a cube"}`)
|
|
_, got, _ = call(t, h, secret, http.MethodPost, meshesPath+"/"+j2.ID+"/cancel", nil)
|
|
if got["status"] != "cancelled" {
|
|
t.Errorf("%v", got)
|
|
}
|
|
rt.wakeJobs()
|
|
time.Sleep(100 * time.Millisecond)
|
|
if c.prompts.Load() != before {
|
|
t.Error("a cancelled job was sent")
|
|
}
|
|
}
|
|
|
|
func TestAJobWhoseGatewayDiedIsTakenByAnother(t *testing.T) {
|
|
c := newJobComfy(t)
|
|
rt, secret := jobRouter(t, c)
|
|
h := rt.Handler()
|
|
// As a gateway that died mid-job left it: running, its lease run out.
|
|
j := queueJob(t, rt, "job_orphan", store.JobRunning, `{"model":"flux","prompt":"a cube"}`)
|
|
j.StartedAt = time.Now().Add(-time.Minute)
|
|
_ = rt.store.UpdateJob(t.Context(), j)
|
|
rt.wakeJobs()
|
|
if got := waitJob(t, h, secret, j.ID, jobDone); got["status"] != "completed" {
|
|
t.Errorf("%v", got)
|
|
}
|
|
|
|
// A refusal of the request is the job's end, said.
|
|
bad := queueJob(t, rt, "job_bad", store.JobQueued, `{"model":"flux","prompt":"a cube","n":99}`)
|
|
rt.wakeJobs()
|
|
got := waitJob(t, h, secret, bad.ID, jobDone)
|
|
if got["status"] != "failed" || !strings.Contains(fmt.Sprint(got["error"]), "n is between 1 and 4") {
|
|
t.Errorf("%v", got)
|
|
}
|
|
}
|
|
|
|
func TestJobsAreTheirOwners(t *testing.T) {
|
|
c := newJobComfy(t)
|
|
rt, secret := jobRouter(t, c)
|
|
h := rt.Handler()
|
|
j := queueJob(t, rt, "job_mine", store.JobQueued, `{"model":"flux","prompt":"a cube"}`)
|
|
rt.wakeJobs()
|
|
waitJob(t, h, secret, j.ID, jobDone)
|
|
|
|
org, _ := rt.store.CreateOrg(t.Context(), "other")
|
|
team, _ := rt.store.CreateTeam(t.Context(), org, "t")
|
|
project, _ := rt.store.CreateProject(t.Context(), team, "p")
|
|
other := store.KeyPrefix + strings.Repeat("b", 43)
|
|
sum := sha256.Sum256([]byte(other))
|
|
if err := rt.store.CreateKey(t.Context(), store.Key{ID: "key-2", Hash: sum[:], Prefix: "bbbb", Name: "other",
|
|
ProjectID: project, CreatedAt: time.Now()}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if code, _, _ := call(t, h, other, http.MethodGet, meshesPath+"/"+j.ID, nil); code != 404 {
|
|
t.Errorf("another project read a job: %d", code)
|
|
}
|
|
if code, _, _ := call(t, h, other, http.MethodPost, meshesPath+"/"+j.ID+"/cancel", nil); code != 404 {
|
|
t.Errorf("another project cancelled a job: %d", code)
|
|
}
|
|
_, mine, _ := call(t, h, secret, http.MethodGet, meshesPath+"/"+j.ID, nil)
|
|
if code, _, _ := call(t, h, other, http.MethodGet, "/v1/files/"+mine["output_file_id"].(string)+"/content", nil); code != 404 {
|
|
t.Errorf("another project read a job's result: %d", code)
|
|
}
|
|
}
|
|
|
|
func TestAJobWaitingOnAnotherModelsCardsSaysSo(t *testing.T) {
|
|
c := newJobComfy(t)
|
|
rt, secret := jobRouter(t, c)
|
|
rt.cfg.ActivationWait = 10 * time.Millisecond
|
|
rt.SetActivator(&waitingActivator{fakeActivator: fakeActivator{onDemand: map[string]bool{"sleepy": true}},
|
|
heldBy: "flux-schnell", eta: 90 * time.Second})
|
|
j := queueJob(t, rt, "job_w", store.JobQueued, `{"model":"sleepy","prompt":"a cube"}`)
|
|
rt.wakeJobs()
|
|
|
|
got := waitJob(t, rt.Handler(), secret, j.ID, func(j map[string]any) bool { return j["progress"] != nil })
|
|
p, _ := got["progress"].(map[string]any)
|
|
if got["status"] != "running" || p["state"] != "waiting_gpu" || p["held_by"] != "flux-schnell" || p["eta_seconds"] != 90.0 {
|
|
t.Errorf("job %v", got)
|
|
}
|
|
}
|
|
|
|
type countingUnservable struct {
|
|
unservableActivator
|
|
asked atomic.Int32
|
|
}
|
|
|
|
func (c *countingUnservable) Unservable(model string) (string, bool) {
|
|
c.asked.Add(1)
|
|
return c.unservableActivator.Unservable(model)
|
|
}
|
|
|
|
func TestAJobSaidToFitNowhereKeepsAskingAWhileBeforeFailing(t *testing.T) {
|
|
// An agent restarting re-registers without its engines for a few
|
|
// seconds; five 3D jobs asked for in those seconds failed for good.
|
|
grace, retry := jobUnplacedGrace, jobUnplacedRetry
|
|
jobUnplacedGrace, jobUnplacedRetry = 300*time.Millisecond, 20*time.Millisecond
|
|
t.Cleanup(func() { jobUnplacedGrace, jobUnplacedRetry = grace, retry })
|
|
|
|
c := newJobComfy(t)
|
|
rt, secret := jobRouter(t, c)
|
|
act := &countingUnservable{unservableActivator: unservableActivator{
|
|
fakeActivator: fakeActivator{onDemand: map[string]bool{"sleepy": true}},
|
|
why: map[string]string{"sleepy": "sleepy cannot be placed on any node: needs 15 GB"},
|
|
}}
|
|
rt.SetActivator(act)
|
|
j := queueJob(t, rt, "job_u", store.JobQueued, `{"model":"sleepy","prompt":"a cube"}`)
|
|
start := time.Now()
|
|
rt.wakeJobs()
|
|
|
|
got := waitJob(t, rt.Handler(), secret, j.ID, jobDone)
|
|
if got["status"] != "failed" || !strings.Contains(fmt.Sprint(got["error"]), "needs 15 GB") {
|
|
t.Errorf("job %v", got)
|
|
}
|
|
if took := time.Since(start); took < jobUnplacedGrace {
|
|
t.Errorf("failed after %s, before the grace of %s", took, jobUnplacedGrace)
|
|
}
|
|
if n := act.asked.Load(); n < 3 {
|
|
t.Errorf("asked %d times in the grace", n)
|
|
}
|
|
}
|
|
|
|
func TestAStandbyLeavesJobsToTheLeader(t *testing.T) {
|
|
// A standby's router has no endpoints; a 3D job it leased was told
|
|
// "loading" for seven minutes while the leader served the jobs behind it.
|
|
c := newJobComfy(t)
|
|
rt, secret := jobRouter(t, c)
|
|
var lead atomic.Bool
|
|
rt.SetLeader(lead.Load)
|
|
j := queueJob(t, rt, "job_s", store.JobQueued, `{"model":"flux","prompt":"a cube"}`)
|
|
rt.wakeJobs()
|
|
time.Sleep(200 * time.Millisecond)
|
|
if got, _ := rt.store.JobByID(t.Context(), j.ID); got.Status != store.JobQueued {
|
|
t.Fatalf("a standby took the job: %s", got.Status)
|
|
}
|
|
|
|
lead.Store(true)
|
|
rt.wakeJobs()
|
|
if got := waitJob(t, rt.Handler(), secret, j.ID, jobDone); got["status"] != "completed" {
|
|
t.Errorf("the leader did not serve it: %v", got)
|
|
}
|
|
}
|
|
|
|
func TestAJobWaitingForTheCardDoesNotHoldUpAnotherModelsJob(t *testing.T) {
|
|
// One at a time, an sf3d job waiting for the card held up a job for a
|
|
// model that could have run.
|
|
c := newJobComfy(t)
|
|
rt, secret := jobRouter(t, c)
|
|
rt.cfg.ActivationWait = 10 * time.Millisecond
|
|
rt.SetActivator(&waitingActivator{fakeActivator: fakeActivator{onDemand: map[string]bool{"sleepy": true}},
|
|
heldBy: "flux", eta: time.Hour})
|
|
now := time.Now()
|
|
put := func(id, model, body string, at time.Time) {
|
|
if err := rt.store.PutJob(t.Context(), store.Job{ID: id, Owner: "project:" + mustProjectOf(t, rt, "key-1"),
|
|
KeyID: "key-1", Endpoint: imagesPath, Model: model, Input: []byte(body), Status: store.JobQueued,
|
|
CreatedAt: at, ExpiresAt: at.Add(jobWindow)}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
put("job_wait", "sleepy", `{"model":"sleepy","prompt":"a cube"}`, now.Add(-time.Minute))
|
|
put("job_go", "flux", `{"model":"flux","prompt":"a cube"}`, now)
|
|
rt.wakeJobs()
|
|
|
|
got := waitJob(t, rt.Handler(), secret, "job_go", jobDone)
|
|
if got["status"] != "completed" {
|
|
t.Errorf("the other model's job: %v", got)
|
|
}
|
|
if w, _ := rt.store.JobByID(t.Context(), "job_wait"); w.Status != store.JobRunning {
|
|
t.Errorf("the waiting job is %s", w.Status)
|
|
}
|
|
}
|