CI / test (push) Successful in 10m28s
One job at a time, a 3D job waiting for the card held up another model's job that could have run (seen on the PoC, BACKLOG H7). The leader's worker claims jobs while fewer than eight run, passing over the models that have one running, and starts each in its own goroutine; one ending wakes it for the next. Lease and leader-only rules are as they were. BACKLOG: H5–H7 marked, H8 (presets) and H9 (what the PoC taught) written. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
144 lines
4.7 KiB
Go
144 lines
4.7 KiB
Go
package store
|
|
|
|
import (
|
|
"fmt"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestJobs(t *testing.T) {
|
|
checkJobs(t, capacityStore(t))
|
|
}
|
|
|
|
func TestPostgresJobs(t *testing.T) {
|
|
checkJobs(t, openPG(t))
|
|
}
|
|
|
|
func checkJobs(t *testing.T, db *DB) {
|
|
ctx := t.Context()
|
|
now := time.Now()
|
|
j := Job{ID: "job-1", Owner: "p1", KeyID: "k", Endpoint: "/v1/3d/generations", Model: "trellis2",
|
|
Input: []byte(`{"model":"trellis2"}`), Status: JobQueued, CreatedAt: now, ExpiresAt: now.Add(24 * time.Hour)}
|
|
if err := db.PutJob(ctx, j); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, ok, err := db.ClaimJob(ctx, "gw-a", now, now.Add(time.Minute))
|
|
if err != nil || !ok || got.ID != "job-1" || string(got.Input) != `{"model":"trellis2"}` {
|
|
t.Fatalf("claim: %+v %v %v", got, ok, err)
|
|
}
|
|
if _, ok, _ := db.ClaimJob(ctx, "gw-b", now, now.Add(time.Minute)); ok {
|
|
t.Error("a leased job was claimed twice")
|
|
}
|
|
if held, _ := db.RenewJobLease(ctx, "job-1", "gw-b", now.Add(time.Hour)); held {
|
|
t.Error("another gateway renewed the lease")
|
|
}
|
|
if held, _ := db.RenewJobLease(ctx, "job-1", "gw-a", now.Add(time.Hour)); !held {
|
|
t.Error("the holder could not renew")
|
|
}
|
|
|
|
// Progress is written while it runs, and never over an end.
|
|
got.Status, got.StartedAt = JobRunning, now
|
|
if err := db.UpdateJob(ctx, got); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.SetJobProgress(ctx, "job-1", `{"percent":40}`, "p-7"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.SetJobProgress(ctx, "job-1", `{"percent":60}`, ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cur, _ := db.JobByID(ctx, "job-1")
|
|
if cur.Progress != `{"percent":60}` || cur.PromptID != "p-7" || cur.Status != JobRunning {
|
|
t.Errorf("%+v", cur)
|
|
}
|
|
|
|
// A running job asked to stop is cancelling, for the worker to stop it.
|
|
cur, err = db.CancelJob(ctx, "job-1", now)
|
|
if err != nil || cur.Status != JobCancelling {
|
|
t.Fatalf("%+v %v", cur, err)
|
|
}
|
|
cur.Status, cur.FinishedAt = JobCancelled, now
|
|
_ = db.UpdateJob(ctx, cur)
|
|
if err := db.SetJobProgress(ctx, "job-1", `{"percent":99}`, ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if cur, _ = db.JobByID(ctx, "job-1"); cur.Progress != `{"percent":60}` || cur.Status != JobCancelled {
|
|
t.Errorf("progress written over an end: %+v", cur)
|
|
}
|
|
_ = db.ReleaseJob(ctx, "job-1", "gw-a")
|
|
if _, ok, _ := db.ClaimJob(ctx, "gw-b", now.Add(2*time.Hour), now.Add(3*time.Hour)); ok {
|
|
t.Error("an ended job was claimed")
|
|
}
|
|
|
|
// One not yet taken ends at once when cancelled.
|
|
j2 := j
|
|
j2.ID, j2.CreatedAt = "job-2", now.Add(time.Second)
|
|
_ = db.PutJob(ctx, j2)
|
|
if cur, _ = db.CancelJob(ctx, "job-2", now.Add(2*time.Second)); cur.Status != JobCancelled || cur.FinishedAt.IsZero() {
|
|
t.Errorf("%+v", cur)
|
|
}
|
|
|
|
// Listed newest first, another owner's not at all.
|
|
j3 := j
|
|
j3.ID, j3.Owner, j3.CreatedAt = "job-3", "p2", now.Add(3*time.Second)
|
|
_ = db.PutJob(ctx, j3)
|
|
js, _ := db.Jobs(ctx, "p1", "", 10)
|
|
if len(js) != 2 || js[0].ID != "job-2" || js[1].ID != "job-1" {
|
|
t.Errorf("listed %+v", js)
|
|
}
|
|
if js, _ = db.Jobs(ctx, "p1", "job-2", 10); len(js) != 1 || js[0].ID != "job-1" {
|
|
t.Errorf("after: %+v", js)
|
|
}
|
|
if _, err := db.JobByID(ctx, "nope"); err != ErrNotFound {
|
|
t.Errorf("err %v", err)
|
|
}
|
|
if n, _ := db.DeleteJobsBefore(ctx, now.Add(time.Hour)); n != 2 {
|
|
t.Errorf("deleted %d", n)
|
|
}
|
|
}
|
|
|
|
func TestAnOperatorSeesEveryOwnersJobs(t *testing.T) {
|
|
db := capacityStore(t)
|
|
now := time.Now()
|
|
for i, owner := range []string{"project:a", "project:b", "user:c"} {
|
|
status := JobQueued
|
|
if i == 2 {
|
|
status = JobCompleted
|
|
}
|
|
if err := db.PutJob(t.Context(), Job{ID: fmt.Sprint("job_", i), Owner: owner, KeyID: "k", Endpoint: "/v1/3d/generations",
|
|
Model: "m", Input: []byte("{}"), Status: status, CreatedAt: now.Add(time.Duration(i) * time.Second),
|
|
ExpiresAt: now.Add(time.Hour)}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
all, err := db.AllJobs(t.Context(), "", 0)
|
|
if err != nil || len(all) != 3 || all[0].ID != "job_2" {
|
|
t.Fatalf("%v %v: every owner's, newest first", all, err)
|
|
}
|
|
queued, _ := db.AllJobs(t.Context(), JobQueued, 0)
|
|
if len(queued) != 2 {
|
|
t.Errorf("%d queued", len(queued))
|
|
}
|
|
}
|
|
|
|
func TestAClaimPassesOverTheModelsItIsToldTo(t *testing.T) {
|
|
db := openDB(t)
|
|
ctx := t.Context()
|
|
now := time.Now()
|
|
for i, m := range []string{"sf3d", "flux"} {
|
|
at := now.Add(time.Duration(i) * time.Second)
|
|
if err := db.PutJob(ctx, Job{ID: "job_" + m, Owner: "u", KeyID: "k", Endpoint: "/e", Model: m, Input: []byte("{}"),
|
|
Status: JobQueued, CreatedAt: at, ExpiresAt: at.Add(time.Hour)}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
j, ok, err := db.ClaimJob(ctx, "w", now, now.Add(time.Minute), "sf3d")
|
|
if err != nil || !ok || j.Model != "flux" {
|
|
t.Errorf("claimed %+v %v %v; sf3d was to be passed over", j.ID, ok, err)
|
|
}
|
|
j, ok, _ = db.ClaimJob(ctx, "w", now, now.Add(time.Minute))
|
|
if !ok || j.Model != "sf3d" {
|
|
t.Errorf("then %+v %v", j.ID, ok)
|
|
}
|
|
}
|