/3d beside /images, while some model makes meshes: a picture chosen, the
job asked for through the same path an API client's takes, its progress
shown (waiting for the card, n ahead; making it, the node and percent),
and the mesh in a viewer — model-viewer 4.4.0 (Apache 2.0, with three.js
and lit), served from the gateway since the page's policy loads scripts
from nowhere else — with a download and the model's licence. The mesh is
served to its owner from /3d/jobs/{id}/file; another project's person is
told there is no such job.
A job made from the portal acted for a key that does not exist: the
worker now rebuilds a session's principal from the account that made it,
while it is active and still in the project the job is charged to.
ctl 3d list|get|cancel, over /api/v1/client-jobs (/api/v1/jobs is the
control plane's own work, which ctl jobs shows).
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
123 lines
4.0 KiB
Go
123 lines
4.0 KiB
Go
package store
|
|
|
|
import (
|
|
"fmt"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestJobs(t *testing.T) {
|
|
checkJobs(t, capacityStore(t))
|
|
}
|
|
|
|
func TestPostgresJobs(t *testing.T) {
|
|
checkJobs(t, openPG(t))
|
|
}
|
|
|
|
func checkJobs(t *testing.T, db *DB) {
|
|
ctx := t.Context()
|
|
now := time.Now()
|
|
j := Job{ID: "job-1", Owner: "p1", KeyID: "k", Endpoint: "/v1/3d/generations", Model: "trellis2",
|
|
Input: []byte(`{"model":"trellis2"}`), Status: JobQueued, CreatedAt: now, ExpiresAt: now.Add(24 * time.Hour)}
|
|
if err := db.PutJob(ctx, j); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got, ok, err := db.ClaimJob(ctx, "gw-a", now, now.Add(time.Minute))
|
|
if err != nil || !ok || got.ID != "job-1" || string(got.Input) != `{"model":"trellis2"}` {
|
|
t.Fatalf("claim: %+v %v %v", got, ok, err)
|
|
}
|
|
if _, ok, _ := db.ClaimJob(ctx, "gw-b", now, now.Add(time.Minute)); ok {
|
|
t.Error("a leased job was claimed twice")
|
|
}
|
|
if held, _ := db.RenewJobLease(ctx, "job-1", "gw-b", now.Add(time.Hour)); held {
|
|
t.Error("another gateway renewed the lease")
|
|
}
|
|
if held, _ := db.RenewJobLease(ctx, "job-1", "gw-a", now.Add(time.Hour)); !held {
|
|
t.Error("the holder could not renew")
|
|
}
|
|
|
|
// Progress is written while it runs, and never over an end.
|
|
got.Status, got.StartedAt = JobRunning, now
|
|
if err := db.UpdateJob(ctx, got); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.SetJobProgress(ctx, "job-1", `{"percent":40}`, "p-7"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := db.SetJobProgress(ctx, "job-1", `{"percent":60}`, ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cur, _ := db.JobByID(ctx, "job-1")
|
|
if cur.Progress != `{"percent":60}` || cur.PromptID != "p-7" || cur.Status != JobRunning {
|
|
t.Errorf("%+v", cur)
|
|
}
|
|
|
|
// A running job asked to stop is cancelling, for the worker to stop it.
|
|
cur, err = db.CancelJob(ctx, "job-1", now)
|
|
if err != nil || cur.Status != JobCancelling {
|
|
t.Fatalf("%+v %v", cur, err)
|
|
}
|
|
cur.Status, cur.FinishedAt = JobCancelled, now
|
|
_ = db.UpdateJob(ctx, cur)
|
|
if err := db.SetJobProgress(ctx, "job-1", `{"percent":99}`, ""); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if cur, _ = db.JobByID(ctx, "job-1"); cur.Progress != `{"percent":60}` || cur.Status != JobCancelled {
|
|
t.Errorf("progress written over an end: %+v", cur)
|
|
}
|
|
_ = db.ReleaseJob(ctx, "job-1", "gw-a")
|
|
if _, ok, _ := db.ClaimJob(ctx, "gw-b", now.Add(2*time.Hour), now.Add(3*time.Hour)); ok {
|
|
t.Error("an ended job was claimed")
|
|
}
|
|
|
|
// One not yet taken ends at once when cancelled.
|
|
j2 := j
|
|
j2.ID, j2.CreatedAt = "job-2", now.Add(time.Second)
|
|
_ = db.PutJob(ctx, j2)
|
|
if cur, _ = db.CancelJob(ctx, "job-2", now.Add(2*time.Second)); cur.Status != JobCancelled || cur.FinishedAt.IsZero() {
|
|
t.Errorf("%+v", cur)
|
|
}
|
|
|
|
// Listed newest first, another owner's not at all.
|
|
j3 := j
|
|
j3.ID, j3.Owner, j3.CreatedAt = "job-3", "p2", now.Add(3*time.Second)
|
|
_ = db.PutJob(ctx, j3)
|
|
js, _ := db.Jobs(ctx, "p1", "", 10)
|
|
if len(js) != 2 || js[0].ID != "job-2" || js[1].ID != "job-1" {
|
|
t.Errorf("listed %+v", js)
|
|
}
|
|
if js, _ = db.Jobs(ctx, "p1", "job-2", 10); len(js) != 1 || js[0].ID != "job-1" {
|
|
t.Errorf("after: %+v", js)
|
|
}
|
|
if _, err := db.JobByID(ctx, "nope"); err != ErrNotFound {
|
|
t.Errorf("err %v", err)
|
|
}
|
|
if n, _ := db.DeleteJobsBefore(ctx, now.Add(time.Hour)); n != 2 {
|
|
t.Errorf("deleted %d", n)
|
|
}
|
|
}
|
|
|
|
func TestAnOperatorSeesEveryOwnersJobs(t *testing.T) {
|
|
db := capacityStore(t)
|
|
now := time.Now()
|
|
for i, owner := range []string{"project:a", "project:b", "user:c"} {
|
|
status := JobQueued
|
|
if i == 2 {
|
|
status = JobCompleted
|
|
}
|
|
if err := db.PutJob(t.Context(), Job{ID: fmt.Sprint("job_", i), Owner: owner, KeyID: "k", Endpoint: "/v1/3d/generations",
|
|
Model: "m", Input: []byte("{}"), Status: status, CreatedAt: now.Add(time.Duration(i) * time.Second),
|
|
ExpiresAt: now.Add(time.Hour)}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
all, err := db.AllJobs(t.Context(), "", 0)
|
|
if err != nil || len(all) != 3 || all[0].ID != "job_2" {
|
|
t.Fatalf("%v %v: every owner's, newest first", all, err)
|
|
}
|
|
queued, _ := db.AllJobs(t.Context(), JobQueued, 0)
|
|
if len(queued) != 2 {
|
|
t.Errorf("%d queued", len(queued))
|
|
}
|
|
}
|