A 3D model is a job: queued with an id, asked after, its result a file
POST /v1/3d/generations answers 202 with a job; GET says its status, progress and result file; cancel stops it in the engine's queue too. The request is kept as sent and replayed by whichever gateway holds the lease through the path a client's request takes, so it is charged and judged like any request (tagged job=<id>), and a model still loading is waited out for as long as the job has. Migration 0044. Nothing makes 3D models yet; the door refuses every model until the ComfyUI adapter learns meshes. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
baaaa4e019
commit
b51cc3cf92
16 files changed
+1198
-10
No files matched your search
@@ -68,6 +68,8 @@ var copyOrder = []string{
|
||||
"files",
|
||||
"batches",
|
||||
"batch_results",
|
||||
// After files, which a finished job's result is one of.
|
||||
"jobs",
|
||||
"request_bodies",
|
||||
// Runs before their cases, which point at them.
|
||||
"eval_runs",
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Job is one request that is answered with an id and asked after: work that
|
||||
// takes minutes, which no client should hold a connection for. The request
|
||||
// is kept as it was sent and replayed by whichever gateway holds the lease.
|
||||
type Job struct {
|
||||
ID string
|
||||
Owner string
|
||||
KeyID string
|
||||
Endpoint string
|
||||
Model string
|
||||
// Input is the request's body, as the client sent it.
|
||||
Input []byte
|
||||
Status string
|
||||
// Progress is what the engine said of the work so far, as JSON; empty
|
||||
// until it said anything.
|
||||
Progress string
|
||||
// PromptID is the engine's own id for the work, so a gateway that takes
|
||||
// over a running job asks after the same work rather than queueing it
|
||||
// again.
|
||||
PromptID string
|
||||
OutputFileID string
|
||||
ContentType string
|
||||
Error string
|
||||
CreatedAt time.Time
|
||||
StartedAt time.Time
|
||||
FinishedAt time.Time
|
||||
// ExpiresAt is when a job not yet finished is given up on.
|
||||
ExpiresAt time.Time
|
||||
}
|
||||
|
||||
// Job states.
|
||||
const (
|
||||
JobQueued = "queued"
|
||||
JobRunning = "running"
|
||||
JobCompleted = "completed"
|
||||
JobFailed = "failed"
|
||||
JobCancelling = "cancelling"
|
||||
JobCancelled = "cancelled"
|
||||
JobExpired = "expired"
|
||||
)
|
||||
|
||||
// JobDone is whether a state is an end.
|
||||
func JobDone(status string) bool {
|
||||
switch status {
|
||||
case JobCompleted, JobFailed, JobCancelled, JobExpired:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
const jobCols = `id, owner, key_id, endpoint, model, input, status, progress, prompt_id, output_file_id,
|
||||
content_type, error, created_at, started_at, finished_at, expires_at`
|
||||
|
||||
func scanJob(sc interface{ Scan(...any) error }) (Job, error) {
|
||||
var j Job
|
||||
var created, started, finished, expires int64
|
||||
err := sc.Scan(&j.ID, &j.Owner, &j.KeyID, &j.Endpoint, &j.Model, &j.Input, &j.Status, &j.Progress, &j.PromptID,
|
||||
&j.OutputFileID, &j.ContentType, &j.Error, &created, &started, &finished, &expires)
|
||||
if err != nil {
|
||||
return Job{}, err
|
||||
}
|
||||
at := func(v int64) time.Time {
|
||||
if v == 0 {
|
||||
return time.Time{}
|
||||
}
|
||||
return time.UnixMilli(v)
|
||||
}
|
||||
j.CreatedAt, j.StartedAt, j.FinishedAt, j.ExpiresAt = at(created), at(started), at(finished), at(expires)
|
||||
return j, nil
|
||||
}
|
||||
|
||||
// PutJob records a new job.
|
||||
func (s *DB) PutJob(ctx context.Context, j Job) error {
|
||||
if j.Input == nil {
|
||||
j.Input = []byte{}
|
||||
}
|
||||
_, err := s.db.ExecContext(ctx, `INSERT INTO jobs (id, owner, key_id, endpoint, model, input, status, created_at, expires_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
j.ID, j.Owner, j.KeyID, j.Endpoint, j.Model, j.Input, j.Status, millis(j.CreatedAt), millis(j.ExpiresAt))
|
||||
return err
|
||||
}
|
||||
|
||||
// UpdateJob writes how a job stands: its state, result, error and times.
|
||||
// Its progress and prompt id are written by SetJobProgress, which the work
|
||||
// calls often and which must not undo an end written here.
|
||||
func (s *DB) UpdateJob(ctx context.Context, j Job) error {
|
||||
_, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = ?, output_file_id = ?, content_type = ?, error = ?,
|
||||
started_at = ?, finished_at = ? WHERE id = ?`,
|
||||
j.Status, j.OutputFileID, j.ContentType, j.Error, optMillis(j.StartedAt), optMillis(j.FinishedAt), j.ID)
|
||||
return err
|
||||
}
|
||||
|
||||
// SetJobProgress writes what the engine said of a running job, and its id
|
||||
// for the work when it is known. A job that has ended keeps its end.
|
||||
func (s *DB) SetJobProgress(ctx context.Context, id, progress, promptID string) error {
|
||||
_, err := s.db.ExecContext(ctx, `UPDATE jobs SET progress = ?,
|
||||
prompt_id = CASE WHEN ? = '' THEN prompt_id ELSE ? END
|
||||
WHERE id = ? AND status IN (?, ?)`, progress, promptID, promptID, id, JobRunning, JobCancelling)
|
||||
return err
|
||||
}
|
||||
|
||||
// CancelJob asks for a job to stop: cancelling, unless it has ended already.
|
||||
// It says the job as it stands.
|
||||
func (s *DB) CancelJob(ctx context.Context, id string, now time.Time) (Job, error) {
|
||||
// One not yet taken ends at once: there is nothing to stop.
|
||||
if _, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = ?, finished_at = ?
|
||||
WHERE id = ? AND status = ? AND lease_until < ?`, JobCancelled, millis(now), id, JobQueued, millis(now)); err != nil {
|
||||
return Job{}, err
|
||||
}
|
||||
if _, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = ? WHERE id = ? AND status IN (?, ?)`,
|
||||
JobCancelling, id, JobQueued, JobRunning); err != nil {
|
||||
return Job{}, err
|
||||
}
|
||||
return s.JobByID(ctx, id)
|
||||
}
|
||||
|
||||
// JobByID finds a job, ErrNotFound when there is none.
|
||||
func (s *DB) JobByID(ctx context.Context, id string) (Job, error) {
|
||||
j, err := scanJob(s.db.QueryRowContext(ctx, `SELECT `+jobCols+` FROM jobs WHERE id = ?`, id))
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Job{}, ErrNotFound
|
||||
}
|
||||
return j, err
|
||||
}
|
||||
|
||||
// Jobs lists an owner's jobs, the newest first, those made before the job
|
||||
// after when it is given.
|
||||
func (s *DB) Jobs(ctx context.Context, owner, after string, limit int) ([]Job, error) {
|
||||
q := `SELECT ` + jobCols + ` FROM jobs WHERE owner = ?`
|
||||
args := []any{owner}
|
||||
if after != "" {
|
||||
q += ` AND created_at < (SELECT created_at FROM jobs WHERE id = ?)`
|
||||
args = append(args, after)
|
||||
}
|
||||
if limit <= 0 || limit > 100 {
|
||||
limit = 20
|
||||
}
|
||||
q += ` ORDER BY created_at DESC LIMIT ?`
|
||||
args = append(args, limit)
|
||||
rows, err := s.db.QueryContext(ctx, q, args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Job
|
||||
for rows.Next() {
|
||||
j, err := scanJob(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, j)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// ClaimJob takes the lease on the oldest job with work left that no gateway
|
||||
// holds, for owner until the time given. ok is false when there is none.
|
||||
func (s *DB) ClaimJob(ctx context.Context, owner string, now, until time.Time) (Job, bool, error) {
|
||||
var id string
|
||||
err := s.db.QueryRowContext(ctx, `UPDATE jobs SET lease_owner = ?, lease_until = ?
|
||||
WHERE id = (SELECT id FROM jobs WHERE status IN (?, ?, ?) AND lease_until < ?
|
||||
ORDER BY created_at LIMIT 1)
|
||||
AND lease_until < ?
|
||||
RETURNING id`, owner, millis(until), JobQueued, JobRunning, JobCancelling, millis(now), millis(now)).Scan(&id)
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
return Job{}, false, nil
|
||||
}
|
||||
if err != nil {
|
||||
return Job{}, false, err
|
||||
}
|
||||
j, err := s.JobByID(ctx, id)
|
||||
return j, err == nil, err
|
||||
}
|
||||
|
||||
// RenewJobLease keeps a lease held; false when it is held no longer.
|
||||
func (s *DB) RenewJobLease(ctx context.Context, id, owner string, until time.Time) (bool, error) {
|
||||
res, err := s.db.ExecContext(ctx, `UPDATE jobs SET lease_until = ? WHERE id = ? AND lease_owner = ?`,
|
||||
millis(until), id, owner)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
n, _ := res.RowsAffected()
|
||||
return n > 0, nil
|
||||
}
|
||||
|
||||
// ReleaseJob gives a lease up.
|
||||
func (s *DB) ReleaseJob(ctx context.Context, id, owner string) error {
|
||||
_, err := s.db.ExecContext(ctx, `UPDATE jobs SET lease_until = 0, lease_owner = '' WHERE id = ? AND lease_owner = ?`, id, owner)
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteJobsBefore forgets the jobs that ended before the time given, whose
|
||||
// result files are gone by then too.
|
||||
func (s *DB) DeleteJobsBefore(ctx context.Context, before time.Time) (int64, error) {
|
||||
res, err := s.db.ExecContext(ctx, `DELETE FROM jobs WHERE finished_at > 0 AND finished_at < ?`, millis(before))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return res.RowsAffected()
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestJobs(t *testing.T) {
|
||||
checkJobs(t, capacityStore(t))
|
||||
}
|
||||
|
||||
func TestPostgresJobs(t *testing.T) {
|
||||
checkJobs(t, openPG(t))
|
||||
}
|
||||
|
||||
func checkJobs(t *testing.T, db *DB) {
|
||||
ctx := t.Context()
|
||||
now := time.Now()
|
||||
j := Job{ID: "job-1", Owner: "p1", KeyID: "k", Endpoint: "/v1/3d/generations", Model: "trellis2",
|
||||
Input: []byte(`{"model":"trellis2"}`), Status: JobQueued, CreatedAt: now, ExpiresAt: now.Add(24 * time.Hour)}
|
||||
if err := db.PutJob(ctx, j); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, ok, err := db.ClaimJob(ctx, "gw-a", now, now.Add(time.Minute))
|
||||
if err != nil || !ok || got.ID != "job-1" || string(got.Input) != `{"model":"trellis2"}` {
|
||||
t.Fatalf("claim: %+v %v %v", got, ok, err)
|
||||
}
|
||||
if _, ok, _ := db.ClaimJob(ctx, "gw-b", now, now.Add(time.Minute)); ok {
|
||||
t.Error("a leased job was claimed twice")
|
||||
}
|
||||
if held, _ := db.RenewJobLease(ctx, "job-1", "gw-b", now.Add(time.Hour)); held {
|
||||
t.Error("another gateway renewed the lease")
|
||||
}
|
||||
if held, _ := db.RenewJobLease(ctx, "job-1", "gw-a", now.Add(time.Hour)); !held {
|
||||
t.Error("the holder could not renew")
|
||||
}
|
||||
|
||||
// Progress is written while it runs, and never over an end.
|
||||
got.Status, got.StartedAt = JobRunning, now
|
||||
if err := db.UpdateJob(ctx, got); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.SetJobProgress(ctx, "job-1", `{"percent":40}`, "p-7"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.SetJobProgress(ctx, "job-1", `{"percent":60}`, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cur, _ := db.JobByID(ctx, "job-1")
|
||||
if cur.Progress != `{"percent":60}` || cur.PromptID != "p-7" || cur.Status != JobRunning {
|
||||
t.Errorf("%+v", cur)
|
||||
}
|
||||
|
||||
// A running job asked to stop is cancelling, for the worker to stop it.
|
||||
cur, err = db.CancelJob(ctx, "job-1", now)
|
||||
if err != nil || cur.Status != JobCancelling {
|
||||
t.Fatalf("%+v %v", cur, err)
|
||||
}
|
||||
cur.Status, cur.FinishedAt = JobCancelled, now
|
||||
_ = db.UpdateJob(ctx, cur)
|
||||
if err := db.SetJobProgress(ctx, "job-1", `{"percent":99}`, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cur, _ = db.JobByID(ctx, "job-1"); cur.Progress != `{"percent":60}` || cur.Status != JobCancelled {
|
||||
t.Errorf("progress written over an end: %+v", cur)
|
||||
}
|
||||
_ = db.ReleaseJob(ctx, "job-1", "gw-a")
|
||||
if _, ok, _ := db.ClaimJob(ctx, "gw-b", now.Add(2*time.Hour), now.Add(3*time.Hour)); ok {
|
||||
t.Error("an ended job was claimed")
|
||||
}
|
||||
|
||||
// One not yet taken ends at once when cancelled.
|
||||
j2 := j
|
||||
j2.ID, j2.CreatedAt = "job-2", now.Add(time.Second)
|
||||
_ = db.PutJob(ctx, j2)
|
||||
if cur, _ = db.CancelJob(ctx, "job-2", now.Add(2*time.Second)); cur.Status != JobCancelled || cur.FinishedAt.IsZero() {
|
||||
t.Errorf("%+v", cur)
|
||||
}
|
||||
|
||||
// Listed newest first, another owner's not at all.
|
||||
j3 := j
|
||||
j3.ID, j3.Owner, j3.CreatedAt = "job-3", "p2", now.Add(3*time.Second)
|
||||
_ = db.PutJob(ctx, j3)
|
||||
js, _ := db.Jobs(ctx, "p1", "", 10)
|
||||
if len(js) != 2 || js[0].ID != "job-2" || js[1].ID != "job-1" {
|
||||
t.Errorf("listed %+v", js)
|
||||
}
|
||||
if js, _ = db.Jobs(ctx, "p1", "job-2", 10); len(js) != 1 || js[0].ID != "job-1" {
|
||||
t.Errorf("after: %+v", js)
|
||||
}
|
||||
if _, err := db.JobByID(ctx, "nope"); err != ErrNotFound {
|
||||
t.Errorf("err %v", err)
|
||||
}
|
||||
if n, _ := db.DeleteJobsBefore(ctx, now.Add(time.Hour)); n != 2 {
|
||||
t.Errorf("deleted %d", n)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
-- Ported from the SQLite migration of the same number.
|
||||
-- Jobs (BACKLOG G1): a request that takes minutes — a 3D model from a
|
||||
-- picture — answered with an id and asked after, rather than held open.
|
||||
-- The request is kept as it was sent, so whichever gateway holds the lease
|
||||
-- replays it through the same path a client's request takes; the result is
|
||||
-- a file in the files table, kept for the batch files' retention.
|
||||
CREATE TABLE jobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
owner TEXT NOT NULL,
|
||||
key_id TEXT NOT NULL,
|
||||
endpoint TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
input BYTEA NOT NULL,
|
||||
-- status is queued, running, completed, failed, cancelling, cancelled or
|
||||
-- expired.
|
||||
status TEXT NOT NULL,
|
||||
-- progress is what the engine said of the work so far, as JSON: queue
|
||||
-- position, percent, step. Empty until it says anything.
|
||||
progress TEXT NOT NULL DEFAULT '',
|
||||
-- prompt_id is the engine's own id for the work (ComfyUI's prompt_id),
|
||||
-- so a gateway that takes over a running job asks after the same work.
|
||||
prompt_id TEXT NOT NULL DEFAULT '',
|
||||
output_file_id TEXT NOT NULL DEFAULT '',
|
||||
content_type TEXT NOT NULL DEFAULT '',
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
created_at BIGINT NOT NULL,
|
||||
started_at BIGINT NOT NULL DEFAULT 0,
|
||||
finished_at BIGINT NOT NULL DEFAULT 0,
|
||||
expires_at BIGINT NOT NULL,
|
||||
lease_owner TEXT NOT NULL DEFAULT '',
|
||||
lease_until BIGINT NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX jobs_owner ON jobs (owner, created_at);
|
||||
CREATE INDEX jobs_status ON jobs (status, lease_until);
|
||||
@@ -0,0 +1,33 @@
|
||||
-- Jobs (BACKLOG G1): a request that takes minutes — a 3D model from a
|
||||
-- picture — answered with an id and asked after, rather than held open.
|
||||
-- The request is kept as it was sent, so whichever gateway holds the lease
|
||||
-- replays it through the same path a client's request takes; the result is
|
||||
-- a file in the files table, kept for the batch files' retention.
|
||||
CREATE TABLE jobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
owner TEXT NOT NULL,
|
||||
key_id TEXT NOT NULL,
|
||||
endpoint TEXT NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
input BLOB NOT NULL,
|
||||
-- status is queued, running, completed, failed, cancelling, cancelled or
|
||||
-- expired.
|
||||
status TEXT NOT NULL,
|
||||
-- progress is what the engine said of the work so far, as JSON: queue
|
||||
-- position, percent, step. Empty until it says anything.
|
||||
progress TEXT NOT NULL DEFAULT '',
|
||||
-- prompt_id is the engine's own id for the work (ComfyUI's prompt_id),
|
||||
-- so a gateway that takes over a running job asks after the same work.
|
||||
prompt_id TEXT NOT NULL DEFAULT '',
|
||||
output_file_id TEXT NOT NULL DEFAULT '',
|
||||
content_type TEXT NOT NULL DEFAULT '',
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
created_at INTEGER NOT NULL,
|
||||
started_at INTEGER NOT NULL DEFAULT 0,
|
||||
finished_at INTEGER NOT NULL DEFAULT 0,
|
||||
expires_at INTEGER NOT NULL,
|
||||
lease_owner TEXT NOT NULL DEFAULT '',
|
||||
lease_until INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX jobs_owner ON jobs (owner, created_at);
|
||||
CREATE INDEX jobs_status ON jobs (status, lease_until);
|
||||
Reference in new issue
Block a user