A 3D model is a job: queued with an id, asked after, its result a file

POST /v1/3d/generations answers 202 with a job; GET says its status,
progress and result file; cancel stops it in the engine's queue too. The
request is kept as sent and replayed by whichever gateway holds the lease
through the path a client's request takes, so it is charged and judged
like any request (tagged job=<id>), and a model still loading is waited
out for as long as the job has. Migration 0044. Nothing makes 3D models
yet; the door refuses every model until the ComfyUI adapter learns meshes.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Fable 5.1 committed 2026-10-10 16:04:11 +02:00
1 parent baaaa4e019
commit b51cc3cf92
16 files changed
+1198 -10

No files matched your search

+2
View File
@@ -68,6 +68,8 @@ var copyOrder = []string{
"files",
"batches",
"batch_results",
// After files, which a finished job's result is one of.
"jobs",
"request_bodies",
// Runs before their cases, which point at them.
"eval_runs",
+208
View File
@@ -0,0 +1,208 @@
package store
import (
"context"
"database/sql"
"errors"
"time"
)
// Job is one request that is answered with an id and asked after: work that
// takes minutes, which no client should hold a connection for. The request
// is kept as it was sent and replayed by whichever gateway holds the lease.
type Job struct {
ID string
Owner string
KeyID string
Endpoint string
Model string
// Input is the request's body, as the client sent it.
Input []byte
Status string
// Progress is what the engine said of the work so far, as JSON; empty
// until it said anything.
Progress string
// PromptID is the engine's own id for the work, so a gateway that takes
// over a running job asks after the same work rather than queueing it
// again.
PromptID string
OutputFileID string
ContentType string
Error string
CreatedAt time.Time
StartedAt time.Time
FinishedAt time.Time
// ExpiresAt is when a job not yet finished is given up on.
ExpiresAt time.Time
}
// Job states.
const (
JobQueued = "queued"
JobRunning = "running"
JobCompleted = "completed"
JobFailed = "failed"
JobCancelling = "cancelling"
JobCancelled = "cancelled"
JobExpired = "expired"
)
// JobDone is whether a state is an end.
func JobDone(status string) bool {
switch status {
case JobCompleted, JobFailed, JobCancelled, JobExpired:
return true
}
return false
}
const jobCols = `id, owner, key_id, endpoint, model, input, status, progress, prompt_id, output_file_id,
content_type, error, created_at, started_at, finished_at, expires_at`
func scanJob(sc interface{ Scan(...any) error }) (Job, error) {
var j Job
var created, started, finished, expires int64
err := sc.Scan(&j.ID, &j.Owner, &j.KeyID, &j.Endpoint, &j.Model, &j.Input, &j.Status, &j.Progress, &j.PromptID,
&j.OutputFileID, &j.ContentType, &j.Error, &created, &started, &finished, &expires)
if err != nil {
return Job{}, err
}
at := func(v int64) time.Time {
if v == 0 {
return time.Time{}
}
return time.UnixMilli(v)
}
j.CreatedAt, j.StartedAt, j.FinishedAt, j.ExpiresAt = at(created), at(started), at(finished), at(expires)
return j, nil
}
// PutJob records a new job.
func (s *DB) PutJob(ctx context.Context, j Job) error {
if j.Input == nil {
j.Input = []byte{}
}
_, err := s.db.ExecContext(ctx, `INSERT INTO jobs (id, owner, key_id, endpoint, model, input, status, created_at, expires_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
j.ID, j.Owner, j.KeyID, j.Endpoint, j.Model, j.Input, j.Status, millis(j.CreatedAt), millis(j.ExpiresAt))
return err
}
// UpdateJob writes how a job stands: its state, result, error and times.
// Its progress and prompt id are written by SetJobProgress, which the work
// calls often and which must not undo an end written here.
func (s *DB) UpdateJob(ctx context.Context, j Job) error {
_, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = ?, output_file_id = ?, content_type = ?, error = ?,
started_at = ?, finished_at = ? WHERE id = ?`,
j.Status, j.OutputFileID, j.ContentType, j.Error, optMillis(j.StartedAt), optMillis(j.FinishedAt), j.ID)
return err
}
// SetJobProgress writes what the engine said of a running job, and its id
// for the work when it is known. A job that has ended keeps its end.
func (s *DB) SetJobProgress(ctx context.Context, id, progress, promptID string) error {
_, err := s.db.ExecContext(ctx, `UPDATE jobs SET progress = ?,
prompt_id = CASE WHEN ? = '' THEN prompt_id ELSE ? END
WHERE id = ? AND status IN (?, ?)`, progress, promptID, promptID, id, JobRunning, JobCancelling)
return err
}
// CancelJob asks for a job to stop: cancelling, unless it has ended already.
// It says the job as it stands.
func (s *DB) CancelJob(ctx context.Context, id string, now time.Time) (Job, error) {
// One not yet taken ends at once: there is nothing to stop.
if _, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = ?, finished_at = ?
WHERE id = ? AND status = ? AND lease_until < ?`, JobCancelled, millis(now), id, JobQueued, millis(now)); err != nil {
return Job{}, err
}
if _, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = ? WHERE id = ? AND status IN (?, ?)`,
JobCancelling, id, JobQueued, JobRunning); err != nil {
return Job{}, err
}
return s.JobByID(ctx, id)
}
// JobByID finds a job, ErrNotFound when there is none.
func (s *DB) JobByID(ctx context.Context, id string) (Job, error) {
j, err := scanJob(s.db.QueryRowContext(ctx, `SELECT `+jobCols+` FROM jobs WHERE id = ?`, id))
if errors.Is(err, sql.ErrNoRows) {
return Job{}, ErrNotFound
}
return j, err
}
// Jobs lists an owner's jobs, the newest first, those made before the job
// after when it is given.
func (s *DB) Jobs(ctx context.Context, owner, after string, limit int) ([]Job, error) {
q := `SELECT ` + jobCols + ` FROM jobs WHERE owner = ?`
args := []any{owner}
if after != "" {
q += ` AND created_at < (SELECT created_at FROM jobs WHERE id = ?)`
args = append(args, after)
}
if limit <= 0 || limit > 100 {
limit = 20
}
q += ` ORDER BY created_at DESC LIMIT ?`
args = append(args, limit)
rows, err := s.db.QueryContext(ctx, q, args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Job
for rows.Next() {
j, err := scanJob(rows)
if err != nil {
return nil, err
}
out = append(out, j)
}
return out, rows.Err()
}
// ClaimJob takes the lease on the oldest job with work left that no gateway
// holds, for owner until the time given. ok is false when there is none.
func (s *DB) ClaimJob(ctx context.Context, owner string, now, until time.Time) (Job, bool, error) {
var id string
err := s.db.QueryRowContext(ctx, `UPDATE jobs SET lease_owner = ?, lease_until = ?
WHERE id = (SELECT id FROM jobs WHERE status IN (?, ?, ?) AND lease_until < ?
ORDER BY created_at LIMIT 1)
AND lease_until < ?
RETURNING id`, owner, millis(until), JobQueued, JobRunning, JobCancelling, millis(now), millis(now)).Scan(&id)
if errors.Is(err, sql.ErrNoRows) {
return Job{}, false, nil
}
if err != nil {
return Job{}, false, err
}
j, err := s.JobByID(ctx, id)
return j, err == nil, err
}
// RenewJobLease keeps a lease held; false when it is held no longer.
func (s *DB) RenewJobLease(ctx context.Context, id, owner string, until time.Time) (bool, error) {
res, err := s.db.ExecContext(ctx, `UPDATE jobs SET lease_until = ? WHERE id = ? AND lease_owner = ?`,
millis(until), id, owner)
if err != nil {
return false, err
}
n, _ := res.RowsAffected()
return n > 0, nil
}
// ReleaseJob gives a lease up.
func (s *DB) ReleaseJob(ctx context.Context, id, owner string) error {
_, err := s.db.ExecContext(ctx, `UPDATE jobs SET lease_until = 0, lease_owner = '' WHERE id = ? AND lease_owner = ?`, id, owner)
return err
}
// DeleteJobsBefore forgets the jobs that ended before the time given, whose
// result files are gone by then too.
func (s *DB) DeleteJobsBefore(ctx context.Context, before time.Time) (int64, error) {
res, err := s.db.ExecContext(ctx, `DELETE FROM jobs WHERE finished_at > 0 AND finished_at < ?`, millis(before))
if err != nil {
return 0, err
}
return res.RowsAffected()
}
+97
View File
@@ -0,0 +1,97 @@
package store
import (
"testing"
"time"
)
func TestJobs(t *testing.T) {
checkJobs(t, capacityStore(t))
}
func TestPostgresJobs(t *testing.T) {
checkJobs(t, openPG(t))
}
func checkJobs(t *testing.T, db *DB) {
ctx := t.Context()
now := time.Now()
j := Job{ID: "job-1", Owner: "p1", KeyID: "k", Endpoint: "/v1/3d/generations", Model: "trellis2",
Input: []byte(`{"model":"trellis2"}`), Status: JobQueued, CreatedAt: now, ExpiresAt: now.Add(24 * time.Hour)}
if err := db.PutJob(ctx, j); err != nil {
t.Fatal(err)
}
got, ok, err := db.ClaimJob(ctx, "gw-a", now, now.Add(time.Minute))
if err != nil || !ok || got.ID != "job-1" || string(got.Input) != `{"model":"trellis2"}` {
t.Fatalf("claim: %+v %v %v", got, ok, err)
}
if _, ok, _ := db.ClaimJob(ctx, "gw-b", now, now.Add(time.Minute)); ok {
t.Error("a leased job was claimed twice")
}
if held, _ := db.RenewJobLease(ctx, "job-1", "gw-b", now.Add(time.Hour)); held {
t.Error("another gateway renewed the lease")
}
if held, _ := db.RenewJobLease(ctx, "job-1", "gw-a", now.Add(time.Hour)); !held {
t.Error("the holder could not renew")
}
// Progress is written while it runs, and never over an end.
got.Status, got.StartedAt = JobRunning, now
if err := db.UpdateJob(ctx, got); err != nil {
t.Fatal(err)
}
if err := db.SetJobProgress(ctx, "job-1", `{"percent":40}`, "p-7"); err != nil {
t.Fatal(err)
}
if err := db.SetJobProgress(ctx, "job-1", `{"percent":60}`, ""); err != nil {
t.Fatal(err)
}
cur, _ := db.JobByID(ctx, "job-1")
if cur.Progress != `{"percent":60}` || cur.PromptID != "p-7" || cur.Status != JobRunning {
t.Errorf("%+v", cur)
}
// A running job asked to stop is cancelling, for the worker to stop it.
cur, err = db.CancelJob(ctx, "job-1", now)
if err != nil || cur.Status != JobCancelling {
t.Fatalf("%+v %v", cur, err)
}
cur.Status, cur.FinishedAt = JobCancelled, now
_ = db.UpdateJob(ctx, cur)
if err := db.SetJobProgress(ctx, "job-1", `{"percent":99}`, ""); err != nil {
t.Fatal(err)
}
if cur, _ = db.JobByID(ctx, "job-1"); cur.Progress != `{"percent":60}` || cur.Status != JobCancelled {
t.Errorf("progress written over an end: %+v", cur)
}
_ = db.ReleaseJob(ctx, "job-1", "gw-a")
if _, ok, _ := db.ClaimJob(ctx, "gw-b", now.Add(2*time.Hour), now.Add(3*time.Hour)); ok {
t.Error("an ended job was claimed")
}
// One not yet taken ends at once when cancelled.
j2 := j
j2.ID, j2.CreatedAt = "job-2", now.Add(time.Second)
_ = db.PutJob(ctx, j2)
if cur, _ = db.CancelJob(ctx, "job-2", now.Add(2*time.Second)); cur.Status != JobCancelled || cur.FinishedAt.IsZero() {
t.Errorf("%+v", cur)
}
// Listed newest first, another owner's not at all.
j3 := j
j3.ID, j3.Owner, j3.CreatedAt = "job-3", "p2", now.Add(3*time.Second)
_ = db.PutJob(ctx, j3)
js, _ := db.Jobs(ctx, "p1", "", 10)
if len(js) != 2 || js[0].ID != "job-2" || js[1].ID != "job-1" {
t.Errorf("listed %+v", js)
}
if js, _ = db.Jobs(ctx, "p1", "job-2", 10); len(js) != 1 || js[0].ID != "job-1" {
t.Errorf("after: %+v", js)
}
if _, err := db.JobByID(ctx, "nope"); err != ErrNotFound {
t.Errorf("err %v", err)
}
if n, _ := db.DeleteJobsBefore(ctx, now.Add(time.Hour)); n != 2 {
t.Errorf("deleted %d", n)
}
}
@@ -0,0 +1,34 @@
-- Ported from the SQLite migration of the same number.
-- Jobs (BACKLOG G1): a request that takes minutes — a 3D model from a
-- picture — answered with an id and asked after, rather than held open.
-- The request is kept as it was sent, so whichever gateway holds the lease
-- replays it through the same path a client's request takes; the result is
-- a file in the files table, kept for the batch files' retention.
CREATE TABLE jobs (
id TEXT PRIMARY KEY,
owner TEXT NOT NULL,
key_id TEXT NOT NULL,
endpoint TEXT NOT NULL,
model TEXT NOT NULL,
input BYTEA NOT NULL,
-- status is queued, running, completed, failed, cancelling, cancelled or
-- expired.
status TEXT NOT NULL,
-- progress is what the engine said of the work so far, as JSON: queue
-- position, percent, step. Empty until it says anything.
progress TEXT NOT NULL DEFAULT '',
-- prompt_id is the engine's own id for the work (ComfyUI's prompt_id),
-- so a gateway that takes over a running job asks after the same work.
prompt_id TEXT NOT NULL DEFAULT '',
output_file_id TEXT NOT NULL DEFAULT '',
content_type TEXT NOT NULL DEFAULT '',
error TEXT NOT NULL DEFAULT '',
created_at BIGINT NOT NULL,
started_at BIGINT NOT NULL DEFAULT 0,
finished_at BIGINT NOT NULL DEFAULT 0,
expires_at BIGINT NOT NULL,
lease_owner TEXT NOT NULL DEFAULT '',
lease_until BIGINT NOT NULL DEFAULT 0
);
CREATE INDEX jobs_owner ON jobs (owner, created_at);
CREATE INDEX jobs_status ON jobs (status, lease_until);
@@ -0,0 +1,33 @@
-- Jobs (BACKLOG G1): a request that takes minutes — a 3D model from a
-- picture — answered with an id and asked after, rather than held open.
-- The request is kept as it was sent, so whichever gateway holds the lease
-- replays it through the same path a client's request takes; the result is
-- a file in the files table, kept for the batch files' retention.
CREATE TABLE jobs (
id TEXT PRIMARY KEY,
owner TEXT NOT NULL,
key_id TEXT NOT NULL,
endpoint TEXT NOT NULL,
model TEXT NOT NULL,
input BLOB NOT NULL,
-- status is queued, running, completed, failed, cancelling, cancelled or
-- expired.
status TEXT NOT NULL,
-- progress is what the engine said of the work so far, as JSON: queue
-- position, percent, step. Empty until it says anything.
progress TEXT NOT NULL DEFAULT '',
-- prompt_id is the engine's own id for the work (ComfyUI's prompt_id),
-- so a gateway that takes over a running job asks after the same work.
prompt_id TEXT NOT NULL DEFAULT '',
output_file_id TEXT NOT NULL DEFAULT '',
content_type TEXT NOT NULL DEFAULT '',
error TEXT NOT NULL DEFAULT '',
created_at INTEGER NOT NULL,
started_at INTEGER NOT NULL DEFAULT 0,
finished_at INTEGER NOT NULL DEFAULT 0,
expires_at INTEGER NOT NULL,
lease_owner TEXT NOT NULL DEFAULT '',
lease_until INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX jobs_owner ON jobs (owner, created_at);
CREATE INDEX jobs_status ON jobs (status, lease_until);