/3d beside /images, while some model makes meshes: a picture chosen, the
job asked for through the same path an API client's takes, its progress
shown (waiting for the card, n ahead; making it, the node and percent),
and the mesh in a viewer — model-viewer 4.4.0 (Apache 2.0, with three.js
and lit), served from the gateway since the page's policy loads scripts
from nowhere else — with a download and the model's licence. The mesh is
served to its owner from /3d/jobs/{id}/file; another project's person is
told there is no such job.
A job made from the portal acted for a key that does not exist: the
worker now rebuilds a session's principal from the account that made it,
while it is active and still in the project the job is charged to.
ctl 3d list|get|cancel, over /api/v1/client-jobs (/api/v1/jobs is the
control plane's own work, which ctl jobs shows).
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
239 lines
7.7 KiB
Go
239 lines
7.7 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"time"
|
|
)
|
|
|
|
// Job is one request that is answered with an id and asked after: work that
|
|
// takes minutes, which no client should hold a connection for. The request
|
|
// is kept as it was sent and replayed by whichever gateway holds the lease.
|
|
type Job struct {
|
|
ID string
|
|
Owner string
|
|
KeyID string
|
|
Endpoint string
|
|
Model string
|
|
// Input is the request's body, as the client sent it.
|
|
Input []byte
|
|
Status string
|
|
// Progress is what the engine said of the work so far, as JSON; empty
|
|
// until it said anything.
|
|
Progress string
|
|
// PromptID is the engine's own id for the work, so a gateway that takes
|
|
// over a running job asks after the same work rather than queueing it
|
|
// again.
|
|
PromptID string
|
|
OutputFileID string
|
|
ContentType string
|
|
Error string
|
|
CreatedAt time.Time
|
|
StartedAt time.Time
|
|
FinishedAt time.Time
|
|
// ExpiresAt is when a job not yet finished is given up on.
|
|
ExpiresAt time.Time
|
|
}
|
|
|
|
// Job states.
|
|
const (
|
|
JobQueued = "queued"
|
|
JobRunning = "running"
|
|
JobCompleted = "completed"
|
|
JobFailed = "failed"
|
|
JobCancelling = "cancelling"
|
|
JobCancelled = "cancelled"
|
|
JobExpired = "expired"
|
|
)
|
|
|
|
// JobDone is whether a state is an end.
|
|
func JobDone(status string) bool {
|
|
switch status {
|
|
case JobCompleted, JobFailed, JobCancelled, JobExpired:
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
const jobCols = `id, owner, key_id, endpoint, model, input, status, progress, prompt_id, output_file_id,
|
|
content_type, error, created_at, started_at, finished_at, expires_at`
|
|
|
|
func scanJob(sc interface{ Scan(...any) error }) (Job, error) {
|
|
var j Job
|
|
var created, started, finished, expires int64
|
|
err := sc.Scan(&j.ID, &j.Owner, &j.KeyID, &j.Endpoint, &j.Model, &j.Input, &j.Status, &j.Progress, &j.PromptID,
|
|
&j.OutputFileID, &j.ContentType, &j.Error, &created, &started, &finished, &expires)
|
|
if err != nil {
|
|
return Job{}, err
|
|
}
|
|
at := func(v int64) time.Time {
|
|
if v == 0 {
|
|
return time.Time{}
|
|
}
|
|
return time.UnixMilli(v)
|
|
}
|
|
j.CreatedAt, j.StartedAt, j.FinishedAt, j.ExpiresAt = at(created), at(started), at(finished), at(expires)
|
|
return j, nil
|
|
}
|
|
|
|
// PutJob records a new job.
|
|
func (s *DB) PutJob(ctx context.Context, j Job) error {
|
|
if j.Input == nil {
|
|
j.Input = []byte{}
|
|
}
|
|
_, err := s.db.ExecContext(ctx, `INSERT INTO jobs (id, owner, key_id, endpoint, model, input, status, created_at, expires_at)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
j.ID, j.Owner, j.KeyID, j.Endpoint, j.Model, j.Input, j.Status, millis(j.CreatedAt), millis(j.ExpiresAt))
|
|
return err
|
|
}
|
|
|
|
// UpdateJob writes how a job stands: its state, result, error and times.
|
|
// Its progress and prompt id are written by SetJobProgress, which the work
|
|
// calls often and which must not undo an end written here.
|
|
func (s *DB) UpdateJob(ctx context.Context, j Job) error {
|
|
_, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = ?, output_file_id = ?, content_type = ?, error = ?,
|
|
started_at = ?, finished_at = ? WHERE id = ?`,
|
|
j.Status, j.OutputFileID, j.ContentType, j.Error, optMillis(j.StartedAt), optMillis(j.FinishedAt), j.ID)
|
|
return err
|
|
}
|
|
|
|
// SetJobProgress writes what the engine said of a running job, and its id
|
|
// for the work when it is known. A job that has ended keeps its end.
|
|
func (s *DB) SetJobProgress(ctx context.Context, id, progress, promptID string) error {
|
|
_, err := s.db.ExecContext(ctx, `UPDATE jobs SET progress = ?,
|
|
prompt_id = CASE WHEN ? = '' THEN prompt_id ELSE ? END
|
|
WHERE id = ? AND status IN (?, ?)`, progress, promptID, promptID, id, JobRunning, JobCancelling)
|
|
return err
|
|
}
|
|
|
|
// CancelJob asks for a job to stop: cancelling, unless it has ended already.
|
|
// It says the job as it stands.
|
|
func (s *DB) CancelJob(ctx context.Context, id string, now time.Time) (Job, error) {
|
|
// One not yet taken ends at once: there is nothing to stop.
|
|
if _, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = ?, finished_at = ?
|
|
WHERE id = ? AND status = ? AND lease_until < ?`, JobCancelled, millis(now), id, JobQueued, millis(now)); err != nil {
|
|
return Job{}, err
|
|
}
|
|
if _, err := s.db.ExecContext(ctx, `UPDATE jobs SET status = ? WHERE id = ? AND status IN (?, ?)`,
|
|
JobCancelling, id, JobQueued, JobRunning); err != nil {
|
|
return Job{}, err
|
|
}
|
|
return s.JobByID(ctx, id)
|
|
}
|
|
|
|
// JobByID finds a job, ErrNotFound when there is none.
|
|
func (s *DB) JobByID(ctx context.Context, id string) (Job, error) {
|
|
j, err := scanJob(s.db.QueryRowContext(ctx, `SELECT `+jobCols+` FROM jobs WHERE id = ?`, id))
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return Job{}, ErrNotFound
|
|
}
|
|
return j, err
|
|
}
|
|
|
|
// Jobs lists an owner's jobs, the newest first, those made before the job
|
|
// after when it is given.
|
|
func (s *DB) Jobs(ctx context.Context, owner, after string, limit int) ([]Job, error) {
|
|
q := `SELECT ` + jobCols + ` FROM jobs WHERE owner = ?`
|
|
args := []any{owner}
|
|
if after != "" {
|
|
q += ` AND created_at < (SELECT created_at FROM jobs WHERE id = ?)`
|
|
args = append(args, after)
|
|
}
|
|
if limit <= 0 || limit > 100 {
|
|
limit = 20
|
|
}
|
|
q += ` ORDER BY created_at DESC LIMIT ?`
|
|
args = append(args, limit)
|
|
rows, err := s.db.QueryContext(ctx, q, args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var out []Job
|
|
for rows.Next() {
|
|
j, err := scanJob(rows)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, j)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
// AllJobs is every owner's jobs, newest first, those in one status when it
|
|
// is named: what an operator looks over.
|
|
func (s *DB) AllJobs(ctx context.Context, status string, limit int) ([]Job, error) {
|
|
q := `SELECT ` + jobCols + ` FROM jobs`
|
|
var args []any
|
|
if status != "" {
|
|
q += ` WHERE status = ?`
|
|
args = append(args, status)
|
|
}
|
|
if limit <= 0 || limit > 500 {
|
|
limit = 50
|
|
}
|
|
q += ` ORDER BY created_at DESC LIMIT ?`
|
|
args = append(args, limit)
|
|
rows, err := s.db.QueryContext(ctx, q, args...)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
var out []Job
|
|
for rows.Next() {
|
|
j, err := scanJob(rows)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out = append(out, j)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
// ClaimJob takes the lease on the oldest job with work left that no gateway
|
|
// holds, for owner until the time given. ok is false when there is none.
|
|
func (s *DB) ClaimJob(ctx context.Context, owner string, now, until time.Time) (Job, bool, error) {
|
|
var id string
|
|
err := s.db.QueryRowContext(ctx, `UPDATE jobs SET lease_owner = ?, lease_until = ?
|
|
WHERE id = (SELECT id FROM jobs WHERE status IN (?, ?, ?) AND lease_until < ?
|
|
ORDER BY created_at LIMIT 1)
|
|
AND lease_until < ?
|
|
RETURNING id`, owner, millis(until), JobQueued, JobRunning, JobCancelling, millis(now), millis(now)).Scan(&id)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return Job{}, false, nil
|
|
}
|
|
if err != nil {
|
|
return Job{}, false, err
|
|
}
|
|
j, err := s.JobByID(ctx, id)
|
|
return j, err == nil, err
|
|
}
|
|
|
|
// RenewJobLease keeps a lease held; false when it is held no longer.
|
|
func (s *DB) RenewJobLease(ctx context.Context, id, owner string, until time.Time) (bool, error) {
|
|
res, err := s.db.ExecContext(ctx, `UPDATE jobs SET lease_until = ? WHERE id = ? AND lease_owner = ?`,
|
|
millis(until), id, owner)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
n, _ := res.RowsAffected()
|
|
return n > 0, nil
|
|
}
|
|
|
|
// ReleaseJob gives a lease up.
|
|
func (s *DB) ReleaseJob(ctx context.Context, id, owner string) error {
|
|
_, err := s.db.ExecContext(ctx, `UPDATE jobs SET lease_until = 0, lease_owner = '' WHERE id = ? AND lease_owner = ?`, id, owner)
|
|
return err
|
|
}
|
|
|
|
// DeleteJobsBefore forgets the jobs that ended before the time given, whose
|
|
// result files are gone by then too.
|
|
func (s *DB) DeleteJobsBefore(ctx context.Context, before time.Time) (int64, error) {
|
|
res, err := s.db.ExecContext(ctx, `DELETE FROM jobs WHERE finished_at > 0 AND finished_at < ?`, millis(before))
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
return res.RowsAffected()
|
|
}
|